This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Download handler for HTTP/2 supporting proxy CONNECT over HTTP/1. | |
You can use this by configuring it in the spider settings: | |
DOWNLOAD_HANDLERS = { | |
'https': 'scrapy_h2_proxy.H2DownloadHandler', | |
} | |
Tested with Scrapy 2.5.0. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse, json, logging | |
import apache_beam as beam | |
import apache_beam.pvalue as pvalue | |
from apache_beam.io import ReadFromText | |
from apache_beam.options.pipeline_options import PipelineOptions | |
from apache_beam.options.pipeline_options import SetupOptions | |
class EnrichElementsFn(beam.DoFn): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse, json, logging, time | |
import apache_beam as beam | |
import apache_beam.transforms.window as window | |
from apache_beam.io import filesystems | |
from apache_beam.options.pipeline_options import PipelineOptions | |
from apache_beam.options.pipeline_options import SetupOptions | |
class AddWindowingInfoFn(beam.DoFn): |