Processor.App.Downloader.downloader.DownloaderFull#

class Processor.App.Downloader.downloader.DownloaderFull(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 3)#
__init__(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 3)#

Methods

__init__([base_url, digest_verification, ...])

aclose(exc_type, exc_val[, exc_tb])

aopen()

download(domain_record)

unwrap(response, domain_record)