Spider/bulk-downloader-for-reddit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add option to specify wait time as option

Browse Source
This commit is contained in:
Serene-Arc 2021-04-12 17:12:41 +10:00 committed by Ali Parlakci
parent af95f3ff66
commit 36291d5ea6
3 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bulkredditdownloader/__main__.py
View File

@ -48,6 +48,7 @@ def cli():
@click.option('--file-scheme', default=None, type=str)
@click.option('--folder-scheme', default=None, type=str)
@click.option('--make-hard-links', is_flag=True, default=None)
@click.option('--max-wait-time', type=int, default=None)
@click.option('--no-dupes', is_flag=True, default=None)
@click.option('--search-existing', is_flag=True, default=None)
@click.option('--skip', default=None, multiple=True)

1
bulkredditdownloader/configuration.py
View File

@ -17,6 +17,7 @@ class Configuration(Namespace):
self.exclude_id_file = []
self.limit: Optional[int] = None
self.link: list[str] = []
self.max_wait_time = None
self.multireddit: list[str] = []
self.no_dupes: bool = False
self.saved: bool = False

16
bulkredditdownloader/downloader.py
View File

@ -70,6 +70,8 @@ class RedditDownloader:
self._load_config()
self._create_file_logger()
self._read_config()
self.download_filter = self._create_download_filter()
logger.log(9, 'Created download filter')
self.time_filter = self._create_time_filter()
@ -91,6 +93,18 @@ class RedditDownloader:
self.authenticator = self._create_authenticator()
logger.log(9, 'Created site authenticator')
def _read_config(self):
"""Read any cfg values that need to be processed"""
if self.args.max_wait_time is None:
if not self.cfg_parser.has_option('DEFAULT', 'max_wait_time'):
self.cfg_parser.set('DEFAULT', 'max_wait_time', '120')
logger.log(9, 'Wrote default download wait time download to config file')
self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time')
logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds')
# Update config on disk
with open(self.config_location, 'w') as file:
self.cfg_parser.write(file)
def _create_reddit_instance(self):
if self.args.authenticate:
logger.debug('Using authenticated Reddit instance')
@ -371,7 +385,7 @@ class RedditDownloader:
logger.debug(f'File {destination} already exists, continuing')
else:
try:
res.download(self.cfg_parser.getint('DEFAULT', 'max_wait_time', fallback=120))
res.download(self.args.max_wait_time)
except errors.BulkDownloaderException as e:
logger.error(
f'Failed to download resource {res.url} with downloader {downloader_class.__name__}: {e}')