Spider/bulk-downloader-for-reddit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Check submission URL against filter before factory

Browse Source
This commit is contained in:
Serene-Arc 2021-06-23 14:30:39 +10:00
parent 71930e06a8
commit e5be624f1e
2 changed files with 18 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bdfr/downloader.py
View File

@ -54,6 +54,9 @@ class RedditDownloader(RedditConnector):
elif not isinstance(submission, praw.models.Submission):
logger.warning(f'{submission.id} is not a submission')
return
elif not self.download_filter.check_url(submission.url):
logger.debug(f'Submission {submission.id} filtered due to URL {submission.url}')
return
logger.debug(f'Attempting to download submission {submission.id}')
try:
@ -76,7 +79,7 @@ class RedditDownloader(RedditConnector):
logger.debug(f'File {destination} from submission {submission.id} already exists, continuing')
continue
elif not self.download_filter.check_resource(res):
logger.debug(f'Download filter removed {submission.id} with URL {submission.url}')
logger.debug(f'Download filter removed {submission.id} file with URL {submission.url}')
continue
try:
res.download(self.args.max_wait_time)

14
tests/test_integration.py
View File

@ -189,6 +189,20 @@ def test_cli_download_download_filters(test_args: list[str], tmp_path: Path):
assert 'Download filter removed ' in result.output
@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests')
@pytest.mark.parametrize('test_args', (
['--subreddit', 'tumblr', '-L', '10', '--skip-domain', 'i.redd.it'],
))
def test_cli_download_download_filter_domain(test_args: list[str], tmp_path: Path):
runner = CliRunner()
test_args = create_basic_args_for_download_runner(test_args, tmp_path)
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert 'filtered due to URL' in result.output
@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.slow