Spider/bulk-downloader-for-reddit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #601 from Serene-Arc/bug_fix_598

Browse Source
This commit is contained in:
Serene 2022-02-18 12:09:29 +10:00 committed by GitHub
commit 5adb9f9545
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 57 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
bdfr/site_downloaders/redgifs.py
View File

@ -17,11 +17,11 @@ class Redgifs(BaseDownloader):
super().__init__(post)
def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
media_url = self._get_link(self.post.url)
return [Resource(self.post, media_url, Resource.retry_download(media_url), '.mp4')]
media_urls = self._get_link(self.post.url)
return [Resource(self.post, m, Resource.retry_download(m), '.mp4') for m in media_urls]
@staticmethod
def _get_link(url: str) -> str:
def _get_link(url: str) -> set[str]:
try:
redgif_id = re.match(r'.*/(.*?)/?$', url).group(1)
except AttributeError:
@ -32,16 +32,33 @@ class Redgifs(BaseDownloader):
'Chrome/90.0.4430.93 Safari/537.36',
}
content = Redgifs.retrieve_url(f'https://api.redgifs.com/v1/gfycats/{redgif_id}', headers=headers)
content = Redgifs.retrieve_url(f'https://api.redgifs.com/v2/gifs/{redgif_id}', headers=headers)
if content is None:
raise SiteDownloaderError('Could not read the page source')
try:
out = json.loads(content.text)['gfyItem']['mp4Url']
except (KeyError, AttributeError):
raise SiteDownloaderError('Failed to find JSON data in page')
response_json = json.loads(content.text)
except json.JSONDecodeError as e:
raise SiteDownloaderError(f'Received data was not valid JSON: {e}')
out = set()
try:
if response_json['gif']['type'] == 1: # type 1 is a video
out.add(response_json['gif']['urls']['hd'])
elif response_json['gif']['type'] == 2: # type 2 is an image
if response_json['gif']['gallery']:
content = Redgifs.retrieve_url(
f'https://api.redgifs.com/v2/gallery/{response_json["gif"]["gallery"]}',
headers=headers,
)
response_json = json.loads(content.text)
out = {p['urls']['hd'] for p in response_json['gifs']}
else:
out.add(response_json['gif']['urls']['hd'])
else:
raise KeyError
except (KeyError, AttributeError):
raise SiteDownloaderError('Failed to find JSON data in page')
return out

48
tests/site_downloaders/test_redgifs.py
View File

@ -12,30 +12,48 @@ from bdfr.site_downloaders.redgifs import Redgifs
@pytest.mark.online
@pytest.mark.parametrize(('test_url', 'expected'), (
('https://redgifs.com/watch/frighteningvictorioussalamander',
'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'),
{'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'}),
('https://redgifs.com/watch/springgreendecisivetaruca',
'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'),
{'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'}),
('https://www.redgifs.com/watch/palegoldenrodrawhalibut',
'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'),
{'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'}),
('https://redgifs.com/watch/hollowintentsnowyowl',
{'https://thumbs2.redgifs.com/HollowIntentSnowyowl-large.jpg'}),
('https://www.redgifs.com/watch/lustrousstickywaxwing',
{'https://thumbs2.redgifs.com/EntireEnchantingHypsilophodon-large.jpg',
'https://thumbs2.redgifs.com/FancyMagnificentAdamsstaghornedbeetle-large.jpg',
'https://thumbs2.redgifs.com/LustrousStickyWaxwing-large.jpg',
'https://thumbs2.redgifs.com/ParchedWindyArmyworm-large.jpg',
'https://thumbs2.redgifs.com/ThunderousColorlessErmine-large.jpg',
'https://thumbs2.redgifs.com/UnripeUnkemptWoodpecker-large.jpg'}),
))
def test_get_link(test_url: str, expected: str):
def test_get_link(test_url: str, expected: set[str]):
result = Redgifs._get_link(test_url)
assert result == expected
@pytest.mark.online
@pytest.mark.parametrize(('test_url', 'expected_hash'), (
('https://redgifs.com/watch/frighteningvictorioussalamander', '4007c35d9e1f4b67091b5f12cffda00a'),
('https://redgifs.com/watch/springgreendecisivetaruca', '8dac487ac49a1f18cc1b4dabe23f0869'),
('https://redgifs.com/watch/leafysaltydungbeetle', '076792c660b9c024c0471ef4759af8bd'),
('https://www.redgifs.com/watch/palegoldenrodrawhalibut', '46d5aa77fe80c6407de1ecc92801c10e'),
@pytest.mark.parametrize(('test_url', 'expected_hashes'), (
('https://redgifs.com/watch/frighteningvictorioussalamander', {'4007c35d9e1f4b67091b5f12cffda00a'}),
('https://redgifs.com/watch/springgreendecisivetaruca', {'8dac487ac49a1f18cc1b4dabe23f0869'}),
('https://redgifs.com/watch/leafysaltydungbeetle', {'076792c660b9c024c0471ef4759af8bd'}),
('https://www.redgifs.com/watch/palegoldenrodrawhalibut', {'46d5aa77fe80c6407de1ecc92801c10e'}),
('https://redgifs.com/watch/hollowintentsnowyowl',
{'5ee51fa15e0a58e98f11dea6a6cca771'}),
('https://www.redgifs.com/watch/lustrousstickywaxwing',
{'b461e55664f07bed8d2f41d8586728fa',
'30ba079a8ed7d7adf17929dc3064c10f',
'0d4f149d170d29fc2f015c1121bab18b',
'53987d99cfd77fd65b5fdade3718f9f1',
'fb2e7d972846b83bf4016447d3060d60',
'44fb28f72ec9a5cca63fa4369ab4f672'}),
))
def test_download_resource(test_url: str, expected_hash: str):
def test_download_resource(test_url: str, expected_hashes: set[str]):
mock_submission = Mock()
mock_submission.url = test_url
test_site = Redgifs(mock_submission)
resources = test_site.find_resources()
assert len(resources) == 1
assert isinstance(resources[0], Resource)
resources[0].download()
assert resources[0].hash.hexdigest() == expected_hash
results = test_site.find_resources()
assert all([isinstance(res, Resource) for res in results])
[res.download() for res in results]
hashes = set([res.hash.hexdigest() for res in results])
assert hashes == set(expected_hashes)