From 47a49512798d0a32642eec37dc9c880398da9e6e Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 23 May 2021 12:13:44 +1000 Subject: [PATCH 1/3] Rename variable --- bdfr/downloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bdfr/downloader.py b/bdfr/downloader.py index cc4e8bb..4a13823 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -22,13 +22,13 @@ logger = logging.getLogger(__name__) def _calc_hash(existing_file: Path): - CHUNK_SIZE = 1024 * 1024 + chunk_size = 1024 * 1024 md5_hash = hashlib.md5() with open(existing_file, 'rb') as file: - chunk = file.read(CHUNK_SIZE) + chunk = file.read(chunk_size) while chunk: md5_hash.update(chunk) - chunk = file.read(CHUNK_SIZE) + chunk = file.read(chunk_size) file_hash = md5_hash.hexdigest() return existing_file, file_hash From e2582ecb3eafd8df20bbb9b616669b71131ecc97 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 23 May 2021 12:17:14 +1000 Subject: [PATCH 2/3] Catch error with MacOS writing per issue #407 --- bdfr/downloader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 4a13823..a262dae 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -94,9 +94,13 @@ class RedditDownloader(RedditConnector): f'Hard link made linking {destination} to {self.master_hash_list[resource_hash]}' f' in submission {submission.id}') return - with open(destination, 'wb') as file: - file.write(res.content) - logger.debug(f'Written file to {destination}') + try: + with open(destination, 'wb') as file: + file.write(res.content) + logger.debug(f'Written file to {destination}') + except OSError as e: + logger.exception(e) + logger.error(f'Failed to write file to {destination} in submission {submission.id}: {e}') creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple()) os.utime(destination, (creation_time, creation_time)) self.master_hash_list[resource_hash] = destination From 1b23e38ce4793c85d76bf8f551b772010dc03fd5 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 23 May 2021 12:25:04 +1000 Subject: [PATCH 3/3] Update script to include new message --- scripts/extract_failed_ids.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/extract_failed_ids.sh b/scripts/extract_failed_ids.sh index 7108592..89f1896 100755 --- a/scripts/extract_failed_ids.sh +++ b/scripts/extract_failed_ids.sh @@ -18,4 +18,5 @@ fi grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ; grep 'Failed to download resource' "$file" | awk '{ print $15 }' ; grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ; + grep 'Failed to write file' "$file" | awk '{ print $16 }' | rev | cut -c 2- | rev ; } >>"$output"