NLP/wikiextractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #100 from orangain/fix-oom-in-single-core

Browse Source 
Ensure that process_count >= 1
This commit is contained in:
Giuseppe Attardi 2017-01-15 14:38:19 +01:00 committed by GitHub
commit b5d97f64f9
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
WikiExtractor.py
View File

@ -2762,6 +2762,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
# - pages to be processed are dispatched to workers # - pages to be processed are dispatched to workers
# - a reduce process collects the results, sort them and print them. # - a reduce process collects the results, sort them and print them.
process_count = max(1, process_count)
maxsize = 10 * process_count maxsize = 10 * process_count
# output queue # output queue
output_queue = Queue(maxsize=maxsize) output_queue = Queue(maxsize=maxsize)
@ -2769,7 +2770,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
if out_file == '-': if out_file == '-':
out_file = None out_file = None
worker_count = max(1, process_count) worker_count = process_count
# load balancing # load balancing
max_spool_length = 10000 max_spool_length = 10000
@ -2963,7 +2964,7 @@ def main():
help="Minimum expanded text length required to write document (default=%(default)s)") help="Minimum expanded text length required to write document (default=%(default)s)")
groupP.add_argument("--filter_disambig_pages", action="store_true", default=filter_disambig_pages, groupP.add_argument("--filter_disambig_pages", action="store_true", default=filter_disambig_pages,
help="Remove pages from output that contain disabmiguation markup (default=%(default)s)") help="Remove pages from output that contain disabmiguation markup (default=%(default)s)")
default_process_count = cpu_count() - 1 default_process_count = max(1, cpu_count() - 1)
parser.add_argument("--processes", type=int, default=default_process_count, parser.add_argument("--processes", type=int, default=default_process_count,
help="Number of processes to use (default %(default)s)") help="Number of processes to use (default %(default)s)")