NLP/wikiextractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix NameError #225.

Browse Source
This commit is contained in:
attardi 2020-12-04 18:43:22 +01:00
parent 8ef37c87e2
commit 2ba214ab99
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
wikiextractor/WikiExtractor.py
View File

@ -49,7 +49,6 @@ collecting template definitions.
import argparse import argparse
import bz2 import bz2
import codecs
import fileinput import fileinput
import logging import logging
import os.path import os.path
@ -198,10 +197,11 @@ def load_templates(file, output_file=None):
global moduleNamespace, modulePrefix global moduleNamespace, modulePrefix
modulePrefix = moduleNamespace + ':' modulePrefix = moduleNamespace + ':'
articles = 0 articles = 0
templates = 0
page = [] page = []
inText = False inText = False
if output_file: if output_file:
output = codecs.open(output_file, 'wb', 'utf-8') output = open(output_file, 'wb')
for line in file: for line in file:
line = line.decode('utf-8') line = line.decode('utf-8')
if '<' not in line: # faster than doing re.search() if '<' not in line: # faster than doing re.search()
@ -249,13 +249,14 @@ def load_templates(file, output_file=None):
output.write(line) output.write(line)
output.write(' </text>\n') output.write(' </text>\n')
output.write('</page>\n') output.write('</page>\n')
templates += 1
page = [] page = []
articles += 1 articles += 1
if articles % 100000 == 0: if articles % 100000 == 0:
logging.info("Preprocessed %d pages", articles) logging.info("Preprocessed %d pages", articles)
if output_file: if output_file:
output.close() output.close()
logging.info("Saved %d templates to '%s'", len(templates), output_file) logging.info("Saved %d templates to '%s'", templates, output_file)
def process_dump(input_file, template_file, out_file, file_size, file_compress, def process_dump(input_file, template_file, out_file, file_size, file_compress,