NLP/wikiextractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

encoding fix

Browse Source
This commit is contained in:
root 2017-04-28 02:04:29 -04:00
parent 1f76fd9473
commit b8323a8efc
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
WikiExtractor.py
View File

@ -3246,20 +3246,21 @@ def main():
with open(filter_category) as f:
for line in f.readlines():
try:
line = line.strip().encode('utf-8')
line = str(line.strip())
if line.startswith('#') or len(line) == 0:
continue;
elif line.startswith('^'):
options.filter_category_exclude.add(line.lstrip('^'))
else:
options.filter_category_include.add(line)
except:
except Exception as e:
print(e)
print(u"Category not in utf8, ignored:\t")
print(line)
print("Excluding categories:",)
print(str(options.filter_category_exclude))
print("Including categories:")
print(str(options.filter_category_include))
print(str(len(options.filter_category_include)))
process_dump(input_file, args.templates, output_path, file_size,
args.compress, args.processes)