NLP/wikiextractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add encoding parameter in load_templates

Browse Source
This commit is contained in:
Gaëtan Caillaut 2021-11-05 14:29:47 +01:00
parent f4f9d2d008
commit eabef5cf7b
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wikiextractor/WikiExtractor.py
View File

@ -191,7 +191,7 @@ tagRE = re.compile(r'(.*?)<(/?\w+)[^>]*>(?:([^<]*)(<.*?>)?)?')
# 1 2 3 4
def load_templates(file, output_file=None):
def load_templates(file, output_file=None, encoding='utf-8'):
"""
Load templates from :param file:.
:param output_file: file where to save templates and modules.
@ -205,7 +205,7 @@ def load_templates(file, output_file=None):
page = []
inText = False
if output_file:
output = open(output_file, 'w')
output = open(output_file, 'w', encoding=encoding)
for line in file:
#line = line.decode('utf-8')
if '<' not in line: # faster than doing re.search()