Thesis/ADNC
1
0
Fork 0
mirror of https://github.com/JoergFranke/ADNC.git synced 2024-11-17 13:58:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

update cnn data load

Browse Source
This commit is contained in:
Joerg Franke 2018-07-05 00:15:47 +02:00
parent f8df19541b
commit 98f56912f3
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
adnc/data/tasks/cnn_rc.py
View File

@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================== # ==============================================================================
import os
import operator import operator
import pathlib import pathlib
import tarfile import tarfile
import pickle
from collections import Counter, OrderedDict from collections import Counter, OrderedDict
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
@ -249,6 +251,15 @@ class ReadingComprehension():
return x_word, prediction_decode return x_word, prediction_decode
def save_dictionary(self, dir):
with open(os.path.join(dir, 'word_idx_dict.pkl'), 'wb') as outfile:
pickle.dump(self.word_idx_dict, outfile)
def load_dictionary(self, dir):
with open(os.path.join(dir, 'word_idx_dict.pkl'), 'rb') as outfile:
self.word_idx_dict = pickle.load(outfile)
self.idx_word_dict = {v:k for k,v in self.word_idx_dict.items()}
@property @property
def vocabulary_size(self): def vocabulary_size(self):
return self.word_idx_dict.__len__() return self.word_idx_dict.__len__()