111 lines
4.3 KiB
Python
111 lines
4.3 KiB
Python
|
# encoding:utf-8
|
|||
|
|
"""
|
|||
|
|
@author = 'XXY'
|
|||
|
|
@contact = '529379497@qq.com'
|
|||
|
|
@researchFie1d = 'NLP DL ML'
|
|||
|
|
@date= '2017/12/21 10:18'
|
|||
|
|
"""
|
|||
|
|
import json, os
|
|||
|
|
import jieba.posseg as pseg
|
|||
|
|
import logging
|
|||
|
|
from data_helper import *
|
|||
|
|
import numpy as np
|
|||
|
|
from sklearn.model_selection import train_test_split
|
|||
|
|
import pandas as pd
|
|||
|
|
import tensorflow as tf
|
|||
|
|
from tensorflow.contrib import learn
|
|||
|
|
from sklearn.metrics import classification_report, accuracy_score
|
|||
|
|
|
|||
|
|
logging.getLogger().setLevel(logging.INFO)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def make_submission(file, prediction, encoding):
|
|||
|
|
valid_id = []
|
|||
|
|
label2int = {u"人类作者": 0, u"机器作者": 1, u"机器翻译": 2, u"自动摘要": 3}
|
|||
|
|
int2label = {0.0: u"人类作者", 1.0: u"机器作者", 2.0: u"机器翻译", 3.0: u"自动摘要"}
|
|||
|
|
|
|||
|
|
for line in open('./data/validation.txt'):
|
|||
|
|
text = json.loads(line.strip())
|
|||
|
|
valid_id.append(text['id'])
|
|||
|
|
result = pd.DataFrame({'id': valid_id, 'label': prediction})
|
|||
|
|
result['label'] = result['label'].apply(lambda x: int2label[x])
|
|||
|
|
print(result.head())
|
|||
|
|
result.to_csv(file, header=None, index=None, encoding=encoding)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def predict_unseen_data():
|
|||
|
|
X_char = []
|
|||
|
|
X_punc = []
|
|||
|
|
y = []
|
|||
|
|
|
|||
|
|
# 读取字符特征
|
|||
|
|
print "读取char特征"
|
|||
|
|
with open('./data/validation_char.txt') as f:
|
|||
|
|
for line in f:
|
|||
|
|
temp = line.strip().split('\t')
|
|||
|
|
text = temp[0][1:-1].split(',')
|
|||
|
|
label = temp[1]
|
|||
|
|
X_char.append(text)
|
|||
|
|
y.append(label)
|
|||
|
|
|
|||
|
|
# 读取标点符号结构特征
|
|||
|
|
print "读取punc特征"
|
|||
|
|
with open('./data/validation_punc.txt') as f:
|
|||
|
|
for line in f:
|
|||
|
|
temp = line.strip().split('\t')
|
|||
|
|
text = temp[0][1:-1].split(',')
|
|||
|
|
X_punc.append(text)
|
|||
|
|
|
|||
|
|
print "读取全连接层特征"
|
|||
|
|
X_validation_fc_feat = pd.read_csv('./data/validation_fc_feat_norm_200.txt', sep=',')
|
|||
|
|
print"数据加载完毕！"
|
|||
|
|
|
|||
|
|
|
|||
|
|
X_validation_char = np.array(X_char)
|
|||
|
|
X_validation_punc = np.array(X_punc)
|
|||
|
|
X_validation_fc_feat = X_validation_fc_feat.values
|
|||
|
|
params = json.loads(open('./config/cnn_parameters.json').read())
|
|||
|
|
print "validation数据大小：", X_validation_char.shape, X_validation_punc.shape
|
|||
|
|
print "validation集加载完毕！"
|
|||
|
|
|
|||
|
|
checkpoint_dir = 'models/cnn_models/trained_model_1528419951/'
|
|||
|
|
if not checkpoint_dir.endswith('/'):
|
|||
|
|
checkpoint_dir += '/'
|
|||
|
|
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir + 'checkpoints') # 加载最近保存的模型
|
|||
|
|
# checkpoint_file = '/home/h325/data/Xxy/SMP_NEW/models/cnn_models/trained_model_1528257857/checkpoints/model-12400'
|
|||
|
|
logging.critical('Loaded the trained model: {}'.format(checkpoint_file))
|
|||
|
|
|
|||
|
|
graph = tf.Graph()
|
|||
|
|
with graph.as_default():
|
|||
|
|
session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
|
|||
|
|
sess = tf.Session(config=session_conf)
|
|||
|
|
|
|||
|
|
with sess.as_default():
|
|||
|
|
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
|
|||
|
|
saver.restore(sess, checkpoint_file)
|
|||
|
|
input_x_char = graph.get_operation_by_name("input/input_x_char").outputs[0]
|
|||
|
|
input_x_punc = graph.get_operation_by_name("input/input_x_punc").outputs[0]
|
|||
|
|
input_x_fc_feat = graph.get_operation_by_name('input/input_x_fc_feat').outputs[0]
|
|||
|
|
dropout_keep_prob = graph.get_operation_by_name("dropout/dropout_keep_prob").outputs[0]
|
|||
|
|
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
|
|||
|
|
|
|||
|
|
dev_predictions = []
|
|||
|
|
for i in range(int(len(X_validation_char) / params['batch_size']) + 1):
|
|||
|
|
start_index = i * params['batch_size']
|
|||
|
|
end_index = min((i + 1) * params['batch_size'], len(X_validation_char))
|
|||
|
|
X_validation_char_batch = X_validation_char[start_index: end_index]
|
|||
|
|
X_validation_punc_batch = X_validation_punc[start_index: end_index]
|
|||
|
|
X_validation_fc_feat_batch = X_validation_fc_feat[start_index: end_index]
|
|||
|
|
prediction= sess.run(predictions, {input_x_char: X_validation_char_batch, input_x_punc:X_validation_punc_batch,
|
|||
|
|
input_x_fc_feat:X_validation_fc_feat_batch, dropout_keep_prob: 1.0})
|
|||
|
|
dev_predictions = np.concatenate([dev_predictions, prediction])
|
|||
|
|
|
|||
|
|
# for test_batch in test_batches:
|
|||
|
|
# X_test_batch = test_batch
|
|||
|
|
# prediction= sess.run(predictions, {input_x1: X_test_batch, dropout_keep_prob: 1.0})
|
|||
|
|
# dev_predictions = np.concatenate([dev_predictions, prediction])
|
|||
|
|
make_submission('results/cnn_char_punc_fc_feat_result4.csv', dev_predictions, encoding='utf-8')
|
|||
|
|
logging.critical('The prediction is complete')
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
predict_unseen_data()
|