暂未解决数据问题

ml/InverseIndex.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*- 
 import jieba
+import jieba.analyse as analyse
 import json
 import sys
 reload(sys)
@@ -70,10 +71,11 @@ class InverseIndex:
 			if not line:
 				break
 			data = json.loads(line)
-			seg_list = jieba.cut(data['content'],cut_all=True)
-			for seg in seg_list:
-				seg=''.join(seg.split())
-				if(seg!='' and seg!="\n" and seg!="\n\n"):
+#	seg_list = jieba.cut(data['content'],cut_all=True)
+
+			keyword = analyse.extract_tags(data['content'],topK=10)
+			seg = " ".join(keyword)
+			print seg
 			doc.append(seg)
 		return doc
 
@@ -89,6 +91,7 @@ class InverseIndex:
 		i = 0
 		indexdoc = dict()
 		f = open(Global.inverse_dir+'id.txt','wb')
+		word = vectorizer.get_feature_names()
 		for name in vectorizer.get_feature_names():
 			indexdoc[name] = i
 			i+=1
@@ -99,8 +102,8 @@ class InverseIndex:
 		row = tfidf.shape[0]
 		for i in range(0,colnum):
 			filename = Global.inverse_dir+str(i/Global.filesize)+'.txt'
-			f = open(filename,'a'
-			idx_list = dict()
+			f = open(filename,'a')
+			idx_list = list()
 			for j in range(0,row):
 				val = tfidf[j,i]
 				if val > 0:
@@ -116,3 +119,4 @@ class InverseIndex:
 #test
 ii = InverseIndex()
 ii.CalcTFIDF()
+#ii.loadDataFromCutFile(20)

test/test_tool.py

@@ -1,8 +1,16 @@
 import sys
 sys.path.append("..")
 from tools.show import show
+import tools.Global as Global
+from ml.Cut import Cut
 
 s = show()
 #s.showcount()
 #s.shownews(1)
-s.showKeyWord()
+#s.showKeyWord()
+#s.showitem(2608)
+
+c = Cut()
+line = c.getRow(3176,Global.cutnews_dir,Global.filesize)
+s.showitem(line)
+