From c1b54b4b4e4c1dd9982fcba742306b4a975c6201 Mon Sep 17 00:00:00 2001
From: mxd <838425805@qq.com>
Date: Sun, 10 Nov 2019 17:38:36 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=A4=BA=E4=BE=8Bdemo?=
=?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=90=8D=E7=A7=B0=E9=94=99=E8=AF=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
db/spiderflow.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/db/spiderflow.sql b/db/spiderflow.sql
index 4dff72a..0fdc69b 100644
--- a/db/spiderflow.sql
+++ b/db/spiderflow.sql
@@ -19,7 +19,7 @@ CREATE TABLE `sp_flow` (
INSERT INTO `sp_flow` VALUES ('b45fb98d2a564c23ba623a377d5e12e9', '爬取码云GVP', '\n \n \n \n {"spiderName":"爬取码云GVP","threadCount":""}\n \n \n \n \n \n \n {"shape":"start"}\n \n \n \n \n \n {"value":"抓取首页","loopVariableName":"","sleep":"","timeout":"","response-charset":"","method":"GET","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"https://gitee.com/gvp/all","proxy":"","request-body":[""],"follow-redirect":"1","shape":"request"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取项目名、地址","loopVariableName":"","variable-name":["projectUrls","projectNames"],"loopCount":"","variable-value":["${extract.selectors(resp.html,'.categorical-project-card a','attr','href')}","${extract.selectors(resp.html,'.project-name')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"抓取详情页","loopVariableName":"projectIndex","sleep":"","timeout":"","response-charset":"","method":"GET","body-type":"none","body-content-type":"text/plain","loopCount":"10","url":"https://gitee.com/${projectUrls[projectIndex]}","proxy":"","request-body":[""],"follow-redirect":"1","shape":"request"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取项目描述","loopVariableName":"","variable-name":["projectDesc"],"loopCount":"","variable-value":["${extract.selector(resp.html,'.git-project-desc-text')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"输出","output-name":["项目名","项目地址","项目描述"],"output-value":["${projectNames[projectIndex]}","https://gitee.com${projectUrls[projectIndex]}","${projectDesc}"],"shape":"output"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n\n', null, '0', '2019-08-22 13:46:54', null, null, null);
INSERT INTO `sp_flow` VALUES ('f0a67f17ee1a498a9b2f4ca30556f3c3', '抓取每日菜价', '\n \n \n \n {"spiderName":"抓取每日菜价","threadCount":""}\n \n \n \n \n \n \n {"shape":"start"}\n \n \n \n \n \n {"value":"开始抓取","loopVariableName":"","sleep":"","timeout":"","response-charset":"","method":"GET","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"http://www.beijingprice.cn:8086/price/priceToday/PageLoad/LoadPrice?jsoncallback=1","proxy":"","request-body":[""],"follow-redirect":"1","shape":"request"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"解析JSON","loopVariableName":"","variable-name":["jsonstr","jsondata","data"],"loopCount":"","variable-value":["${string.substring(resp.html,2,resp.html.length()-1)}","${json.parse(jsonstr)}","${extract.jsonpath(jsondata[0],'data')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"输出","loopVariableName":"i","output-name":["菜名","菜价","单位"],"loopCount":"${list.length(data)}","output-value":["${data[i].ItemName}","${data[i].Price04}","${data[i].ItemUnit}"],"shape":"output"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n\n', null, '0', '2019-08-22 13:48:22', null, null, null);
-INSERT INTO `sp_flow` VALUES ('b4430885ba8349588d1220d37eac831d', '爬取开源年中国动弹', '\n \n \n \n {"spiderName":"爬取开源年中国动弹","threadCount":""}\n \n \n \n \n \n \n {"shape":"start"}\n \n \n \n \n \n {"value":"爬取动弹","loopVariableName":"","sleep":"","timeout":"","response-charset":"","method":"GET","parameter-name":["type","lastLogId"],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"https://www.oschina.net/tweets/widgets/_tweet_index_list ","proxy":"","parameter-value":["ajax","${lastLogId}"],"request-body":"","follow-redirect":"1","tls-validate":"1","shape":"request"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取lastLogId以及tweets","loopVariableName":"","variable-name":["lastLogId","tweets","fetchCount"],"loopCount":"","variable-value":["${resp.selector('.tweet-item:last-child').attr('data-tweet-id')}","${resp.selectors('.tweet-item[data-tweet-id]')}","${fetchCount == null ? 0 : fetchCount + 1}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"循环","loopVariableName":"index","loopCount":"${list.length(tweets)}","shape":"loop"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取详细信息","loopVariableName":"","variable-name":["content","author","like","reply","publishTime"],"loopCount":"","variable-value":["${tweets[index].selector('.text').text()}","${tweets[index].selector('.user').text()}","${tweets[index].selector('.like span').text()}","${tweets[index].selector('.reply span').text()}","${tweets[index].selector('.date').regx('(.*?) ')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"输出","loopVariableName":"","output-name":["作者","内容","点赞数","评论数","发布时间"],"loopCount":"","output-value":["${author}","${content}","${like}","${reply}","${publishTime}"],"shape":"output"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n \n \n \n \n \n \n {"value":"爬取5页","condition":"${fetchCount < 3}"}\n \n \n \n\n', '', '0', '2019-11-03 17:02:49', '2019-11-04 10:11:31', '2019-11-03 17:30:56', '3');
+INSERT INTO `sp_flow` VALUES ('b4430885ba8349588d1220d37eac831d', '爬取开源中国动弹', '\n \n \n \n {"spiderName":"爬取开源中国动弹","threadCount":""}\n \n \n \n \n \n \n {"shape":"start"}\n \n \n \n \n \n {"value":"爬取动弹","loopVariableName":"","sleep":"","timeout":"","response-charset":"","method":"GET","parameter-name":["type","lastLogId"],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"https://www.oschina.net/tweets/widgets/_tweet_index_list ","proxy":"","parameter-value":["ajax","${lastLogId}"],"request-body":"","follow-redirect":"1","tls-validate":"1","shape":"request"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取lastLogId以及tweets","loopVariableName":"","variable-name":["lastLogId","tweets","fetchCount"],"loopCount":"","variable-value":["${resp.selector('.tweet-item:last-child').attr('data-tweet-id')}","${resp.selectors('.tweet-item[data-tweet-id]')}","${fetchCount == null ? 0 : fetchCount + 1}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"循环","loopVariableName":"index","loopCount":"${list.length(tweets)}","shape":"loop"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"提取详细信息","loopVariableName":"","variable-name":["content","author","like","reply","publishTime"],"loopCount":"","variable-value":["${tweets[index].selector('.text').text()}","${tweets[index].selector('.user').text()}","${tweets[index].selector('.like span').text()}","${tweets[index].selector('.reply span').text()}","${tweets[index].selector('.date').regx('(.*?) ')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"输出","loopVariableName":"","output-name":["作者","内容","点赞数","评论数","发布时间"],"loopCount":"","output-value":["${author}","${content}","${like}","${reply}","${publishTime}"],"shape":"output"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n \n \n \n \n \n \n {"value":"爬取5页","condition":"${fetchCount < 3}"}\n \n \n \n\n', '', '0', '2019-11-03 17:02:49', '2019-11-04 10:11:31', '2019-11-03 17:30:56', '3');
INSERT INTO `sp_flow` VALUES ('663aaa5e36a84c9594ef3cfd6738e9a7', '百度热点', '\n \n \n \n {"spiderName":"百度热点","threadCount":""}\n \n \n \n \n \n \n {"shape":"start"}\n \n \n \n \n \n {"value":"开始抓取","loopVariableName":"","sleep":"","timeout":"","response-charset":"gbk","method":"GET","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"https://top.baidu.com/buzz?b=1&fr=topindex","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","shape":"request"}\n \n \n \n \n \n {"value":"定义变量","loopVariableName":"","variable-name":["elementbd"],"loopCount":"","variable-value":["${resp.xpaths('//*[@id=\\"main\\"]/div[2]/div/table/tbody/tr')}"],"shape":"variable"}\n \n \n \n \n \n {"value":"输出","loopVariableName":"i","output-name":["名称","地址","百度指数","2"],"loopCount":"${elementbd.size()-1}","output-value":["${elementbd[i+1].xpath('//td[2]/a[1]/text()')}","${elementbd[i+1].xpath('//td[2]/a[1]/@href')}","${elementbd[i+1].xpath('//td[4]/span/text()')}","${elementbd[i+1].xpath('//td[3]/a[2]/text()')}"],"shape":"output"}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n \n \n {"value":"","condition":""}\n \n \n \n\n', '0 0/30 * * * ? *', '1', '2019-10-20 17:24:21', '2019-11-04 08:52:05', '2019-10-30 14:52:39', '45');
DROP TABLE IF EXISTS `sp_datasource`;