diff --git a/spider-flow-api/src/main/java/org/spiderflow/context/CookieContext.java b/spider-flow-api/src/main/java/org/spiderflow/context/CookieContext.java new file mode 100644 index 0000000..45b0d77 --- /dev/null +++ b/spider-flow-api/src/main/java/org/spiderflow/context/CookieContext.java @@ -0,0 +1,6 @@ +package org.spiderflow.context; + +import java.util.HashMap; + +public class CookieContext extends HashMap { +} diff --git a/spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java b/spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java index bb8c20b..97ded35 100644 --- a/spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java +++ b/spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java @@ -33,7 +33,9 @@ public class SpiderContext extends HashMap{ private SpiderNode rootNode; - private boolean running = true; + private boolean running = true; + + private CookieContext cookieContext = new CookieContext(); public List getOutputs() { return outputs; @@ -107,7 +109,11 @@ public class SpiderContext extends HashMap{ } this.log(new SpiderLog(level, message, variables)); } - + + public CookieContext getCookieContext() { + return cookieContext; + } + public void log(SpiderLog log){ } diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java b/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java index 65384ab..015f761 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java @@ -3,16 +3,14 @@ package org.spiderflow.core.executor.shape; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.math.NumberUtils; import org.spiderflow.ExpressionEngine; import org.spiderflow.Grammerable; +import org.spiderflow.context.CookieContext; import org.spiderflow.context.SpiderContext; import org.spiderflow.core.io.HttpRequest; import org.spiderflow.core.io.HttpResponse; @@ -44,7 +42,11 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ public static final String PARAMETER_NAME = "parameter-name"; public static final String PARAMETER_VALUE = "parameter-value"; - + + public static final String COOKIE_NAME = "cookie-name"; + + public static final String COOKIE_VALUE = "cookie-value"; + public static final String PARAMETER_FORM_NAME = "parameter-form-name"; public static final String PARAMETER_FORM_VALUE = "parameter-form-value"; @@ -72,6 +74,8 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ public static final String TLS_VALIDATE = "tls-validate"; public static final String LAST_EXECUTE_TIME = "__last_execute_time_"; + + public static final String COOKIE_AUTO_SET = "cookie-auto-set"; @Autowired private ExpressionEngine engine; @@ -89,6 +93,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ @Override public void execute(SpiderNode node, SpiderContext context, Map variables) { + CookieContext cookieContext = context.getCookieContext(); String sleepCondition = node.getStringJsonValue(SLEEP); if(StringUtils.isNotBlank(sleepCondition)){ try { @@ -140,11 +145,33 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ request.validateTLSCertificates(false); context.debug("设置TLS证书验证：{}", false); } - SpiderNode root = context.getRootNode(); //设置请求header setRequestHeader(request, root.getListJsonValue(HEADER_NAME,HEADER_VALUE), context, variables); setRequestHeader(request, node.getListJsonValue(HEADER_NAME,HEADER_VALUE), context, variables); + + //设置全局Cookie + Map cookies = getRequestCookie(request, root.getListJsonValue(COOKIE_NAME, COOKIE_VALUE), context, variables); + if(!cookies.isEmpty()){ + context.debug("设置全局Cookie：{}", cookies); + request.cookies(cookies); + } + //设置自动管理的Cookie + boolean cookieAutoSet = !"0".equals(node.getStringJsonValue(COOKIE_AUTO_SET)); + if(cookieAutoSet){ + request.cookies(cookieContext); + context.debug("自动设置Cookie：{}", cookieContext); + } + //设置本节点Cookie + cookies = getRequestCookie(request, node.getListJsonValue(COOKIE_NAME, COOKIE_VALUE), context, variables); + if(!cookies.isEmpty()){ + context.debug("设置Cookie：{}", cookies); + request.cookies(cookies); + } + if(cookieAutoSet){ + cookieContext.putAll(cookies); + } + String bodyType = node.getStringJsonValue(BODY_TYPE); List streams = null; if("raw".equals(bodyType)){ @@ -188,6 +215,8 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ response.setCharset(charset); context.debug("设置response charset:{}",charset); } + //cookie存入cookieContext + cookieContext.putAll(response.getCookies()); //结果存入变量 variables.put("resp", response); } catch (IOException e) { @@ -247,40 +276,63 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{ } return streams; } - - private void setRequestParameter(HttpRequest request,List> parameters,SpiderContext context,Map variables){ - if(parameters != null){ - for (Map nameValue : parameters) { + + private Map getRequestCookie(HttpRequest request, List> cookies, SpiderContext context, Map variables) { + Map cookieMap = new HashMap<>(); + if (cookies != null) { + for (Map nameValue : cookies) { + Object value = null; + String cookieName = nameValue.get(COOKIE_NAME); + if (StringUtils.isNotBlank(cookieName)) { + String cookieValue = nameValue.get(COOKIE_VALUE); + try { + value = engine.execute(cookieValue, variables); + if (value != null) { + cookieMap.put(cookieName, cookieValue); + context.debug("设置请求Cookie：{}={}", cookieName, value); + } + } catch (Exception e) { + context.error("设置请求Cookie：{}出错,异常信息：{}", cookieName, e); + } + } + } + } + return cookieMap; + } + + private void setRequestParameter(HttpRequest request, List> parameters, SpiderContext context, Map variables) { + if (parameters != null) { + for (Map nameValue : parameters) { Object value = null; String parameterName = nameValue.get(PARAMETER_NAME); - if(StringUtils.isNotBlank(parameterName)){ + if (StringUtils.isNotBlank(parameterName)) { String parameterValue = nameValue.get(PARAMETER_VALUE); try { value = engine.execute(parameterValue, variables); - context.debug("设置请求参数：{}={}",parameterName,value); + context.debug("设置请求参数：{}={}", parameterName, value); } catch (Exception e) { - context.error("设置请求参数：{}出错,异常信息：{}",parameterName,e); + context.error("设置请求参数：{}出错,异常信息：{}", parameterName, e); } request.data(parameterName, value); } } } } - - private void setRequestHeader(HttpRequest request,List> headers,SpiderContext context,Map variables){ - if(headers != null){ - for (Map nameValue : headers) { + + private void setRequestHeader(HttpRequest request, List> headers, SpiderContext context, Map variables) { + if (headers != null) { + for (Map nameValue : headers) { Object value = null; String headerName = nameValue.get(HEADER_NAME); - if(StringUtils.isNotBlank(headerName)){ + if (StringUtils.isNotBlank(headerName)) { String headerValue = nameValue.get(HEADER_VALUE); try { value = engine.execute(headerValue, variables); - context.debug("设置请求Header：{}={}",headerName,value); + context.debug("设置请求Header：{}={}", headerName, value); } catch (Exception e) { - context.error("设置请求Header：{}出错,异常信息：{}",headerName,e); + context.error("设置请求Header：{}出错,异常信息：{}", headerName, e); } - request.header(headerName,value); + request.header(headerName, value); } } } diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpRequest.java b/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpRequest.java index 5cf5e3d..6f94c23 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpRequest.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpRequest.java @@ -45,6 +45,18 @@ public class HttpRequest { } return this; } + + public HttpRequest cookies(Map cookies){ + this.connection.cookies(cookies); + return this; + } + + public HttpRequest cookie(String name, String value) { + if (value != null) { + this.connection.cookie(name, value); + } + return this; + } public HttpRequest contentType(String contentType){ this.connection.header("Content-Type", contentType); @@ -110,7 +122,7 @@ public class HttpRequest { this.connection.ignoreContentType(true); this.connection.ignoreHttpErrors(true); this.connection.maxBodySize(0); - + Response response = connection.execute(); return new HttpResponse(response); } diff --git a/spider-flow-web/src/main/resources/static/css/layui-blue.css b/spider-flow-web/src/main/resources/static/css/layui-blue.css index 6edca51..9f4319e 100644 --- a/spider-flow-web/src/main/resources/static/css/layui-blue.css +++ b/spider-flow-web/src/main/resources/static/css/layui-blue.css @@ -85,6 +85,9 @@ iframe{ .layui-nav .layui-this:after, .layui-nav-bar, .layui-nav-tree .layui-nav-itemed:after{ display: none; } +.layui-tab-title li{ + padding: 0 5px; +} .layui-tab-title li .layui-tab-close{ visibility: hidden; } diff --git a/spider-flow-web/src/main/resources/static/js/editor.js b/spider-flow-web/src/main/resources/static/js/editor.js index d182e79..b270ee0 100644 --- a/spider-flow-web/src/main/resources/static/js/editor.js +++ b/spider-flow-web/src/main/resources/static/js/editor.js @@ -54,7 +54,7 @@ function serializeForm(){ cell.data.set(name,value); } }); - $(".properties-container form [lay-skin=switch]").each(function(){ + $(".properties-container form input[type=checkbox]").each(function(){ cell.data.set(this.value,$(this).is(":checked") ? '1': '0'); }); cell.data.set('shape',shape); @@ -163,7 +163,7 @@ $(function(){ if($('.sidebar-container')[0].scrollWidth>$('.sidebar-container')[0].clientWidth){ $('.sidebar-container').width($('.sidebar-container').width() + 3); } - layui.form.on('switch',function(e){ + layui.form.on('checkbox',function(e){ serializeForm(); }); //节点名称输入框事件 @@ -207,7 +207,7 @@ $(function(){ }).on("click",".editor-form-node .variable-add",function(){ $(this).parent().parent().before('
'); renderCodeMirror(); - }).on("click",".editor-form-node .header-remove,.editor-form-node .parameter-remove,.editor-form-node .variable-remove,.editor-form-node .output-remove",function(){ //移除多行 + }).on("click",".editor-form-node .header-remove,.editor-form-node .cookie-remove,.editor-form-node .parameter-remove,.editor-form-node .variable-remove,.editor-form-node .output-remove",function(){ //移除多行 var $dom = $(this).parent(); $dom.prev().remove(); $dom.next().remove(); @@ -219,6 +219,9 @@ $(function(){ }).on("click",".editor-form-node .parameter-add",function(){ $(this).parent().parent().before('
'); renderCodeMirror(); + }).on("click",".editor-form-node .cookie-add",function(){ + $(this).parent().parent().before('
'); + renderCodeMirror(); }).on("click",".editor-form-node .output-add",function(){ $(this).parent().parent().before('
'); renderCodeMirror(); diff --git a/spider-flow-web/src/main/resources/static/resources/templates/request.html b/spider-flow-web/src/main/resources/static/resources/templates/request.html index 8c2af04..3f2041a 100644 --- a/spider-flow-web/src/main/resources/static/resources/templates/request.html +++ b/spider-flow-web/src/main/resources/static/resources/templates/request.html @@ -2,6 +2,7 @@
  • 基本配置
  • 参数
    • +
  • Cookie
  • Header
  • Body
@@ -66,15 +67,11 @@
- +
- -
-
-
- -
- + + +
@@ -102,6 +99,29 @@ +
+
+ {{# layui.each(d.data.object['cookie-name'],function(index){ }} +
+ + +
+ +
+
+
+ +
+
+
+ {{# }) }} +
+
+ +
+
+
+
{{# layui.each(d.data.object['header-name'],function(index){ }} diff --git a/spider-flow-web/src/main/resources/static/resources/templates/root.html b/spider-flow-web/src/main/resources/static/resources/templates/root.html index 2eca1b5..16891b1 100644 --- a/spider-flow-web/src/main/resources/static/resources/templates/root.html +++ b/spider-flow-web/src/main/resources/static/resources/templates/root.html @@ -2,6 +2,7 @@
  • 全局配置
  • 全局参数
    • +
  • 全局Cookie
  • 全局Header
@@ -44,6 +45,29 @@
+
+
+ {{# layui.each(d.data.object['cookie-name'],function(index){ }} +
+ + +
+ +
+
+
+ +
+
+
+ {{# }) }} +
+
+ +
+
+
+
{{# layui.each(d.data.object['header-name'],function(index){ }}