Cookie自动管理
This commit is contained in:
mxd
parent
aa02f561af
commit
785c3fc857
@ -0,0 +1,6 @@
|
||||
package org.spiderflow.context;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
public class CookieContext extends HashMap<String, String> {
|
||||
}
|
||||
@ -33,7 +33,9 @@ public class SpiderContext extends HashMap<String, Object>{
|
||||
|
||||
private SpiderNode rootNode;
|
||||
|
||||
private boolean running = true;
|
||||
private boolean running = true;
|
||||
|
||||
private CookieContext cookieContext = new CookieContext();
|
||||
|
||||
public List<SpiderOutput> getOutputs() {
|
||||
return outputs;
|
||||
@ -107,7 +109,11 @@ public class SpiderContext extends HashMap<String, Object>{
|
||||
}
|
||||
this.log(new SpiderLog(level, message, variables));
|
||||
}
|
||||
|
||||
|
||||
public CookieContext getCookieContext() {
|
||||
return cookieContext;
|
||||
}
|
||||
|
||||
public void log(SpiderLog log){
|
||||
|
||||
}
|
||||
|
||||
@ -3,16 +3,14 @@ package org.spiderflow.core.executor.shape;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.spiderflow.ExpressionEngine;
|
||||
import org.spiderflow.Grammerable;
|
||||
import org.spiderflow.context.CookieContext;
|
||||
import org.spiderflow.context.SpiderContext;
|
||||
import org.spiderflow.core.io.HttpRequest;
|
||||
import org.spiderflow.core.io.HttpResponse;
|
||||
@ -44,7 +42,11 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
public static final String PARAMETER_NAME = "parameter-name";
|
||||
|
||||
public static final String PARAMETER_VALUE = "parameter-value";
|
||||
|
||||
|
||||
public static final String COOKIE_NAME = "cookie-name";
|
||||
|
||||
public static final String COOKIE_VALUE = "cookie-value";
|
||||
|
||||
public static final String PARAMETER_FORM_NAME = "parameter-form-name";
|
||||
|
||||
public static final String PARAMETER_FORM_VALUE = "parameter-form-value";
|
||||
@ -72,6 +74,8 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
public static final String TLS_VALIDATE = "tls-validate";
|
||||
|
||||
public static final String LAST_EXECUTE_TIME = "__last_execute_time_";
|
||||
|
||||
public static final String COOKIE_AUTO_SET = "cookie-auto-set";
|
||||
|
||||
@Autowired
|
||||
private ExpressionEngine engine;
|
||||
@ -89,6 +93,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
CookieContext cookieContext = context.getCookieContext();
|
||||
String sleepCondition = node.getStringJsonValue(SLEEP);
|
||||
if(StringUtils.isNotBlank(sleepCondition)){
|
||||
try {
|
||||
@ -140,11 +145,33 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
request.validateTLSCertificates(false);
|
||||
context.debug("设置TLS证书验证：{}", false);
|
||||
}
|
||||
|
||||
SpiderNode root = context.getRootNode();
|
||||
//设置请求header
|
||||
setRequestHeader(request, root.getListJsonValue(HEADER_NAME,HEADER_VALUE), context, variables);
|
||||
setRequestHeader(request, node.getListJsonValue(HEADER_NAME,HEADER_VALUE), context, variables);
|
||||
|
||||
//设置全局Cookie
|
||||
Map<String, String> cookies = getRequestCookie(request, root.getListJsonValue(COOKIE_NAME, COOKIE_VALUE), context, variables);
|
||||
if(!cookies.isEmpty()){
|
||||
context.debug("设置全局Cookie：{}", cookies);
|
||||
request.cookies(cookies);
|
||||
}
|
||||
//设置自动管理的Cookie
|
||||
boolean cookieAutoSet = !"0".equals(node.getStringJsonValue(COOKIE_AUTO_SET));
|
||||
if(cookieAutoSet){
|
||||
request.cookies(cookieContext);
|
||||
context.debug("自动设置Cookie：{}", cookieContext);
|
||||
}
|
||||
//设置本节点Cookie
|
||||
cookies = getRequestCookie(request, node.getListJsonValue(COOKIE_NAME, COOKIE_VALUE), context, variables);
|
||||
if(!cookies.isEmpty()){
|
||||
context.debug("设置Cookie：{}", cookies);
|
||||
request.cookies(cookies);
|
||||
}
|
||||
if(cookieAutoSet){
|
||||
cookieContext.putAll(cookies);
|
||||
}
|
||||
|
||||
String bodyType = node.getStringJsonValue(BODY_TYPE);
|
||||
List<InputStream> streams = null;
|
||||
if("raw".equals(bodyType)){
|
||||
@ -188,6 +215,8 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
response.setCharset(charset);
|
||||
context.debug("设置response charset:{}",charset);
|
||||
}
|
||||
//cookie存入cookieContext
|
||||
cookieContext.putAll(response.getCookies());
|
||||
//结果存入变量
|
||||
variables.put("resp", response);
|
||||
} catch (IOException e) {
|
||||
@ -247,40 +276,63 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
}
|
||||
return streams;
|
||||
}
|
||||
|
||||
private void setRequestParameter(HttpRequest request,List<Map<String, String>> parameters,SpiderContext context,Map<String,Object> variables){
|
||||
if(parameters != null){
|
||||
for (Map<String,String> nameValue : parameters) {
|
||||
|
||||
private Map<String,String> getRequestCookie(HttpRequest request, List<Map<String, String>> cookies, SpiderContext context, Map<String, Object> variables) {
|
||||
Map<String,String> cookieMap = new HashMap<>();
|
||||
if (cookies != null) {
|
||||
for (Map<String, String> nameValue : cookies) {
|
||||
Object value = null;
|
||||
String cookieName = nameValue.get(COOKIE_NAME);
|
||||
if (StringUtils.isNotBlank(cookieName)) {
|
||||
String cookieValue = nameValue.get(COOKIE_VALUE);
|
||||
try {
|
||||
value = engine.execute(cookieValue, variables);
|
||||
if (value != null) {
|
||||
cookieMap.put(cookieName, cookieValue);
|
||||
context.debug("设置请求Cookie：{}={}", cookieName, value);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
context.error("设置请求Cookie：{}出错,异常信息：{}", cookieName, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return cookieMap;
|
||||
}
|
||||
|
||||
private void setRequestParameter(HttpRequest request, List<Map<String, String>> parameters, SpiderContext context, Map<String, Object> variables) {
|
||||
if (parameters != null) {
|
||||
for (Map<String, String> nameValue : parameters) {
|
||||
Object value = null;
|
||||
String parameterName = nameValue.get(PARAMETER_NAME);
|
||||
if(StringUtils.isNotBlank(parameterName)){
|
||||
if (StringUtils.isNotBlank(parameterName)) {
|
||||
String parameterValue = nameValue.get(PARAMETER_VALUE);
|
||||
try {
|
||||
value = engine.execute(parameterValue, variables);
|
||||
context.debug("设置请求参数：{}={}",parameterName,value);
|
||||
context.debug("设置请求参数：{}={}", parameterName, value);
|
||||
} catch (Exception e) {
|
||||
context.error("设置请求参数：{}出错,异常信息：{}",parameterName,e);
|
||||
context.error("设置请求参数：{}出错,异常信息：{}", parameterName, e);
|
||||
}
|
||||
request.data(parameterName, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void setRequestHeader(HttpRequest request,List<Map<String, String>> headers,SpiderContext context,Map<String,Object> variables){
|
||||
if(headers != null){
|
||||
for (Map<String,String> nameValue : headers) {
|
||||
|
||||
private void setRequestHeader(HttpRequest request, List<Map<String, String>> headers, SpiderContext context, Map<String, Object> variables) {
|
||||
if (headers != null) {
|
||||
for (Map<String, String> nameValue : headers) {
|
||||
Object value = null;
|
||||
String headerName = nameValue.get(HEADER_NAME);
|
||||
if(StringUtils.isNotBlank(headerName)){
|
||||
if (StringUtils.isNotBlank(headerName)) {
|
||||
String headerValue = nameValue.get(HEADER_VALUE);
|
||||
try {
|
||||
value = engine.execute(headerValue, variables);
|
||||
context.debug("设置请求Header：{}={}",headerName,value);
|
||||
context.debug("设置请求Header：{}={}", headerName, value);
|
||||
} catch (Exception e) {
|
||||
context.error("设置请求Header：{}出错,异常信息：{}",headerName,e);
|
||||
context.error("设置请求Header：{}出错,异常信息：{}", headerName, e);
|
||||
}
|
||||
request.header(headerName,value);
|
||||
request.header(headerName, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,6 +45,18 @@ public class HttpRequest {
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest cookies(Map<String,String> cookies){
|
||||
this.connection.cookies(cookies);
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest cookie(String name, String value) {
|
||||
if (value != null) {
|
||||
this.connection.cookie(name, value);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public HttpRequest contentType(String contentType){
|
||||
this.connection.header("Content-Type", contentType);
|
||||
@ -110,7 +122,7 @@ public class HttpRequest {
|
||||
this.connection.ignoreContentType(true);
|
||||
this.connection.ignoreHttpErrors(true);
|
||||
this.connection.maxBodySize(0);
|
||||
|
||||
|
||||
Response response = connection.execute();
|
||||
return new HttpResponse(response);
|
||||
}
|
||||
|
||||
@ -85,6 +85,9 @@ iframe{
|
||||
.layui-nav .layui-this:after, .layui-nav-bar, .layui-nav-tree .layui-nav-itemed:after{
|
||||
display: none;
|
||||
}
|
||||
.layui-tab-title li{
|
||||
padding: 0 5px;
|
||||
}
|
||||
.layui-tab-title li .layui-tab-close{
|
||||
visibility: hidden;
|
||||
}
|
||||
|
||||
@ -54,7 +54,7 @@ function serializeForm(){
|
||||
cell.data.set(name,value);
|
||||
}
|
||||
});
|
||||
$(".properties-container form [lay-skin=switch]").each(function(){
|
||||
$(".properties-container form input[type=checkbox]").each(function(){
|
||||
cell.data.set(this.value,$(this).is(":checked") ? '1': '0');
|
||||
});
|
||||
cell.data.set('shape',shape);
|
||||
@ -163,7 +163,7 @@ $(function(){
|
||||
if($('.sidebar-container')[0].scrollWidth>$('.sidebar-container')[0].clientWidth){
|
||||
$('.sidebar-container').width($('.sidebar-container').width() + 3);
|
||||
}
|
||||
layui.form.on('switch',function(e){
|
||||
layui.form.on('checkbox',function(e){
|
||||
serializeForm();
|
||||
});
|
||||
//节点名称输入框事件
|
||||
@ -207,7 +207,7 @@ $(function(){
|
||||
}).on("click",".editor-form-node .variable-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close variable-remove"></i><label class="layui-form-label">变量名</label><div class="layui-input-block"><input type="text" name="variable-name" placeholder="变量名称" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">变量值</label><div class="layui-input-block array" codemirror="variable-value" placeholder="请输入变量值"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".editor-form-node .header-remove,.editor-form-node .parameter-remove,.editor-form-node .variable-remove,.editor-form-node .output-remove",function(){ //移除多行
|
||||
}).on("click",".editor-form-node .header-remove,.editor-form-node .cookie-remove,.editor-form-node .parameter-remove,.editor-form-node .variable-remove,.editor-form-node .output-remove",function(){ //移除多行
|
||||
var $dom = $(this).parent();
|
||||
$dom.prev().remove();
|
||||
$dom.next().remove();
|
||||
@ -219,6 +219,9 @@ $(function(){
|
||||
}).on("click",".editor-form-node .parameter-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close parameter-remove"></i><label class="layui-form-label">参数名</label><div class="layui-input-block"><input type="text" name="parameter-name" placeholder="请输入参数名" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">参数值</label><div class="layui-input-block array" codemirror="parameter-value" placeholder="请输入参数值"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".editor-form-node .cookie-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close cookie-remove"></i><label class="layui-form-label">Cookie名</label><div class="layui-input-block"><input type="text" name="cookie-name" placeholder="请输入Cookie名" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">Cookie值</label><div class="layui-input-block array" codemirror="cookie-value" placeholder="请输入Cookie值"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".editor-form-node .output-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close output-remove"></i><label class="layui-form-label">输出项</label><div class="layui-input-block"><input type="text" name="output-name" placeholder="请输入输出项" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">输出值</label><div class="layui-input-block array" codemirror="output-value" placeholder="请输入输出值"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
<ul class="layui-tab-title">
|
||||
<li class="layui-this">基本配置</li>
|
||||
<li>参数</li>
|
||||
<li>Cookie</li>
|
||||
<li>Header</li>
|
||||
<li>Body</li>
|
||||
</ul>
|
||||
@ -66,15 +67,11 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">跟随重定向</label>
|
||||
<label class="layui-form-label">请求设置</label>
|
||||
<div class="layui-input-block">
|
||||
<input type="checkbox" value="follow-redirect" lay-skin="switch" {{d.data.object['follow-redirect'] == '0' ? '' : 'checked'}}/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">TLS证书验证</label>
|
||||
<div class="layui-input-block">
|
||||
<input type="checkbox" value="tls-validate" lay-skin="switch" {{d.data.object['tls-validate'] == '0' ? '' : 'checked'}}/>
|
||||
<input type="checkbox" title="跟随重定向" value="follow-redirect" lay-skin="primary" {{d.data.object['follow-redirect'] == '0' ? '' : 'checked'}}/>
|
||||
<input type="checkbox" title="TLS证书验证" value="tls-validate" lay-skin="primary" {{d.data.object['tls-validate'] == '0' ? '' : 'checked'}}/>
|
||||
<input type="checkbox" title="自动管理Cookie" value="cookie-auto-set" lay-skin="primary" {{d.data.object['cookie-auto-set'] == '0' ? '' : 'checked'}}/>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
@ -102,6 +99,29 @@
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="layui-tab-item">
|
||||
<form class="layui-form">
|
||||
{{# layui.each(d.data.object['cookie-name'],function(index){ }}
|
||||
<div class="layui-form-item layui-form-relative">
|
||||
<i class="layui-icon layui-icon-close cookie-remove"></i>
|
||||
<label class="layui-form-label">Cookie名</label>
|
||||
<div class="layui-input-block">
|
||||
<input type="text" name="cookie-name" placeholder="请输入Cookie名" autocomplete="off" class="layui-input array" value="{{=d.data.object['cookie-name'][index]}}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">Cookie值</label>
|
||||
<div class="layui-input-block array" placeholder="请输入Cookie值" codemirror="cookie-value" data-value="{{=d.data.object['cookie-value'][index]}}"></div>
|
||||
</div>
|
||||
<hr>
|
||||
{{# }) }}
|
||||
<div class="layui-form-item">
|
||||
<div class="layui-input-block">
|
||||
<button class="layui-btn cookie-add" type="button">添加一个Cookie</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="layui-tab-item">
|
||||
<form class="layui-form">
|
||||
{{# layui.each(d.data.object['header-name'],function(index){ }}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
<ul class="layui-tab-title">
|
||||
<li class="layui-this">全局配置</li>
|
||||
<li>全局参数</li>
|
||||
<li>全局Cookie</li>
|
||||
<li>全局Header</li>
|
||||
</ul>
|
||||
<div class="layui-tab-content editor-form-node">
|
||||
@ -44,6 +45,29 @@
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="layui-tab-item">
|
||||
<form class="layui-form">
|
||||
{{# layui.each(d.data.object['cookie-name'],function(index){ }}
|
||||
<div class="layui-form-item layui-form-relative">
|
||||
<i class="layui-icon layui-icon-close cookie-remove"></i>
|
||||
<label class="layui-form-label">Cookie名</label>
|
||||
<div class="layui-input-block">
|
||||
<input type="text" name="cookie-name" placeholder="请输入Cookie名" autocomplete="off" class="layui-input array" value="{{=d.data.object['cookie-name'][index]}}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">Cookie值</label>
|
||||
<div class="layui-input-block array" placeholder="请输入Cookie值" codemirror="cookie-value" data-value="{{=d.data.object['cookie-value'][index]}}"></div>
|
||||
</div>
|
||||
<hr>
|
||||
{{# }) }}
|
||||
<div class="layui-form-item">
|
||||
<div class="layui-input-block">
|
||||
<button class="layui-btn cookie-add" type="button">添加一个Cookie</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="layui-tab-item">
|
||||
<form class="layui-form">
|
||||
{{# layui.each(d.data.object['header-name'],function(index){ }}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user