Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加全局变量

Browse Source
This commit is contained in:
mxd 2019-11-03 17:01:42 +08:00
parent 9d629edba3
commit 21af629b86
10 changed files with 332 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
spider-flow-core/src/main/java/org/spiderflow/core/expression/DefaultExpressionEngine.java
View File

@ -38,6 +38,7 @@ public class DefaultExpressionEngine implements ExpressionEngine{
for (FunctionExecutor executor : functionExecutors) {
context.set(executor.getFunctionPrefix(), executor);
}
ExpressionGlobalVariables.getVariables().forEach(context::set);
Object value = ExpressionTemplate.create(expression).render(context);
return value;
}

35
spider-flow-core/src/main/java/org/spiderflow/core/expression/ExpressionGlobalVariables.java Normal file
View File

@ -0,0 +1,35 @@
package org.spiderflow.core.expression;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class ExpressionGlobalVariables {
private static Map<String, Object> variables = new HashMap<>();
private static ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
public static void reset(Map<String, Object> map){
Lock lock = readWriteLock.writeLock();
lock.lock();
try {
variables.clear();
variables.putAll(map);
} finally {
lock.unlock();
}
}
public static Map<String, Object> getVariables(){
Lock lock = readWriteLock.readLock();
lock.lock();
try {
return variables;
} finally {
lock.unlock();
}
}
}

9
spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java
View File

@ -18,11 +18,6 @@ import com.alibaba.fastjson.JSON;
public class HttpResponse implements SpiderResponse{
private Response response;
public HttpResponse() {
super();
}
public HttpResponse(Response response) {
super();
@ -44,10 +39,6 @@ public class HttpResponse implements SpiderResponse{
return JSON.parse(getHtml());
}
public Document getDocument(){
return Jsoup.parse(getHtml());
}
@Override
public Map<String,String> getCookies(){
return response.cookies();

7
spider-flow-core/src/main/java/org/spiderflow/core/mapper/VariableMapper.java Normal file
View File

@ -0,0 +1,7 @@
package org.spiderflow.core.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.spiderflow.core.model.Variable;
public interface VariableMapper extends BaseMapper<Variable> {
}

62
spider-flow-core/src/main/java/org/spiderflow/core/model/Variable.java Normal file
View File

@ -0,0 +1,62 @@
package org.spiderflow.core.model;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
@TableName("sp_variable")
public class Variable {
@TableId(type = IdType.AUTO)
private Integer id;
private String name;
private String value;
private String description;
private Date createDate;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public Date getCreateDate() {
return createDate;
}
public void setCreateDate(Date createDate) {
this.createDate = createDate;
}
}

36
spider-flow-core/src/main/java/org/spiderflow/core/service/VariableService.java Normal file
View File

@ -0,0 +1,36 @@
package org.spiderflow.core.service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.spiderflow.core.expression.ExpressionGlobalVariables;
import org.spiderflow.core.mapper.VariableMapper;
import org.spiderflow.core.model.Variable;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.Serializable;
import java.util.Map;
import java.util.stream.Collectors;
@Service
public class VariableService extends ServiceImpl<VariableMapper, Variable> {
@Override
public boolean removeById(Serializable id) {
boolean ret = super.removeById(id);
this.resetGlobalVariables();
return ret;
}
@Override
public boolean saveOrUpdate(Variable entity) {
boolean ret = super.saveOrUpdate(entity);
this.resetGlobalVariables();
return ret;
}
@PostConstruct
private void resetGlobalVariables(){
Map<String, Object> variables = this.list().stream().collect(Collectors.toMap(Variable::getName, Variable::getValue));
ExpressionGlobalVariables.reset(variables);
}
}

13
spider-flow-web/src/main/java/org/spiderflow/controller/VariableController.java Normal file
View File

@ -0,0 +1,13 @@
package org.spiderflow.controller;
import org.spiderflow.common.CURDController;
import org.spiderflow.core.mapper.VariableMapper;
import org.spiderflow.core.model.Variable;
import org.spiderflow.core.service.VariableService;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/variable")
public class VariableController extends CURDController<VariableService, VariableMapper, Variable> {
}

3
spider-flow-web/src/main/resources/static/index.html
View File

@ -11,13 +11,14 @@
</head>
<body class="layui-layout layui-layout-admin">
<div class="layui-header">
<div class="layui-logo">SpiderFlow v1.0平台</div>
<div class="layui-logo">SpiderFlow平台v0.2.0</div>
<ul class="layui-nav layui-layout-left"></ul>
</div>
<div class="layui-side">
<div class="layui-side-scroll menu-list">
<ul class="layui-nav layui-nav-tree">
<li class="layui-nav-item layui-nav-itemed layui-this"><a data-link="spiderList.html" title="爬虫列表">爬虫列表</a></li>
<li class="layui-nav-item layui-nav-itemed"><a data-link="variables.html" title="全局变量">全局变量</a></li>
<li class="layui-nav-item layui-nav-itemed"><a data-link="datasources.html" title="数据源管理">数据源管理</a></li>
</ul>
</div>

93
spider-flow-web/src/main/resources/static/variable-edit.html Normal file
View File

@ -0,0 +1,93 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>DataSource</title>
<link rel="stylesheet" href="js/layui/css/layui.css" />
<link rel="stylesheet" href="css/layui-blue.css" />
<script type="text/javascript" src="js/layui/layui.all.js" ></script>
<script type="text/javascript" src="js/common.js" ></script>
<style type="text/css">
html,body{
width:100%;
}
.layui-form{
width : 700px;
margin-top:10px;
}
.layui-form-label{
width : 140px;
}
.layui-input-block{
margin-left : 170px;
}
.btns-submit{
text-align : center;
}
</style>
</head>
<body>
<form class="layui-form" autocomplete="off" lay-filter="form">
<div class="layui-form-item">
<label class="layui-form-label">变量名</label>
<div class="layui-input-block">
<input type="text" name="name" placeholder="请输入变量名" autocomplete="off" class="layui-input" lay-verify="required"/>
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">变量值</label>
<div class="layui-input-block">
<input type="text" name="value" placeholder="请输入变量值" autocomplete="off" class="layui-input" lay-verify="required"/>
</div>
</div>
<div class="layui-form-item layui-form-text">
<label class="layui-form-label">变量说明</label>
<div class="layui-input-block">
<textarea name="description" placeholder="请输入变量说明" autocomplete="off" class="layui-textarea" lay-verify="required"></textarea>
</div>
</div>
<div class="btns-submit">
<button class="layui-btn layui-btn-normal" lay-submit lay-filter="save">保存</button>
<button class="layui-btn layui-btn-primary btn-return" type="button" onclick="history.go(-1);">返回</button>
</div>
</form>
<script type="text/javascript">
var $ = layui.$;
var varId = getQueryString('id');
if(varId){
$.ajax({
url : 'variable/get',
data : {
id : varId
},
success : function(json){
layui.form.val('form',json.data)
}
})
}
layui.form.on('submit(save)',function(){
$.ajax({
url : 'variable/save',
type : 'post',
data : {
id : varId,
name : $("input[name=name]").val(),
value : $("input[name=value]").val(),
description : $("textarea[name=description]").val()
},
success : function(json){
layui.layer.msg('保存成功',{
time : 800
},function(){
location.href = 'variables.html';
})
},
error : function(){
layui.layer.msg('请求失败');
}
})
return false;
})
</script>
</body>
</html>

83
spider-flow-web/src/main/resources/static/variables.html Normal file
View File

@ -0,0 +1,83 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>DataSource</title>
<link rel="stylesheet" href="js/layui/css/layui.css" />
<link rel="stylesheet" href="css/layui-blue.css" />
<script type="text/javascript" src="js/layui/layui.all.js" ></script>
</head>
<body style="padding:5px;">
<a class="layui-btn layui-btn-sm layui-btn-normal" href="variable-edit.html"><i class="layui-icon">&#xe654;</i> 添加变量</a>
<hr>
<table class="layui-table" id="table" lay-filter="table"></table>
<script>
var $ = layui.$;
var $table = layui.table.render({
id : 'table',
elem : '#table',
url : 'variable/list',
page : true,
parseData : function(resp){
return {
code : 0,
data : resp.records,
count : resp.total
}
},
cols : [[{
title : '序号',
width : 60,
type : 'numbers',
align : 'center'
},{
title : '变量名',
field : 'name'
},{
title : '变量值',
field : 'value'
},{
title : '变量描述',
field : 'description'
},{
title : '创建时间',
width : 160,
field : 'createDate',
align : 'center'
},{
title : '操作',
width : 120,
align : 'center',
templet : '#buttons'
}]]
})
$("body").on('click','.btn-remove',function(){
var id = $(this).data('id');
layui.layer.confirm('您确定要删除此变量吗？',{
title : '删除'
},function(index){
$table.reload();
$.ajax({
url : 'variable/delete',
data : {
id : id
},
success : function(){
layui.layer.msg('删除成功',{time : 500},function(){
$table.reload();
})
},
error : function(){
layui.layer.msg('删除失败')
}
})
layui.layer.close(index);
})
})
</script>
<script type="text/html" id="buttons">
<a class="layui-btn layui-btn-sm" href="variable-edit.html?id={{d.id}}">编辑</a>
<a class="layui-btn layui-btn-sm btn-remove" data-id="{{d.id}}">删除</a>
</script>
</body>
</html>