增加历史版本

This commit is contained in:
mxd 2020-04-02 18:52:06 +08:00
parent 80c7aa32a0
commit 3c4e81c2f4
12 changed files with 209 additions and 77 deletions

View File

@ -2,6 +2,7 @@
<img src="https://www.spiderflow.org/images/logo.svg" width="600">
</p>
<p align="center">
<a target="_blank" href="https://github.com/javamxd/spider-flow/releases"><img src="https://img.shields.io/github/v/release/javamxd/spider-flow"></a>
<a target="_blank" href="https://www.oracle.com/technetwork/java/javase/downloads/index.html"><img src="https://img.shields.io/badge/JDK-1.8+-green.svg" /></a>
<a target="_blank" href="https://www.spiderflow.org"><img src="https://img.shields.io/badge/Docs-latest-blue.svg"/></a>
<a target="_blank" href='https://gitee.com/jmxd/spider-flow'><img src="https://gitee.com/jmxd/spider-flow/badge/star.svg?theme=white" /></a>

View File

@ -43,8 +43,8 @@ public class SpiderJob extends QuartzJobBean {
@Value("${spider.job.enable:true}")
private boolean spiderJobEnable;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
@Value("${spider.workspace}")
private String workspace;
private static Logger logger = LoggerFactory.getLogger(SpiderJob.class);
@ -70,7 +70,7 @@ public class SpiderJob extends QuartzJobBean {
task.setBeginTime(new Date());
try {
taskService.save(task);
context = SpiderJobContext.create(this.spiderLogPath, spiderFlow.getId() + task.getId() + ".log");
context = SpiderJobContext.create(this.workspace, spiderFlow.getId(),task.getId());
SpiderContextHolder.set(context);
contextMap.put(task.getId(), context);
logger.info("开始执行任务{}", spiderFlow.getName());

View File

@ -32,14 +32,15 @@ public class SpiderJobContext extends SpiderContext{
return this.outputstream;
}
public static SpiderJobContext create(String directory,String file){
public static SpiderJobContext create(String directory,String id,Integer taskId){
OutputStream os = null;
try {
File dirFile = new File(directory);
File file = new File(new File(directory),id + File.separator + "logs" + File.separator + taskId + ".log");
File dirFile = file.getParentFile();
if(!dirFile.exists()){
dirFile.mkdirs();
}
os = new FileOutputStream(new File(directory,file), true);
os = new FileOutputStream(file, true);
} catch (Exception e) {
logger.error("创建日志文件出错",e);
}

View File

@ -1,29 +1,31 @@
package org.spiderflow.core.service;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import javax.annotation.PostConstruct;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
import org.quartz.TriggerBuilder;
import org.quartz.TriggerUtils;
import org.quartz.spi.OperableTrigger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.core.job.SpiderJobManager;
import org.spiderflow.core.mapper.SpiderFlowMapper;
import org.spiderflow.core.model.SpiderFlow;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import javax.annotation.PostConstruct;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
/**
* 爬虫流程执行服务
@ -39,6 +41,11 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
@Autowired
private SpiderJobManager spiderJobManager;
private static Logger logger = LoggerFactory.getLogger(SpiderFlowService.class);
@Value("${spider.workspace}")
private String workspace;
//项目启动后自动查询需要执行的任务进行爬取
@PostConstruct
private void initJobs(){
@ -99,7 +106,6 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
.build();
spiderFlow.setNextExecuteTime(trigger.getStartTime());
}
//
if(StringUtils.isNotEmpty(spiderFlow.getId())){ //update 任务
sfMapper.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml());
spiderJobManager.remove(spiderFlow.getId());
@ -112,6 +118,12 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
sfMapper.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml());
spiderFlow.setId(id);
}
File file = new File(workspace,spiderFlow.getId() + File.separator + "xmls" + File.separator + System.currentTimeMillis() + ".xml");
try {
FileUtils.write(file,spiderFlow.getXml(),"UTF-8");
} catch (IOException e) {
logger.error("保存历史记录出错",e);
}
return true;
}
@ -177,6 +189,29 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
return list;
}
public List<Long> historyList(String id){
File directory = new File(workspace, id + File.separator + "xmls");
if(directory.exists() && directory.isDirectory()){
File[] files = directory.listFiles((dir, name) -> name.endsWith(".xml"));
if(files != null && files.length > 0){
return Arrays.stream(files).map(f-> Long.parseLong(f.getName().replace(".xml",""))).sorted().collect(Collectors.toList());
}
}
return Collections.emptyList();
}
public String readHistory(String id,String timestamp){
File file = new File(workspace, id + File.separator + "xmls" + File.separator + timestamp + ".xml");
if(file.exists()){
try {
return FileUtils.readFileToString(file,"UTF-8");
} catch (IOException e) {
logger.error("读取历史版本出错",e);
}
}
return null;
}
public Integer getFlowMaxTaskId(String flowId){
return sfMapper.getFlowMaxTaskId(flowId);
}

View File

@ -63,8 +63,8 @@ public class SpiderFlowController {
@Autowired(required = false)
private List<PluginConfig> pluginConfigs;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
@Value("${spider.workspace}")
private String workspace;
private final List<Grammer> grammers = new ArrayList<Grammer>();
@ -109,6 +109,15 @@ public class SpiderFlowController {
spiderFlowService.save(spiderFlow);
return spiderFlow.getId();
}
@RequestMapping("/history")
public JsonBean<?> history(String id,String timestamp){
if(StringUtils.isNotBlank(timestamp)){
return new JsonBean<>(spiderFlowService.readHistory(id,timestamp));
}else{
return new JsonBean<>(spiderFlowService.historyList(id));
}
}
@RequestMapping("/get")
public SpiderFlow get(String id){
@ -159,8 +168,7 @@ public class SpiderFlowController {
Integer maxId = spiderFlowService.getFlowMaxTaskId(id);
taskId = maxId == null ? "" : maxId.toString();
}
String finalTaskId = taskId;
File file = new File(spiderLogPath, id + finalTaskId + ".log");
File file = new File(workspace, id + File.separator + "logs" + File.separator + taskId + ".log");
return ResponseEntity.ok()
.header("Content-Disposition","attachment; filename=spider.log")
.contentType(MediaType.parseMediaType("application/octet-stream"))
@ -173,7 +181,8 @@ public class SpiderFlowController {
Integer maxId = spiderFlowService.getFlowMaxTaskId(id);
taskId = maxId == null ? "" : maxId.toString();
}
try (RandomAccessFileReader reader = new RandomAccessFileReader(new RandomAccessFile(new File(spiderLogPath,id + taskId +".log"),"r"), index == null ? -1 : index, reversed == null || reversed)){
File logFile = new File(workspace, id + File.separator + "logs" + File.separator + taskId + ".log");
try (RandomAccessFileReader reader = new RandomAccessFileReader(new RandomAccessFile(logFile,"r"), index == null ? -1 : index, reversed == null || reversed)){
return new JsonBean<>(reader.readLine(count == null ? 10 : count,keywords,matchcase != null && matchcase,regx != null && regx));
} catch(FileNotFoundException e){
return new JsonBean<>(0,"日志文件不存在");

View File

@ -1,8 +1,5 @@
package org.spiderflow.controller;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.core.Spider;
@ -18,6 +15,9 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/rest")
public class SpiderRestController {
@ -30,8 +30,8 @@ public class SpiderRestController {
@Autowired
private Spider spider;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
@Value("${spider.workspace}")
private String workspace;
@RequestMapping("/run/{id}")
public JsonBean<List<SpiderOutput>> run(@PathVariable("id")String id,@RequestBody(required = false)Map<String,Object> params){
@ -41,8 +41,7 @@ public class SpiderRestController {
}
List<SpiderOutput> outputs = null;
Integer maxId = spiderFlowService.getFlowMaxTaskId(id);
String taskId = maxId == null ? "" : maxId.toString();
SpiderJobContext context = SpiderJobContext.create(spiderLogPath, id + taskId + ".log");
SpiderJobContext context = SpiderJobContext.create(workspace, id,maxId);
try{
outputs = spider.run(flow,context, params);
}catch(Exception e){

View File

@ -8,8 +8,8 @@ spider.thread.max=64
spider.thread.default=8
#设置为true时定时任务才生效
spider.job.enable=false
#爬虫任务的执行日志保存路径
spider.job.log.path=/data/spider/logs/
#爬虫任务的工作空间
spider.workspace=/data/spider
#死循环检测(节点执行次数超过该值时认为是死循环)默认值为5000
#spider.detect.dead-cycle=5000

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty scope="context" name="LOG_LEVEL" source="logging.level.root" defaultValue="DEBUG"/>
<springProperty scope="context" name="LOG_HOME" source="spider.job.log.path" defaultValue="/data/spider/logs"/>
<springProperty scope="context" name="WORKSPACE" source="spider.workspace" defaultValue="/data/spider/logs"/>
<!-- 控制台输出 -->
<appender name="Stdout" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@ -11,7 +11,7 @@
</appender>
<!-- 输出日志文件 -->
<appender name="File" class="org.spiderflow.logback.SpiderFlowFileAppender">
<file>${LOG_HOME}/spider-flow.log</file>
<file>${WORKSPACE}/logs/spider-flow.log</file>
<append>true</append>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出:%d表示日期%thread表示线程名%-5level级别从左显示5个字符宽度%msg日志消息%n是换行符-->

View File

@ -345,6 +345,9 @@ html,body{
.toolbar-container ul li.btn-resume.disabled{
background-image : url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACYAAAAgCAYAAAB+ZAqzAAAC5UlEQVRYR7WYy2oUQRSG/9N5AN/DhRsfYOpUWpGgSPCKBGMUFYJ4JagoiToQogQSEVGiGDFIUIxIIEiYpKqYLIKKILh34yqoIEIWCp0jPc6EcZjO1NRMatX0ufxfna4+XV1UKBS2RFF0IYqi/QC+AZhJkmQ6juMfyBjOuQMAxkXkg4jktdYfs3xD75MxpouI5moS9DHz06yk1loLQJXtqwDyzHw7FKJeHFlrJwEcqzHOMHNawbrDWmsAcI1xIYqifC6XK7YDMAWrnn0lp2PmWuF1PWttAUBcd6ZEeaXUYKtwQWDOuXkR2ZklTkTLIjLIzAuhgEFg1tq3AHY1EhWRMa31xUZ+WWus6UdpjJkjoi5Pwc9JkpyL4zjV8R5BFXPOzYrIHm8VAEQ0oZQ67RsTBGatfQNgr69IxY+IvqQdQCm11Cg2FOw1gO5GybPsIjKltT66UXwo2CsA+0LBynErRNStlFpu2+J3zr0QkYMtgpXCRWRSa328NldoxaYBHG4HWDnH6tramu7s7Hy/vh5DOr9z7rmIHGkjWCkVET1SSp0qXYeAWWunAPS0G6ySj5kpFCzdefRuFljavIPAnHNPRKRvs8A6Ojq2B4FZax8DOLEZYETUr5R6EARmjJkgopPtBCOiryKyjZl/trL4HwLw/u41mgAR3VFKXa72C6qYc+6+iPQ3EvSwfwewg5k/tavB3gNwxkM404WIXiqlDmU5BFXMGHOXiM6GgBHRLxHpZeZ0h5INHthgxwCcDwCbY+bdPnFBFbPWjgK45CNQ9vkN4Aozj/vGBIEZY+4Q0YCnyEKSJD1xHK94+v/7bgY+yhEA/73e9URFZEhrfasZoIpvEJgxZpiIrm4guCQiA1rrdyFQrVQsD+B6RpWGtdbXQoGqK9b0EYFz7mb6Q1stLiLpJm9Qaz3fKlSpYiGHKouLizeiKBqqAhhlZt+XwYubQo6hisXi1iRJnhHRHyIayeVys15qTTj9BSbJmzEMQ5JCAAAAAElFTkSuQmCC");
}
.toolbar-container ul li.btn-history{
background-image : url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADWUlEQVRYR8WXW2gUZxTHf2c2iRQvRFEz24JgKwoKKih9qOCNolgUH4qhor55AZXsRBRpXxpBFBGzk6CC2gdfpHhB0AdbEGl8qL644AWVSr2gmJ0YqVRtITW7p3zDLG6WmZ1xkGSeFr7zne/3nXO+8z8rjPAnI3w+oQC2q78ZMM+RJdWAnx3W6aVBVgErUT5FyILvo0+gT+F8qZEL/dvkz6QXiwLQAMBfz3bqMs2wHmVDQscXVejuy8mVOPu6AOUSczMZdiusDRz1C1wG7pfhZkkp8B9vR40mW3rnR8REZhXCdGMvwk/FnGyqB1EXAOgHJgUOjpYtOl+0ycO4W2W7dLMqm4F5wG3PkTlRe+IAzL6LKHnfQYYXXpvciwOorNuungHWVKezdm8SgCF7RJhfzEkhMUReOxB+BH7xHPkmKcARYGbIIfcGlb0v26WYFMAv4i7dp8r3YTWRug9k87pOLV55ObmUBMZ29RYwW4Wvq19HKgA7r4sRTK/oqe0VUTB2XnciHDQ15TmyumI3bACTXW2xhPso47GYVSnmYQMwN7ZdPQd8i7LHa5cOv1ckyV+tTZoUGB8tnfqdWPwscK3oyIJhB7C7dSZl7gIPPUemDQGIEqCwCKWNwIRuHddU5m/grefI2FqAN8CYwQHGvdwt5nfklxZg4gEd2zCK11EARkK/yDQw4/l2eZAUAOVqHdser116KuuBnP8RmoKsq78rfIWypHpTTArq13BVtfuvIOgfoUVYaZfAIc+RnXGvw3Z1K8rkGLshEch26QlVNoY+w6yryxV+RXngtcuMOIA067arj4CpoY0oqNA7wBQRthRzcjzNIVF7ghnhWN1W7IcVjBIWPEfmf0wA29UbZkCJFSPbVaNuK4CzniOtHwPiw+S4QxvsZt75B9dUcRoY+/1AUrA+YWHvFvm32k/4RNSlrSin/U4l7C/m5IdUh1eNZFYjU3q3ybNaP5Fi1JLXpSJUxuqCCMeTFmbNUOqJsDJqjKuvhms0Yy9gP7ArIH9soLTMqUwjvQP/4I9mUWM5cLKpiR1Pt8qrqAgmkmO7U1uxfJDPE6biSVA/J+PsEwEYJ815bW6CxSIssuBLhRYj8cEB/l8zhKtGGwYsrv/VJkZ0Yr/EALGeUhqMOMD/LOV+MJQ5P0UAAAAASUVORK5CYII=");
}
.spiderflow-debug-tooltip{
position: absolute;
z-index: 2147483647;
@ -528,4 +531,41 @@ html,body{
}
#test-window{
height:340px !important;
}
.history-version{
list-style: disc;
padding : 2px 5px;
max-height: 200px;
overflow: auto;
}
.history-version::-webkit-scrollbar {
width: 4px;
}
.history-version::-webkit-scrollbar-track {
background-color:#ccc;
-webkit-border-radius: 2em;
-moz-border-radius: 2em;
border-radius:2em;
}
.history-version::-webkit-scrollbar-thumb {
background-color:#999;
-webkit-border-radius: 2em;
-moz-border-radius: 2em;
border-radius:2em;
}
.history-version li{
color: #333;
cursor: pointer;
font-size: 12px;
height:22px;
line-height: 22px;
border-bottom: 1px solid #eee;
list-style: disc inside;
}
.history-version li:nth-last-child(1){
border-bottom: none;
}
.history-version li:hover{
color : #1890FF;
}

View File

@ -36,6 +36,7 @@
<span>|</span>
<li class="btn-undo" title="撤销Ctrl+Z"></li>
<li class="btn-redo" title="反撤销Ctrl+Y"></li>
<li class="btn-history" title="历史版本"></li>
<span>|</span>
<li class="btn-selectAll" title="全选Ctrl+A"></li>
<li class="btn-cut" title="剪切Ctrl+X"></li>
@ -128,6 +129,14 @@
<div class="layui-input-block array" codemirror="output-value" placeholder="输出值" data-value="{{=d['output-value']}}"></div>
</script>
<script type="text/html" id="history-version-tmpl">
<ul class="history-version">
{{# layui.each(d,function(index,item){ }}
<li data-timestamp="{{item.timestamp}}">{{item.time}}</li>
{{# });}}
</ul>
</script>
<script type="text/html" id="common-operation">
<a class="layui-btn layui-btn-sm table-row-up">上移</a>
<a class="layui-btn layui-btn-sm table-row-down">下移</a>

View File

@ -6,6 +6,21 @@ function getQueryString(name) {
}
return null;
}
Date.prototype.format = function(b) {
var a = this;
var c = {
"M+": a.getMonth() + 1,
"d+": a.getDate(),
"h+": a.getHours(),
"m+": a.getMinutes(),
"s+": a.getSeconds(),
"q+": Math.floor((a.getMonth() + 3) / 3),
S: a.getMilliseconds()
};
/(y+)/.test(b) && (b = b.replace(RegExp.$1, (a.getFullYear() + "").substr(4 - RegExp.$1.length)));
for(var d in c) new RegExp("(" + d + ")").test(b) && (b = b.replace(RegExp.$1, 1 == RegExp.$1.length ? c[d] : ("00" + c[d]).substr(("" + c[d]).length)));
return b
}
var sf = {};
sf.ajax = function(options){
var loading;

View File

@ -3,6 +3,7 @@ var editor;
var flows;
var codeMirrorInstances = {};
var socket;
var version = 'lastest';
function renderCodeMirror(){
codeMirrorInstances = {};
$('[codemirror]').each(function(){
@ -48,7 +49,11 @@ function getCellData(cellId,keys){
return data;
}
function serializeForm(){
var cellId = $(".properties-container").attr('data-cellid');
var $container = $(".properties-container");
if($container.data('version') != version){
return;
}
var cellId = $container.attr('data-cellid');
var model = editor.getModel();
var cell = model.getCell(cellId);
if(!cell){
@ -203,6 +208,7 @@ $(function(){
if(cell.isEdge()){
template = 'edge';
}
var v = version;
var render = function(){
layui.laytpl(templateCache[template]).render({
data : cell.data,
@ -211,7 +217,7 @@ $(function(){
model : model,
cell : cell
},function(html){
$(".properties-container").html(html).attr('data-cellid',cell.id);
$(".properties-container").attr('data-version',v).html(html).attr('data-cellid',cell.id);
layui.form.render();
renderCodeMirror();
resizeSlideBar();
@ -282,6 +288,65 @@ $(function(){
})
}).on("blur","input,textarea",function(){
serializeForm();
}).on("click",".history-version li",function(){
var timestamp = $(this).data("timestamp");
layui.layer.confirm('你确定要恢复到该版本吗',function(index){
layui.layer.close(index);
var layerIndex = layui.layer.load(1);
$.ajax({
url : 'spider/history',
data : {
id : id,
timestamp : timestamp
},
success : function(data){
if(data.code == 1){
version = timestamp;
editor.setXML(data.data);
layui.layer.close(layerIndex);
layui.layer.msg('恢复成功')
}else{
layui.layer.msg(data.message);
}
}
})
});
}).on("click",".btn-history",function(){
$.ajax({
url : 'spider/history',
data : {
id : id
},
success : function(data){
if(data.code == 1){
if(data.data.length > 0){
var array = [];
for(var i = data.data.length - 1;i >=0;i--){
var timestamp = Number(data.data[i])
array.push({
time : new Date(timestamp).format('yyyy-MM-dd hh:mm:ss'),
timestamp : timestamp
})
}
layui.laytpl($('#history-version-tmpl').html()).render(array,function(html){
layui.layer.open({
type : 1,
title : '历史版本',
id : 'history-revert',
shade : 0,
resize : false,
content : html,
offset : 'rt'
})
})
}else{
layui.layer.msg('暂无历史版本');
}
}else{
layui.layer.msg(data.message);
}
}
})
}).on("click",".table-row-add",function(){ //添加一行
serializeForm();
var tableId = $(this).attr('for');
@ -485,7 +550,6 @@ $(function(){
}
})
layui.form.on('select',serializeForm);
//loadTemplate('root',graph.getModel().getRoot(),graph);
var id = getQueryString('id');
if(id != null){
$.ajax({
@ -502,47 +566,6 @@ $(function(){
}
editor.onSelectedCell();
}
/**
* 处理选择事件
*/
function processCellEvent(cell,graph){
if(cell != null){
if(cell.isEdge()){
cell.data = cell.data || new JsonProperty();
loadTemplate('edge',cell,graph);
}else{
cell.data = cell.data || new JsonProperty();
if(cell.data.shape != 'start'){
loadTemplate(cell.data.object.shape,cell,graph);
}
}
}else{
loadTemplate('root',graph.getModel().getRoot(),graph);
}
}
/**
* 重置已设表单array参数变量Headers
*/
function resetFormArray(graph,prefix,key){
var cell = graph.getSelectionCell() || graph.getModel().getRoot();
var array = [];
var names = [];
var values = [];
$(".editor-form-node input[name="+prefix+"-name]").each(function(){
names.push(this.value);
});
$(".editor-form-node input[name="+prefix+"-value]").each(function(){
values.push(this.value);
});
for(var i=0,len = names.length;i<len;i++){
array.push({
name : names[i],
value : values[i]
});
}
cell.data.set(key,array)
}
/**
* 加载各种图形
*/