Merge remote-tracking branch 'origin/dev' into dev

db/spiderflow.sql

@@ -43,3 +43,13 @@ CREATE TABLE `sp_variable` (
   `create_date` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4;
+
+/* v0.3.0 新增 */
+DROP TABLE IF EXISTS `sp_task`;
+CREATE TABLE `sp_task` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `flow_id` varchar(32) NOT NULL,
+  `begin_time` datetime DEFAULT NULL,
+  `end_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8mb4;

spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java

@@ -19,6 +19,10 @@ public class HttpResponse implements SpiderResponse{
 	
 	private Response response;
 
+	private String html;
+
+	private Object json;
+
 	public HttpResponse(Response response) {
 		super();
 		this.response = response;
@@ -31,12 +35,18 @@ public class HttpResponse implements SpiderResponse{
 	
 	@Override
 	public String getHtml(){
-		return response.body();
+		if(html == null){
+			html =  response.body();
+		}
+		return html;
 	}
 	
 	@Override
 	public Object getJson(){
-		return JSON.parse(getHtml());
+		if(json == null){
+			json = JSON.parse(getHtml());
+		}
+		return json;
 	}
 	
 	@Override

spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java

@@ -1,14 +1,19 @@
 package org.spiderflow.core.job;
 
 import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.commons.lang3.time.DateFormatUtils;
 import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
+import org.spiderflow.context.SpiderContext;
 import org.spiderflow.core.Spider;
 import org.spiderflow.core.model.SpiderFlow;
+import org.spiderflow.core.model.Task;
 import org.spiderflow.core.service.SpiderFlowService;
+import org.spiderflow.core.service.TaskService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.quartz.QuartzJobBean;
@@ -16,59 +21,67 @@ import org.springframework.stereotype.Component;
 
 /**
  * 爬虫定时执行
- * @author Administrator
  *
+ * @author Administrator
  */
 @Component
-public class SpiderJob extends QuartzJobBean{
-	
-	private static Spider spider;
-	
-	private static SpiderFlowService spiderFlowService;
-	
+public class SpiderJob extends QuartzJobBean {
+
+	@Autowired
+	private Spider spider;
+
+	@Autowired
+	private SpiderFlowService spiderFlowService;
+
+	@Autowired
+	private TaskService taskService;
+
+	private static Map<Integer, SpiderContext> contextMap = new HashMap<>();
+
 	@Value("${spider.job.enable:true}")
 	private boolean spiderJobEnable;
-	
+
 	@Value("${spider.job.log.path:./}")
 	private String spiderLogPath;
 
 	@Override
 	protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
-		if(!spiderJobEnable){
+		if (!spiderJobEnable) {
 			return;
 		}
 		JobDataMap dataMap = context.getMergedJobDataMap();
 		SpiderFlow spiderFlow = (SpiderFlow) dataMap.get(SpiderJobManager.JOB_PARAM_NAME);
-		run(spiderFlow,context.getNextFireTime());
+		run(spiderFlow, context.getNextFireTime());
 	}
-	
-	public void run(String id){
-		run(spiderFlowService.getById(id),null);
+
+	public void run(String id) {
+		run(spiderFlowService.getById(id), null);
 	}
-	
-	public void run(SpiderFlow spiderFlow,Date nextExecuteTime){
+
+	public void run(SpiderFlow spiderFlow, Date nextExecuteTime) {
 		Date now = new Date();
 		SpiderJobContext context = SpiderJobContext.create(this.spiderLogPath, spiderFlow.getId() + ".log");
-		try{
-			context.info("开始执行任务{}",spiderFlow.getName());
-			spider.run(spiderFlow,context);
-			context.info("执行任务{}完毕，下次执行时间：{}",spiderFlow.getName(),nextExecuteTime == null ? null: DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
+		Task task = new Task();
+		task.setFlowId(spiderFlow.getId());
+		task.setBeginTime(new Date());
+		try {
+			taskService.save(task);
+			contextMap.put(task.getId(), context);
+			context.info("开始执行任务{}", spiderFlow.getName());
+			spider.run(spiderFlow, context);
+			context.info("执行任务{}完毕，下次执行时间：{}", spiderFlow.getName(), nextExecuteTime == null ? null : DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
 		} catch (Exception e) {
-			context.error("执行任务{}出错",spiderFlow.getName(),e);
-		} finally{
+			context.error("执行任务{}出错", spiderFlow.getName(), e);
+		} finally {
 			context.close();
+			task.setEndTime(new Date());
+			taskService.saveOrUpdate(task);
+			contextMap.remove(task.getId());
 		}
 		spiderFlowService.executeCountIncrement(spiderFlow.getId(), now, nextExecuteTime);
 	}
 
-	@Autowired
-	public void setSpider(Spider spider) {
-		SpiderJob.spider = spider;
+	public static SpiderContext getSpiderContext(Integer taskId){
+		return contextMap.get(taskId);
 	}
-
-	@Autowired
-	public void setSpiderFlowService(SpiderFlowService spiderFlowService) {
-		SpiderJob.spiderFlowService = spiderFlowService;
-	}
-
 }

spider-flow-core/src/main/java/org/spiderflow/core/mapper/SpiderFlowMapper.java

@@ -3,6 +3,8 @@ package org.spiderflow.core.mapper;
 import java.util.Date;
 import java.util.List;
 
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
@@ -18,6 +20,20 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
  */
 public interface SpiderFlowMapper extends BaseMapper<SpiderFlow>{
 
+	@Select({
+			"<script>",
+				"select",
+					"id,name,enabled,last_execute_time,next_execute_time,cron,create_date,execute_count,",
+					"(select count(*) from sp_task where flow_id = sf.id and end_time is null) running",
+				"from sp_flow sf",
+				"<if test=\"name != null and name != ''\">",
+					"where name like concat('%',#{name},'%')",
+				"</if>",
+				"order by create_date desc",
+			"</script>"
+	})
+	IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page,@Param("name") String name);
+
 	@Insert("insert into sp_flow(id,name,xml,enabled) values(#{id},#{name},#{xml},'0')")
 	int insertSpiderFlow(@Param("id") String id, @Param("name") String name, @Param("xml") String xml);
 	

spider-flow-core/src/main/java/org/spiderflow/core/mapper/TaskMapper.java

@@ -0,0 +1,9 @@
+package org.spiderflow.core.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Mapper;
+import org.spiderflow.core.model.Task;
+
+@Mapper
+public interface TaskMapper extends BaseMapper<Task> {
+}

spider-flow-core/src/main/java/org/spiderflow/core/model/SpiderFlow.java

@@ -1,6 +1,7 @@
 package org.spiderflow.core.model;
 
 import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
 
@@ -40,6 +41,9 @@ public class SpiderFlow {
      */
     private Integer executeCount;
 
+    @TableField(exist = false)
+    private Integer running;
+
 
     public SpiderFlow() {
     }
@@ -120,4 +124,12 @@ public class SpiderFlow {
     public void setExecuteCount(Integer executeCount) {
         this.executeCount = executeCount;
     }
+
+    public Integer getRunning() {
+        return running;
+    }
+
+    public void setRunning(Integer running) {
+        this.running = running;
+    }
 }

spider-flow-core/src/main/java/org/spiderflow/core/model/Task.java

@@ -0,0 +1,52 @@
+package org.spiderflow.core.model;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+
+import java.util.Date;
+
+@TableName("sp_task")
+public class Task {
+
+	@TableId(type = IdType.AUTO)
+	private Integer id;
+
+	private String flowId;
+
+	private Date beginTime;
+
+	private Date endTime;
+
+	public Integer getId() {
+		return id;
+	}
+
+	public void setId(Integer id) {
+		this.id = id;
+	}
+
+	public String getFlowId() {
+		return flowId;
+	}
+
+	public void setFlowId(String flowId) {
+		this.flowId = flowId;
+	}
+
+	public Date getBeginTime() {
+		return beginTime;
+	}
+
+	public void setBeginTime(Date beginTime) {
+		this.beginTime = beginTime;
+	}
+
+	public Date getEndTime() {
+		return endTime;
+	}
+
+	public void setEndTime(Date endTime) {
+		this.endTime = endTime;
+	}
+}

spider-flow-core/src/main/java/org/spiderflow/core/service/SpiderFlowService.java

@@ -8,6 +8,8 @@ import java.util.UUID;
 
 import javax.annotation.PostConstruct;
 
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import org.apache.commons.lang3.StringUtils;
 import org.quartz.CronScheduleBuilder;
 import org.quartz.CronTrigger;
@@ -49,6 +51,10 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
 			}
 		}
 	}
+
+	public IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page, String name){
+		return sfMapper.selectSpiderPage(page,name);
+	}
 	
 	public int executeCountIncrement(String id, Date lastExecuteTime, Date nextExecuteTime){
 		if(nextExecuteTime == null){

spider-flow-core/src/main/java/org/spiderflow/core/service/TaskService.java

@@ -0,0 +1,11 @@
+package org.spiderflow.core.service;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.spiderflow.core.mapper.TaskMapper;
+import org.spiderflow.core.model.Task;
+import org.springframework.stereotype.Service;
+
+@Service
+public class TaskService extends ServiceImpl<TaskMapper, Task> {
+
+}

spider-flow-web/src/main/java/org/spiderflow/controller/SpiderFlowController.java

@@ -102,7 +102,7 @@ public class SpiderFlowController {
 	 */
 	@RequestMapping("/list")
 	public IPage<SpiderFlow> list(@RequestParam(name = "page", defaultValue = "1") Integer page, @RequestParam(name = "limit", defaultValue = "1") Integer size, @RequestParam(name = "name", defaultValue = "") String name) {
-		return spiderFlowService.page(new Page<>(page, size), new QueryWrapper<SpiderFlow>().like("name", name).orderByDesc("create_date"));
+		return spiderFlowService.selectSpiderPage(new Page<>(page, size), name);
 	}
 	
 	@RequestMapping("/save")

spider-flow-web/src/main/java/org/spiderflow/controller/TaskController.java

@@ -0,0 +1,46 @@
+package org.spiderflow.controller;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.spiderflow.context.SpiderContext;
+import org.spiderflow.core.job.SpiderJob;
+import org.spiderflow.core.model.Task;
+import org.spiderflow.core.service.TaskService;
+import org.spiderflow.model.JsonBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/task")
+public class TaskController {
+
+	@Autowired
+	private TaskService taskService;
+
+	@RequestMapping("/list")
+	public IPage<Task> list(@RequestParam(name = "page", defaultValue = "1") Integer page, @RequestParam(name = "limit", defaultValue = "1") Integer size,String flowId){
+		return taskService.page(new Page<>(page,size),new QueryWrapper<Task>().eq("flow_id",flowId).last("order by isnull(end_time) desc,end_time desc"));
+	}
+
+	/**
+	 * 停止执行任务
+	 * @param id
+	 * @return
+	 */
+	@RequestMapping("/stop")
+	public JsonBean<Boolean> stop(Integer id){
+		SpiderContext context = SpiderJob.getSpiderContext(id);
+		if(context != null){
+			context.setRunning(false);
+		}
+		return new JsonBean<>(context != null);
+	}
+
+	@RequestMapping("/remove")
+	public JsonBean<Boolean> remove(Integer id){
+		return new JsonBean<>(taskService.removeById(id));
+	}
+}

spider-flow-web/src/main/resources/static/spiderList.html

@@ -72,10 +72,11 @@
 			field : 'lastExecuteTime',
 			align : 'center'
 		},{
-			title : '已执行',
-			width : 100,
+			title : '运行中/已完成',
+			width : 120,
 			field : 'executeCount',
-			align : 'center'
+			align : 'center',
+			templet : '#execute-count'
 		},{
 			title : '下次执行时间',
 			width : 160,
@@ -164,6 +165,8 @@
 			})
 			layui.layer.close(index);
 		})
+	}).on('click','.btn-task',function(){
+		parent.openTab($(this).data('name') + '-任务详情','task.html?id=' + $(this).data('id') + '&name=' + encodeURIComponent(encodeURIComponent($(this).data('name'))));
 	}).on('click','.btn-log',function(){
 		parent.openTab($(this).data('name') + '-日志','log.html?id=' + $(this).data('id'));
 	}).on('click','.btn-edit-cron',function(){
@@ -179,6 +182,9 @@
 		});
 	})
 </script>
+<script type="text/html" id="execute-count">
+	<a class="layui-btn layui-btn-sm btn-task" data-name="{{d.name}}" data-id="{{d.id}}">{{d.running || 0}}/{{d.executeCount || 0}}</a>
+</script>
 <script type="text/html" id="buttons">
 	<a class="layui-btn layui-btn-sm btn-run" data-id="{{d.id}}">手动运行</a>
 	<a class="layui-btn layui-btn-sm btn-log" data-name="{{d.name}}" data-id="{{d.id}}">查看日志</a>

spider-flow-web/src/main/resources/static/task.html

@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="UTF-8">
+	<title>SpiderFlow</title>
+	<link rel="stylesheet" href="js/layui/css/layui.css" />
+	<link rel="stylesheet" href="css/layui-blue.css" />
+	<script type="text/javascript" src="js/layui/layui.all.js" ></script>
+	<script type="text/javascript" src="js/common.js" ></script>
+</head>
+<body style="padding:5px;">
+<table class="layui-table" id="table" lay-filter="table"></table>
+<script>
+	var $ = layui.$;
+	var $table = layui.table.render({
+		id : 'table',
+		elem : '#table',
+		url : 'task/list?flowId=' + getQueryString('id'),
+		method : 'post',
+		page : true,
+		parseData : function(resp){
+			return {
+				code : 0,
+				data : resp.records,
+				count : resp.total
+			}
+		},
+		cols : [[{
+			title : '序号',
+			width : 60,
+			type : 'numbers',
+			align : 'center'
+		},{
+			title : '任务开始时间',
+			field : 'beginTime',
+			align : 'center'
+		},{
+			title : '任务结束时间',
+			field : 'endTime',
+			align : 'center'
+		},{
+			title : '操作',
+			width : 200,
+			align : 'center',
+			templet : '#buttons'
+		}]]
+	})
+	$("body").on('click','.btn-remove',function(){
+		var id = $(this).data('id');
+		layui.layer.confirm('您确定要删除此记录吗？',{
+			title : '删除'
+		},function(index){
+			$table.reload();
+			$.ajax({
+				url : 'task/remove',
+				data : {
+					id : id
+				},
+				success : function(){
+					layui.layer.msg('删除成功',{time : 500},function(){
+						$table.reload();
+					})
+				},
+				error : function(){
+					layui.layer.msg('删除失败')
+				}
+			})
+			layui.layer.close(index);
+		})
+	}).on('click','.btn-stop',function(){
+		var id = $(this).data('id');
+		layui.layer.confirm('您确定要停止该任务吗？',{
+			title : '停止任务'
+		},function(index){
+			$table.reload();
+			$.ajax({
+				url : 'task/stop',
+				data : {
+					id : id
+				},
+				success : function(){
+					layui.layer.msg('后台运行停止中...',{time : 500},function(){
+						$table.reload();
+					})
+				},
+				error : function(){
+					layui.layer.msg('停止任务失败')
+				}
+			})
+			layui.layer.close(index);
+		})
+	}).on('click','.btn-log',function(){
+		parent.openTab(decodeURIComponent(decodeURIComponent(getQueryString('name'))) + '-日志','log.html?id=' + $(this).data('id'));
+	})
+</script>
+<script type="text/html" id="buttons">
+	{{# if(!d.endTime){ }}
+		<a class="layui-btn layui-btn-sm btn-stop" data-id="{{d.id}}">停止</a>
+	{{# } }}
+	<a class="layui-btn layui-btn-sm btn-log" data-id="{{d.flowId}}">查看日志</a>
+	<a class="layui-btn layui-btn-sm btn-remove" data-id="{{d.id}}">删除记录</a>
+</script>
+</body>
+</html>