增加定时执行

pom.xml

@@ -22,6 +22,18 @@
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-web</artifactId>
 		</dependency>
+		<dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-quartz</artifactId>
+        </dependency>
+        <dependency>  
+           <groupId>org.springframework.boot</groupId>  
+           <artifactId>spring-boot-starter-websocket</artifactId>  
+       	</dependency>
+		<dependency>
+		    <groupId>org.springframework.boot</groupId>
+		    <artifactId>spring-boot-starter-data-jpa</artifactId>
+		</dependency>
 		<dependency>
 		    <groupId>com.alibaba</groupId>
 		    <artifactId>druid-spring-boot-starter</artifactId>
@@ -47,14 +59,6 @@
 		    <artifactId>JsoupXpath</artifactId>
 		    <version>2.3.2</version>
 		</dependency>  
-		<dependency>  
-           <groupId>org.springframework.boot</groupId>  
-           <artifactId>spring-boot-starter-websocket</artifactId>  
-       	</dependency>
-		<dependency>
-		    <groupId>org.springframework.boot</groupId>
-		    <artifactId>spring-boot-starter-data-jpa</artifactId>
-		</dependency>
 		<dependency>
 		    <groupId>com.h2database</groupId>
 		    <artifactId>h2</artifactId>
@@ -64,10 +68,6 @@
 			<groupId>org.freemarker</groupId>
 			<artifactId>freemarker</artifactId>
 		</dependency>
-		<dependency>
-		    <groupId>org.springframework.boot</groupId>
-		    <artifactId>spring-boot-starter-data-jpa</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>

src/main/java/com/mxd/spider/core/Spider.java

@@ -22,6 +22,8 @@ import com.mxd.spider.core.freemarker.FreeMarkerEngine;
 import com.mxd.spider.core.model.SpiderNode;
 import com.mxd.spider.core.model.SpiderOutput;
 import com.mxd.spider.core.utils.Maps;
+import com.mxd.spider.core.utils.SpiderFlowUtils;
+import com.mxd.spider.web.model.SpiderFlow;
 
 @Component
 public class Spider {
@@ -34,6 +36,14 @@ public class Spider {
 	@Autowired
 	private FreeMarkerEngine engine;
 	
+	public void run(SpiderFlow spiderFlow){
+		SpiderNode root = SpiderFlowUtils.loadXMLFromString(spiderFlow.getXml());
+		SpiderContext context = new SpiderContext();
+		int threadPoolSize = 8;
+		ThreadPoolExecutor pool = new ThreadPoolExecutor(threadPoolSize,threadPoolSize,5000,TimeUnit.MILLISECONDS,new LinkedBlockingQueue<Runnable>());
+		execute(pool,root, context,new HashMap<>());
+	}
+	
 	public List<SpiderOutput> runWithTest(SpiderNode root,SpiderContext context){
 		//开始不允许设置任何东西
 		int threadPoolSize = 8;

src/main/java/com/mxd/spider/job/SpiderJob.java

@@ -0,0 +1,57 @@
+package com.mxd.spider.job;
+
+import java.util.Date;
+
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.springframework.stereotype.Component;
+
+import com.mxd.spider.core.Spider;
+import com.mxd.spider.web.model.SpiderFlow;
+import com.mxd.spider.web.service.SpiderFlowService;
+
+@Component
+public class SpiderJob extends QuartzJobBean{
+	
+	private static Logger logger = LoggerFactory.getLogger(SpiderJob.class);
+	
+	private static Spider spider;
+	
+	private static SpiderFlowService spiderFlowService;
+
+	@Override
+	protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
+		Date now = new Date();
+		//下次执行时间
+		Date nextExecuteTime = context.getNextFireTime();
+		JobDataMap dataMap = context.getMergedJobDataMap();
+		SpiderFlow spiderFlow = (SpiderFlow) dataMap.get(SpiderJobManager.JOB_PARAM_NAME);
+		try {
+			logger.info("开始执行任务{}",spiderFlow.getName());
+			spider.run(spiderFlow);
+			logger.info("执行任务{}完毕，下次执行时间：{}",spiderFlow.getName(),DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
+		} catch (Exception e) {
+			logger.error("执行任务{}出错",spiderFlow.getName(),e);
+		}
+		spiderFlowService.executeCountIncrement(spiderFlow.getId(), now, nextExecuteTime);
+	}
+
+	@Autowired
+	public void setSpider(Spider spider) {
+		SpiderJob.spider = spider;
+	}
+
+	@Autowired
+	public void setSpiderFlowService(SpiderFlowService spiderFlowService) {
+		SpiderJob.spiderFlowService = spiderFlowService;
+	}
+
+	
+
+}

src/main/java/com/mxd/spider/job/SpiderJobManager.java

@@ -0,0 +1,106 @@
+package com.mxd.spider.job;
+
+import org.quartz.CronScheduleBuilder;
+import org.quartz.CronTrigger;
+import org.quartz.JobBuilder;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobKey;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.TriggerBuilder;
+import org.quartz.TriggerKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.mxd.spider.web.model.SpiderFlow;
+
+@Component
+public class SpiderJobManager {
+	
+	private static Logger logger = LoggerFactory.getLogger(SpiderJobManager.class);
+	
+	private final static String JOB_NAME = "SPIDER_TASK_";
+	
+	public final static String JOB_PARAM_NAME = "SPIDER_FLOW";
+	
+	@Autowired
+	private Scheduler scheduler;
+	
+	private JobKey getJobKey(String id){
+		return JobKey.jobKey(JOB_NAME + id);
+	}
+	
+	public CronTrigger getCronTrigger(String id) {
+        try {
+            return (CronTrigger) scheduler.getTrigger(getTriggerKey(id));
+        } catch (SchedulerException e) {
+        	logger.error("获取CronTrigger出错",e);
+            return null;
+        }
+    }
+
+	private TriggerKey getTriggerKey(String id){
+		return TriggerKey.triggerKey(JOB_NAME + id);
+	}
+	
+	public boolean addJob(SpiderFlow spiderFlow){
+		try {
+			JobDetail job = JobBuilder.newJob(SpiderJob.class).withIdentity(getJobKey(spiderFlow.getId())).build();
+			job.getJobDataMap().put(JOB_PARAM_NAME, spiderFlow);
+			
+			CronScheduleBuilder cronScheduleBuilder = CronScheduleBuilder.cronSchedule(spiderFlow.getCron()).withMisfireHandlingInstructionDoNothing();
+			
+			CronTrigger trigger = TriggerBuilder.newTrigger().withIdentity(getTriggerKey(spiderFlow.getId())).withSchedule(cronScheduleBuilder).build();
+			
+			scheduler.scheduleJob(job,trigger);
+			
+			return true;
+		} catch (SchedulerException e) {
+			logger.error("创建定时任务出错",e);
+			return false;
+		}
+	}
+	
+	public boolean updateJob(SpiderFlow spiderFlow){
+		try {
+			if(getCronTrigger(spiderFlow.getId()) != null){
+				TriggerKey triggerKey = getTriggerKey(spiderFlow.getId());
+				CronScheduleBuilder cronScheduleBuilder = CronScheduleBuilder.cronSchedule(spiderFlow.getCron())
+			    		.withMisfireHandlingInstructionDoNothing();
+				CronTrigger trigger = TriggerBuilder.newTrigger().withIdentity(getTriggerKey(spiderFlow.getId())).withSchedule(cronScheduleBuilder).build();
+				trigger.getJobDataMap().put(JOB_PARAM_NAME, spiderFlow);
+				scheduler.rescheduleJob(triggerKey, trigger);
+			}
+			return true;
+		} catch (SchedulerException e) {
+			logger.error("修改定时任务失败",e);
+			return false;
+		}
+	}
+	
+	public boolean run(SpiderFlow spiderFlow){
+		try {
+			JobDataMap dataMap = new JobDataMap();
+			dataMap.put(JOB_PARAM_NAME, spiderFlow);
+			scheduler.triggerJob(getJobKey(spiderFlow.getId()),dataMap);
+			return true;
+		} catch (SchedulerException e) {
+			logger.error("执行定时任务失败",e);
+			return false;
+		}
+	}
+	
+	public boolean remove(String id){
+		try {
+			scheduler.deleteJob(getJobKey(id));
+			return true;
+		} catch (SchedulerException e) {
+			logger.error("删除定时任务失败",e);
+			return false;
+		}
+	}
+
+}

src/main/java/com/mxd/spider/web/controller/SpiderFlowController.java

@@ -39,6 +39,21 @@ public class SpiderFlowController {
 		spiderFlowService.remove(id);
 	}
 	
+	@RequestMapping("/start")
+	public void start(String id){
+		spiderFlowService.start(id);
+	}
+	
+	@RequestMapping("/stop")
+	public void stop(String id){
+		spiderFlowService.stop(id);
+	}
+	
+	@RequestMapping("/cron")
+	public void cron(String id,String cron){
+		spiderFlowService.resetCornExpression(id, cron);
+	}
+	
 	@RequestMapping("/xml")
 	public String xml(String id){
 		return spiderFlowService.get(id).getXml();

src/main/java/com/mxd/spider/web/repository/SpiderFlowRepository.java

@@ -11,7 +11,7 @@ import com.mxd.spider.web.model.SpiderFlow;
 public interface SpiderFlowRepository extends JpaRepository<SpiderFlow, String>{
 
 	@Modifying
-	@Query(value = "insert into sp_flow(id,name,xml,enabled) values(?1,?2,?3,'1')",nativeQuery = true)
+	@Query(value = "insert into sp_flow(id,name,xml,enabled) values(?1,?2,?3,'0')",nativeQuery = true)
 	public int insertSpiderFlow(String id,String name,String xml);
 	
 	@Modifying

src/main/java/com/mxd/spider/web/service/SpiderFlowService.java

@@ -1,15 +1,23 @@
 package com.mxd.spider.web.service;
 
+import java.util.Date;
+import java.util.List;
 import java.util.UUID;
 
+import javax.annotation.PostConstruct;
 import javax.transaction.Transactional;
 
 import org.apache.commons.lang3.StringUtils;
+import org.quartz.CronScheduleBuilder;
+import org.quartz.CronTrigger;
+import org.quartz.TriggerBuilder;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Example;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 
+import com.mxd.spider.job.SpiderJobManager;
 import com.mxd.spider.web.model.SpiderFlow;
 import com.mxd.spider.web.repository.SpiderFlowRepository;
 
@@ -19,22 +27,87 @@ public class SpiderFlowService {
 	@Autowired
 	private SpiderFlowRepository repository;
 	
+	@Autowired
+	private SpiderJobManager spiderJobManager;
+	
+	@PostConstruct
+	private void initJobs(){
+		SpiderFlow spiderFlow = new SpiderFlow();
+		spiderFlow.setEnabled("1");
+		List<SpiderFlow> spiderFlows = repository.findAll(Example.of(spiderFlow));
+		if(spiderFlows != null){
+			for (SpiderFlow sf : spiderFlows) {
+				if(StringUtils.isNotEmpty(sf.getCron())){
+					spiderJobManager.addJob(sf);
+				}
+			}
+		}
+	}
+	
 	public Page<SpiderFlow> findAll(Pageable pageable){
 		return repository.findAll(pageable);
 	}
 	
 	@Transactional
-	public void save(SpiderFlow spiderFlow){
-		if(StringUtils.isNotEmpty(spiderFlow.getId())){	//修改
-			repository.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml());
-		}else{
-			String id = UUID.randomUUID().toString().replace("-", "");
-			repository.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml());
+	public int executeCountIncrement(String id, Date lastExecuteTime, Date nextExecuteTime){
+		return repository.executeCountIncrement(id, lastExecuteTime, nextExecuteTime);
+	}
+	@Transactional
+	public void resetCornExpression(String id, String cron){
+		CronTrigger trigger = TriggerBuilder.newTrigger()
+				.withIdentity("Caclulate Next Execute Date")
+				.withSchedule(CronScheduleBuilder.cronSchedule(cron))
+				.build();
+		repository.resetCornExpression(id, cron, trigger.getStartTime());
+		spiderJobManager.remove(id);
+		SpiderFlow spiderFlow = get(id);
+		if("1".equals(spiderFlow.getEnabled()) && StringUtils.isNotEmpty(spiderFlow.getCron())){
+			spiderJobManager.addJob(spiderFlow);
 		}
 	}
 	
+	@Transactional
+	public void save(SpiderFlow spiderFlow){
+		if(StringUtils.isNotEmpty(spiderFlow.getCron())){
+			CronTrigger trigger = TriggerBuilder.newTrigger()
+							.withIdentity("Caclulate Next Execute Date")
+							.withSchedule(CronScheduleBuilder.cronSchedule(spiderFlow.getCron()))
+							.build();
+			spiderFlow.setNextExecuteTime(trigger.getStartTime());
+		}
+		if(StringUtils.isNotEmpty(spiderFlow.getId())){	//修改
+			repository.updateSpiderFlow(spiderFlow.getId(), spiderFlow.getName(), spiderFlow.getXml());
+			spiderJobManager.remove(spiderFlow.getId());
+			spiderFlow = get(spiderFlow.getId());
+			if("1".equals(spiderFlow.getEnabled()) && StringUtils.isNotEmpty(spiderFlow.getCron())){
+				spiderJobManager.addJob(spiderFlow);
+			}
+			
+		}else{
+			String id = UUID.randomUUID().toString().replace("-", "");
+			repository.insertSpiderFlow(id, spiderFlow.getName(), spiderFlow.getXml());
+			spiderFlow.setId(id);
+		}
+	}
+	@Transactional
+	public void stop(String id){
+		repository.resetSpiderStatus(id,"0");
+		spiderJobManager.remove(id);
+	}
+	@Transactional
+	public void start(String id){
+		spiderJobManager.remove(id);
+		SpiderFlow spiderFlow = get(id);
+		spiderJobManager.addJob(spiderFlow);
+		repository.resetSpiderStatus(id,"1");
+	}
+	@Transactional
+	public void resetExecuteCount(String id){
+		repository.resetExecuteCount(id);
+	}
 	public void remove(String id){
 		repository.deleteById(id);
+		spiderJobManager.remove(id);
 	}
 	
 	public SpiderFlow get(String id){

src/main/resources/static/spiderList.html

@@ -90,10 +90,33 @@
 					})
 					layui.layer.close(index);
 				})
+			}).on('click','.btn-edit-cron',function(){
+				var id = $(this).data('id');
+				layui.layer.prompt({
+					title : '请输入cron表达式',
+					value : $(this).data('cron') || ''
+				},function(value,index){
+					$.ajax({
+						url : 'spider/cron',
+						data : {
+							id : id,
+							cron : value
+						},
+						success : function(){
+							layui.layer.msg('修改成功',{time : 500},function(){
+								$table.reload();
+							})	
+						},
+						error : function(){
+							layui.layer.msg('修改失败')	
+						}
+					})
+					layui.layer.close(index);
+				})
 			})
 		</script>
 		<script type="text/html" id="buttons">
-			<a class="layui-btn layui-btn-sm btn-edit-cron" data-id="{{d.id}}">编辑cron表达式</a>
+			<a class="layui-btn layui-btn-sm btn-edit-cron" data-id="{{d.id}}" data-cron="{{d.cron}}">编辑cron表达式</a>
 			<a class="layui-btn layui-btn-sm" href="editor.html?id={{d.id}}">编辑XML</a>
 			<a class="layui-btn layui-btn-sm btn-remove" data-id="{{d.id}}">删除</a>
 		</script>