Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加调度日志记录、查看日志功能

Browse Source
This commit is contained in:
mxd 2019-10-12 12:13:43 +08:00
parent 9aa32f9f0c
commit 34c9e1d077
10 changed files with 166 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spider-flow-api/src/main/java/org/spiderflow/model/SpiderLog.java
View File

@ -2,6 +2,7 @@ package org.spiderflow.model;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.commons.lang3.exception.ExceptionUtils;
@ -31,7 +32,7 @@ public class SpiderLog {
}
public SpiderLog(String level,String message, Object ... variables) {
this(level,message,Arrays.asList(variables));
this(level,message,variables == null ? Collections.emptyList() : Arrays.asList(variables));
}
public String getLevel() {

7
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java
View File

@ -54,18 +54,17 @@ public class Spider {
executorMap = executors.stream().collect(Collectors.toMap(ShapeExecutor::supportShape, v -> v));
}
public List<SpiderOutput> run(SpiderFlow spiderFlow, Map<String, Object> variables) {
public List<SpiderOutput> run(SpiderFlow spiderFlow, SpiderContext context,Map<String, Object> variables) {
if (variables == null) {
variables = new HashMap<>();
}
SpiderNode root = SpiderFlowUtils.loadXMLFromString(spiderFlow.getXml());
SpiderContext context = new SpiderContext();
executeRoot(root, context, variables);
return context.getOutputs();
}
public List<SpiderOutput> run(SpiderFlow spiderFlow) {
return run(spiderFlow, new HashMap<>());
public List<SpiderOutput> run(SpiderFlow spiderFlow, SpiderContext context) {
return run(spiderFlow, context, new HashMap<>());
}
public List<SpiderOutput> runWithTest(SpiderNode root, SpiderContext context) {

20
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java
View File

@ -6,8 +6,6 @@ import org.apache.commons.lang3.time.DateFormatUtils;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.core.Spider;
import org.spiderflow.core.model.SpiderFlow;
import org.spiderflow.core.service.SpiderFlowService;
@ -24,14 +22,15 @@ import org.springframework.stereotype.Component;
@Component
public class SpiderJob extends QuartzJobBean{
private static Logger logger = LoggerFactory.getLogger(SpiderJob.class);
private static Spider spider;
private static SpiderFlowService spiderFlowService;
@Value("${spider.job.enable:true}")
private boolean spiderJobEnable;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
@Override
protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
@ -49,12 +48,15 @@ public class SpiderJob extends QuartzJobBean{
public void run(SpiderFlow spiderFlow,Date nextExecuteTime){
Date now = new Date();
try {
logger.info("开始执行任务{}",spiderFlow.getName());
spider.run(spiderFlow);
logger.info("执行任务{}完毕，下次执行时间：{}",spiderFlow.getName(),nextExecuteTime == null ? null: DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
SpiderJobContext context = SpiderJobContext.create(this.spiderLogPath, spiderFlow.getId() + ".log");
try{
context.info("开始执行任务{}",spiderFlow.getName());
spider.run(spiderFlow,context);
context.info("执行任务{}完毕，下次执行时间：{}",spiderFlow.getName(),nextExecuteTime == null ? null: DateFormatUtils.format(nextExecuteTime, "yyyy-MM-dd HH:mm:ss"));
} catch (Exception e) {
logger.error("执行任务{}出错",spiderFlow.getName(),e);
context.error("执行任务{}出错",spiderFlow.getName(),e);
} finally{
context.close();
}
spiderFlowService.executeCountIncrement(spiderFlow.getId(), now, nextExecuteTime);
}

62
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJobContext.java Normal file
View File

@ -0,0 +1,62 @@
package org.spiderflow.core.job;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.Date;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.model.SpiderLog;
public class SpiderJobContext extends SpiderContext{
private static final long serialVersionUID = 9099787449108938453L;
private static Logger logger = LoggerFactory.getLogger(SpiderJobContext.class);
private OutputStream outputstream;
public SpiderJobContext(OutputStream outputstream) {
super();
this.outputstream = outputstream;
}
@Override
public void log(SpiderLog log) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("[").append(log.getLevel()).append("] ")
.append(DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss.SSS"))
.append(" ").append(MessageFormatter.arrayFormat(log.getMessage(), log.getVariables().toArray()).getMessage())
.append("\r\n");
try {
outputstream.write(stringBuffer.toString().getBytes("UTF-8"));
outputstream.flush();
} catch (Exception e) {
}
}
public void close(){
try {
this.outputstream.close();
} catch (Exception e) {
}
}
public static SpiderJobContext create(String directory,String file){
OutputStream os = null;
try {
File dirFile = new File(directory);
if(!dirFile.exists()){
dirFile.mkdirs();
}
os = new FileOutputStream(new File(directory,file), true);
} catch (Exception e) {
logger.error("创建日志文件出错",e);
}
return new SpiderJobContext(os);
}
}

25
spider-flow-web/src/main/java/org/spiderflow/controller/SpiderFlowController.java
View File

@ -1,5 +1,7 @@
package org.spiderflow.controller;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -7,7 +9,10 @@ import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.Grammerable;
import org.spiderflow.annotation.Comment;
import org.spiderflow.core.model.SpiderFlow;
@ -21,6 +26,7 @@ import org.spiderflow.model.JsonBean;
import org.spiderflow.model.Plugin;
import org.spiderflow.model.Shape;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -57,8 +63,13 @@ public class SpiderFlowController {
@Autowired(required = false)
private List<PluginConfig> pluginConfigs;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
private final List<Grammer> grammers = new ArrayList<Grammer>();
private static Logger logger = LoggerFactory.getLogger(SpiderFlowController.class);
@PostConstruct
private void init(){
for (FunctionExecutor executor : functionExecutors) {
@ -142,6 +153,20 @@ public class SpiderFlowController {
return spiderFlowService.getById(id).getXml();
}
@RequestMapping("/log")
public String log(String id){
SpiderFlow flow = spiderFlowService.getById(id);
if(flow == null){
return "未找到此爬虫";
}
try {
return FileUtils.readFileToString(new File(spiderLogPath,id + ".log"), "UTF-8");
} catch (IOException e) {
logger.error("读取日志文件出错",e);
return "读取日志文件出错，" + e.getMessage();
}
}
@RequestMapping("/shapes")
public List<Shape> shapes(){
return executors.stream().filter(e-> e.shape() !=null).map(executor -> executor.shape()).collect(Collectors.toList());

10
spider-flow-web/src/main/java/org/spiderflow/controller/SpiderRestController.java
View File

@ -6,11 +6,13 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.core.Spider;
import org.spiderflow.core.job.SpiderJobContext;
import org.spiderflow.core.model.SpiderFlow;
import org.spiderflow.core.service.SpiderFlowService;
import org.spiderflow.model.JsonBean;
import org.spiderflow.model.SpiderOutput;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -28,6 +30,9 @@ public class SpiderRestController {
@Autowired
private Spider spider;
@Value("${spider.job.log.path:./}")
private String spiderLogPath;
@RequestMapping("/run/{id}")
public JsonBean<List<SpiderOutput>> run(@PathVariable("id")String id,@RequestBody(required = false)Map<String,Object> params){
SpiderFlow flow = spiderFlowService.getById(id);
@ -35,11 +40,14 @@ public class SpiderRestController {
return new JsonBean<>(0, "找不到此爬虫信息");
}
List<SpiderOutput> outputs = null;
SpiderJobContext context = SpiderJobContext.create(spiderLogPath, id + ".log");
try{
outputs = spider.run(flow,params);
outputs = spider.run(flow,context, params);
}catch(Exception e){
logger.error("执行爬虫失败",e);
return new JsonBean<>(-1, "执行失败");
} finally{
context.close();
}
return new JsonBean<>(outputs);
}

3
spider-flow-web/src/main/resources/application.properties
View File

@ -3,7 +3,10 @@ server.port=8088
logging.level.root=INFO
#logging.level.org.spiderflow=DEBUG
#设置为true时定时任务才生效
spider.job.enable=false
#爬虫任务的执行日志保存路径
spider.job.log.path=/data/spider/logs/
spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
spring.jackson.time-zone=GMT+8

30
spider-flow-web/src/main/resources/static/js/index.js
View File

@ -1,20 +1,20 @@
var $ = layui.$;
$(function(){
var openTab = function(title,href){
if($(".layui-tab[lay-filter=admin-tab]").find("[lay-id="+title+"]").length > 0){ //判断是否已打开
var $dom = $(".layui-tab[lay-filter=admin-tab]");
var index = $dom.find("[lay-id="+title+"]").index();
$dom.find(".layui-tab-content .layui-tab-item").eq(index).find("iframe").attr("src",href);
}else{
var html = '<iframe src="'+href+'" width="100%" height="100%" scrolling="yes" frameborder="0"></iframe>';
layui.element.tabAdd('admin-tab',{
title:title,
content:html,
id:title,
});
}
layui.element.tabChange("admin-tab",title);
function openTab(title,href){
if($(".layui-tab[lay-filter=admin-tab]").find("[lay-id="+title+"]").length > 0){ //判断是否已打开
var $dom = $(".layui-tab[lay-filter=admin-tab]");
var index = $dom.find("[lay-id="+title+"]").index();
$dom.find(".layui-tab-content .layui-tab-item").eq(index).find("iframe").attr("src",href);
}else{
var html = '<iframe src="'+href+'" width="100%" height="100%" scrolling="yes" frameborder="0"></iframe>';
layui.element.tabAdd('admin-tab',{
title:title,
content:html,
id:title,
});
}
layui.element.tabChange("admin-tab",title);
}
$(function(){
layui.element.init();
$("body").on('click','.menu-list li a',function(){
$(this).parents("ul").siblings().find("li.layui-this,dd.layui-this").removeClass('layui-this')

32
spider-flow-web/src/main/resources/static/log.html Normal file
View File

@ -0,0 +1,32 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>SpiderFlow</title>
<script type="text/javascript" src="js/layui/layui.all.js" ></script>
<script type="text/javascript" src="js/common.js" ></script>
<style type="text/css">
html,body,.log-container{
width : 100%;
height : 100%;
font-weight: bold;
font-family: Consolas;
font-size: 12px;
}
</style>
</head>
<div class="log-container"></div>
<script type="text/javascript">
var id = getQueryString('id');
layui.$.ajax({
url : 'spider/log',
data : {
id : id
},
dataType : 'text',
success : function(content){
layui.$('.log-container').html(content.replace(/\n/g,'<br>').replace(/ /g,'&nbsp;').replace(/\t/g,'&nbsp;&nbsp;&nbsp;&nbsp;'));
}
})
</script>
</html>

5
spider-flow-web/src/main/resources/static/spiderList.html
View File

@ -72,7 +72,7 @@
align : 'center'
},{
title : '操作',
width : 120,
width : 200,
align : 'center',
templet : '#buttons'
}]]
@ -141,6 +141,8 @@
})
layui.layer.close(index);
})
}).on('click','.btn-log',function(){
parent.openTab($(this).data('name') + '-日志','log.html?id=' + $(this).data('id'));
}).on('click','.btn-edit-cron',function(){
var id = $(this).data('id');
var value = $(this).data('cron') || '';
@ -155,6 +157,7 @@
</script>
<script type="text/html" id="buttons">
<a class="layui-btn layui-btn-sm btn-run" data-id="{{d.id}}">手动运行</a>
<a class="layui-btn layui-btn-sm btn-log" data-name="{{d.name}}" data-id="{{d.id}}">查看日志</a>
<a class="layui-btn layui-btn-sm btn-remove" data-id="{{d.id}}">删除</a>
</script>
</body>