Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修复前端日志不输出的BUG

Browse Source
This commit is contained in:
mxd 2019-12-26 10:33:36 +08:00
parent 1368ce07ce
commit 969ce00b18
4 changed files with 74 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java
View File

@ -141,7 +141,7 @@ public class Spider {
return;
}
//判断条件，如果不成立则不执行
if (!executeCondition(fromNode, node, context, variables)) {
if (!executeCondition(fromNode, node, variables)) {
return;
}
logger.debug("执行节点[{}:{}]", node.getNodeName(), node.getNodeId());
@ -189,6 +189,7 @@ public class Spider {
runnableNode.setState(RunnableNode.State.RUNNING);
if (context.isRunning()) {
try {
SpiderContextHolder.set(context);
//死循环检测，当执行节点次数大于阈值时，结束本次测试
AtomicInteger executeCount = context.get(ATOMIC_DEAD_CYCLE);
if (executeCount != null && executeCount.incrementAndGet() > deadCycle) {
@ -218,6 +219,7 @@ public class Spider {
if (node.isSync()) {
context.unlock();
}
SpiderContextHolder.remove();
}
}
});
@ -227,7 +229,7 @@ public class Spider {
}
}
private boolean executeCondition(SpiderNode fromNode, SpiderNode node, SpiderContext context, Map<String, Object> variables) {
private boolean executeCondition(SpiderNode fromNode, SpiderNode node, Map<String, Object> variables) {
if (fromNode != null) {
String condition = node.getCondition(fromNode.getNodeId());
if (StringUtils.isNotBlank(condition)) { // 判断是否有条件

13
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java
View File

@ -1,11 +1,8 @@
package org.spiderflow.core.job;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.quartz.*;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
@ -20,6 +17,10 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.quartz.QuartzJobBean;
import org.springframework.stereotype.Component;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* 爬虫定时执行
*
@ -68,8 +69,8 @@ public class SpiderJob extends QuartzJobBean {
task.setFlowId(spiderFlow.getId());
task.setBeginTime(new Date());
try {
taskService.save(task);
context = SpiderJobContext.create(this.spiderLogPath, spiderFlow.getId() + task.getId() + ".log");
taskService.save(task);
SpiderContextHolder.set(context);
contextMap.put(task.getId(), context);
logger.info("开始执行任务{}", spiderFlow.getName());

54
spider-flow-web/src/main/java/org/spiderflow/model/SpiderWebSocketContext.java
View File

@ -10,34 +10,40 @@ import java.util.Date;
/**
* WebSocket通讯中爬虫的上下文域
* @author Administrator
*
* @author Administrator
*/
public class SpiderWebSocketContext extends SpiderContext{
public class SpiderWebSocketContext extends SpiderContext {
private static final long serialVersionUID = -1205530535069540245L;
private Session session;
public SpiderWebSocketContext(Session session) {
this.session = session;
}
private static final long serialVersionUID = -1205530535069540245L;
@Override
public void addOutput(SpiderOutput output) {
super.addOutput(output);
this.write(new WebSocketEvent<>("output", output));
}
private Session session;
public void log(SpiderLog log) {
write(new WebSocketEvent<>("log", DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss.SSS"), log));
}
public synchronized <T> void write(WebSocketEvent<T> event){
try {
session.getAsyncRemote().sendText(JSON.toJSONString(event, FastJsonSerializer.serializeConfig));
} catch (Throwable ignored) {
public SpiderWebSocketContext(Session session) {
this.session = session;
}
}
}
@Override
public void addOutput(SpiderOutput output) {
super.addOutput(output);
this.write(new WebSocketEvent<>("output", output));
}
public void log(SpiderLog log) {
write(new WebSocketEvent<>("log", DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss.SSS"), log));
}
public <T> void write(WebSocketEvent<T> event) {
try {
String message = JSON.toJSONString(event, FastJsonSerializer.serializeConfig);
if(session.isOpen()){
synchronized (session){
session.getBasicRemote().sendText(message);
}
}else{
System.out.println("close:" + session.getId() + ":"+message);
}
} catch (Throwable ignored) {
}
}
}

69
spider-flow-web/src/main/java/org/spiderflow/websocket/WebSocketEditorServer.java
View File

@ -15,46 +15,43 @@ import javax.websocket.server.ServerEndpoint;
/**
* WebSocket通讯编辑服务
* @author Administrator
*
* @author Administrator
*/
@ServerEndpoint("/ws")
@Component
public class WebSocketEditorServer {
public static Spider spider;
private SpiderWebSocketContext context;
@OnMessage
public void onMessage(String message,Session session){
JSONObject event = JSON.parseObject(message);
if(context == null){
context = new SpiderWebSocketContext(session);
}
String eventType = event.getString("eventType");
if("test".equals(eventType)){
context.setRunning(true);
final SpiderWebSocketContext spiderContext = context;
new Thread(()->{
String xml = event.getString("message");
if(xml != null){
spider.runWithTest(SpiderFlowUtils.loadXMLFromString(xml), spiderContext);
spiderContext.write(new WebSocketEvent<>("finish", null));
}else{
spiderContext.write(new WebSocketEvent<>("error", "xml不正确！"));
}
}).start();
}else if("stop".equals(eventType)){
context.setRunning(false);
}
}
@OnClose
public void onClose(Session session){
if(context != null){
context.setRunning(false);
}
}
public static Spider spider;
private SpiderWebSocketContext context;
@OnMessage
public void onMessage(String message, Session session) {
JSONObject event = JSON.parseObject(message);
String eventType = event.getString("eventType");
if ("test".equals(eventType)) {
context = new SpiderWebSocketContext(session);
context.setRunning(true);
new Thread(() -> {
String xml = event.getString("message");
if (xml != null) {
spider.runWithTest(SpiderFlowUtils.loadXMLFromString(xml), context);
context.write(new WebSocketEvent<>("finish", null));
} else {
context.write(new WebSocketEvent<>("error", "xml不正确！"));
}
context.setRunning(false);
}).start();
} else if ("stop".equals(eventType) && context != null) {
context.setRunning(false);
}
}
@OnClose
public void onClose(Session session) {
System.out.println("close");
context.setRunning(false);
}
}