Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1、新增移除SpiderWebSocketContext中序列化器到FastJsonSerializer中

Browse Source 
2、新增批量参数设置
3、新增输出全部功能
This commit is contained in:
td_zhangyu 2019-12-23 17:39:31 +08:00
parent a55ac0f8ca
commit 53ccffd2fd
4 changed files with 78 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/OutputExecutor.java
View File

@ -1,12 +1,19 @@
package org.spiderflow.core.executor.shape;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.List;
import java.util.Map;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializeConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.ExpressionEngine;
import org.spiderflow.context.RunnableTreeNode;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.core.io.HttpResponse;
import org.spiderflow.core.serializer.FastJsonSerializer;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.SpiderNode;
import org.spiderflow.model.SpiderOutput;
@ -20,6 +27,19 @@ import org.springframework.stereotype.Component;
*/
@Component
public class OutputExecutor implements ShapeExecutor{
public static SerializeConfig serializeConfig;
static {
serializeConfig = new SerializeConfig();
FastJsonSerializer serializer = new FastJsonSerializer();
serializeConfig.put(Long.TYPE, serializer);
serializeConfig.put(Long.class, serializer);
serializeConfig.put(BigDecimal.class, serializer);
serializeConfig.put(BigInteger.class, serializer);
}
public static final String OUTPUT_ALL = "output-all";
public static final String OUTPUT_NAME = "output-name";
@ -35,7 +55,29 @@ public class OutputExecutor implements ShapeExecutor{
SpiderOutput output = new SpiderOutput();
output.setNodeName(node.getNodeName());
output.setNodeId(node.getNodeId());
List<Map<String, String>> outputs = node.getListJsonValue(OUTPUT_NAME,OUTPUT_VALUE);
boolean outputAll = !"0".equals(node.getStringJsonValue(OUTPUT_ALL));
if (outputAll) {
for (Map.Entry<String, Object> item : variables.entrySet()) {
Object value = item.getValue();
//去除不能序列化的参数
try {
JSON.toJSONString(value, serializeConfig);
} catch (Exception e) {
continue;
}
//其他不输出的信息
if("ex".equals(item.getKey()) || value instanceof RunnableTreeNode || value instanceof HttpResponse) {
continue;
}
try {
logger.debug("输出{}={}", item.getKey(), item.getValue());
} catch (Exception e) {
logger.error("输出{}出错，异常信息：{}", item.getKey(), e);
}
output.addOutput(item.getKey(), item.getValue());
}
}
List<Map<String, String>> outputs = node.getListJsonValue(OUTPUT_NAME, OUTPUT_VALUE);
for (Map<String, String> item : outputs) {
Object value = null;
String outputValue = item.get(OUTPUT_VALUE);

26
spider-flow-core/src/main/java/org/spiderflow/core/serializer/FastJsonSerializer.java Normal file
View File

@ -0,0 +1,26 @@
package org.spiderflow.core.serializer;
import com.alibaba.fastjson.serializer.JSONSerializer;
import com.alibaba.fastjson.serializer.ObjectSerializer;
import com.alibaba.fastjson.serializer.SerializerFeature;
import java.io.IOException;
import java.lang.reflect.Type;
/**
* Created on 2019-12-23
*/
public class FastJsonSerializer implements ObjectSerializer {
@Override
public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException {
if(object == null){
if(serializer.isEnabled(SerializerFeature.WriteNullNumberAsZero)){
serializer.out.write("0");
}else{
serializer.out.writeNull();
}
return;
}
serializer.out.writeString(object.toString());
}
}

25
spider-flow-web/src/main/java/org/spiderflow/model/SpiderWebSocketContext.java
View File

@ -16,27 +16,21 @@ import com.alibaba.fastjson.serializer.JSONSerializer;
import com.alibaba.fastjson.serializer.ObjectSerializer;
import com.alibaba.fastjson.serializer.SerializeConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import org.spiderflow.core.executor.shape.OutputExecutor;
/**
* WebSocket通讯中爬虫的上下文域
* @author Administrator
*
*/
public class SpiderWebSocketContext extends SpiderContext implements ObjectSerializer{
public class SpiderWebSocketContext extends SpiderContext{
private static final long serialVersionUID = -1205530535069540245L;
private Session session;
private SerializeConfig serializeConfig;
public SpiderWebSocketContext(Session session) {
this.session = session;
this.serializeConfig = new SerializeConfig();
this.serializeConfig.put(Long.TYPE, this);
this.serializeConfig.put(Long.class, this);
this.serializeConfig.put(BigDecimal.class, this);
this.serializeConfig.put(BigInteger.class, this);
}
@Override
@ -53,7 +47,7 @@ public class SpiderWebSocketContext extends SpiderContext implements ObjectSeria
synchronized (session) {
if (session.isOpen()) {
try {
session.getBasicRemote().sendText(JSON.toJSONString(event,this.serializeConfig));
session.getBasicRemote().sendText(JSON.toJSONString(event, OutputExecutor.serializeConfig));
} catch (IOException e) {
//忽略异常
}
@ -61,17 +55,4 @@ public class SpiderWebSocketContext extends SpiderContext implements ObjectSeria
}
}
@Override
public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features) throws IOException {
if(object == null){
if(serializer.isEnabled(SerializerFeature.WriteNullNumberAsZero)){
serializer.out.write("0");
}else{
serializer.out.writeNull();
}
return;
}
serializer.out.writeString(object.toString());
}
}

6
spider-flow-web/src/main/resources/static/resources/templates/output.html
View File

@ -21,6 +21,12 @@
<label class="layui-form-label">循环次数</label>
<div class="layui-input-block" codemirror="loopCount" placeholder="请输入循环次数" data-value="{{=d.data.object.loopCount}}"></div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">输出设置</label>
<div class="layui-input-block">
<input type="checkbox" title="输出全部参数" value="output-all" lay-skin="primary" {{d.data.object['output-all'] == '0' ? '' : 'checked'}}/>
</div>
</div>
{{# layui.each(d.data.object['output-name'],function(index,output){ }}
<div id="output{{=index}}" class="draggable" draggable="true" ondragstart="drag(event)" ondrop="drop(event)" ondragover="allowDrop(event)">
<hr>