Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!15 魔改版多线程

Browse Source 
Merge pull request !15 from 我是D/dev
This commit is contained in:
小东 2019-12-19 18:04:46 +08:00 committed by Gitee
commit fdbb1f1096
25 changed files with 460 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
spider-flow-api/src/main/java/org/spiderflow/concurrent/CountableThreadPool.java Normal file
View File

@ -0,0 +1,89 @@
package org.spiderflow.concurrent;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
/**
* 同步线程池
* Created on 2019-12-16
*/
public class CountableThreadPool {
private int threadNum;
private AtomicInteger threadAlive = new AtomicInteger();
private ReentrantLock reentrantLock = new ReentrantLock();
private Condition condition = reentrantLock.newCondition();
public CountableThreadPool(int threadNum) {
this.threadNum = threadNum;
this.executorService = Executors.newFixedThreadPool(threadNum);
}
public CountableThreadPool(int threadNum, ExecutorService executorService) {
this.threadNum = threadNum;
this.executorService = executorService;
}
public void setExecutorService(ExecutorService executorService) {
this.executorService = executorService;
}
public int getThreadAlive() {
return threadAlive.get();
}
public int getThreadNum() {
return threadNum;
}
private ExecutorService executorService;
public void execute(final Runnable runnable) {
if (threadAlive.get() >= threadNum) {
try {
reentrantLock.lock();
while (threadAlive.get() >= threadNum) {
try {
condition.await();
} catch (InterruptedException e) {
}
}
} finally {
reentrantLock.unlock();
}
}
threadAlive.incrementAndGet();
executorService.execute(new Runnable() {
@Override
public void run() {
try {
runnable.run();
} finally {
try {
reentrantLock.lock();
threadAlive.decrementAndGet();
condition.signal();
} finally {
reentrantLock.unlock();
}
}
}
});
}
public boolean isShutdown() {
return executorService.isShutdown();
}
public void shutdown() {
executorService.shutdown();
}
}

50
spider-flow-api/src/main/java/org/spiderflow/context/SpiderContext.java
View File

@ -4,9 +4,11 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.concurrent.CountableThreadPool;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import org.spiderflow.model.SpiderLog;
import org.spiderflow.model.SpiderNode;
@ -28,9 +30,22 @@ public class SpiderContext extends HashMap<String, Object>{
* 爬虫输出参数列表
*/
private List<SpiderOutput> outputs = new ArrayList<>();
private SubThreadPoolExecutor threadPool;
/**
* 流程执行线程(异步执行)
*/
private SubThreadPoolExecutor flowPool;
/**
* 末端任务执行线程(同步执行)
*/
private CountableThreadPool taskPool;
/**
* 处理流程同步锁
*/
private ReentrantLock lock = new ReentrantLock();
private SpiderNode rootNode;
private boolean running = true;
@ -65,16 +80,23 @@ public class SpiderContext extends HashMap<String, Object>{
public void addOutput(SpiderOutput output){
this.outputs.add(output);
}
public SubThreadPoolExecutor getThreadPool() {
return threadPool;
public SubThreadPoolExecutor getFlowPool() {
return flowPool;
}
public void setThreadPool(SubThreadPoolExecutor threadPool) {
this.threadPool = threadPool;
public void setFlowPool(SubThreadPoolExecutor flowPool) {
this.flowPool = flowPool;
}
public CountableThreadPool getTaskPool() {
return taskPool;
}
public void setTaskPool(CountableThreadPool taskPool) {
this.taskPool = taskPool;
}
public SpiderNode getRootNode() {
return rootNode;
}
@ -123,4 +145,12 @@ public class SpiderContext extends HashMap<String, Object>{
public void log(SpiderLog log){
}
public void lock(){
lock.lock();
}
public void unlock(){
lock.unlock();
}
}

10
spider-flow-api/src/main/java/org/spiderflow/model/Shape.java
View File

@ -10,6 +10,8 @@ public class Shape {
private String image;
private String desc;
public String getName() {
return name;
}
@ -41,4 +43,12 @@ public class Shape {
public void setImage(String image) {
this.image = image;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
}

11
spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java
View File

@ -36,7 +36,8 @@ public class SpiderNode {
* 节点ID
*/
private String nodeId;
private boolean sync = false;
public String getNodeId() {
return nodeId;
@ -114,6 +115,14 @@ public class SpiderNode {
this.condition.put(fromNodeId, condition);
}
public boolean isSync() {
return sync;
}
public void setSync(boolean sync) {
this.sync = sync;
}
@Override
public String toString() {
return "SpiderNode [jsonProperty=" + jsonProperty + ", nextNodes=" + nextNodes + ", condition=" + condition

10
spider-flow-api/src/main/java/org/spiderflow/model/SpiderOutput.java
View File

@ -20,38 +20,31 @@ public class SpiderOutput {
*/
private List<String> outputNames = new ArrayList<>();
/**
* 输出项的值
*/
private List<Object> values = new ArrayList<>();
public String getNodeName() {
return nodeName;
}
public void setNodeName(String nodeName) {
this.nodeName = nodeName;
}
public List<String> getOutputNames() {
return outputNames;
}
public void setOutputNames(List<String> outputNames) {
this.outputNames = outputNames;
}
public List<Object> getValues() {
return values;
}
public void setValues(List<Object> values) {
this.values = values;
}
@ -61,17 +54,14 @@ public class SpiderOutput {
this.values.add(value);
}
public String getNodeId() {
return nodeId;
}
public void setNodeId(String nodeId) {
this.nodeId = nodeId;
}
@Override
public String toString() {
return "SpiderOutput [nodeName=" + nodeName + ", nodeId=" + nodeId + ", outputNames=" + outputNames

25
spider-flow-core/pom.xml
View File

@ -32,16 +32,21 @@
<version>1.11.3</version>
</dependency>
<dependency>
<groupId>cn.wanghaomiao</groupId>
<artifactId>JsoupXpath</artifactId>
<version>2.3.2</version>
<exclusions>
<exclusion>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
</exclusion>
</exclusions>
</dependency>
<groupId>us.codecraft</groupId>
<artifactId>xsoup</artifactId>
<version>0.3.1</version>
</dependency>
<!--<dependency>-->
<!--<groupId>cn.wanghaomiao</groupId>-->
<!--<artifactId>JsoupXpath</artifactId>-->
<!--<version>2.3.2</version>-->
<!--<exclusions>-->
<!--<exclusion>-->
<!--<groupId>org.jsoup</groupId>-->
<!--<artifactId>jsoup</artifactId>-->
<!--</exclusion>-->
<!--</exclusions>-->
<!--</dependency> -->
<dependency>
<groupId>org.spiderflow</groupId>
<artifactId>spider-flow-api</artifactId>

63
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java
View File

@ -3,6 +3,7 @@ package org.spiderflow.core;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.spiderflow.ExpressionEngine;
import org.spiderflow.concurrent.CountableThreadPool;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor;
import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor;
import org.spiderflow.context.RunnableNode;
@ -24,6 +25,7 @@ import javax.annotation.PostConstruct;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
/**
@ -63,7 +65,7 @@ public class Spider {
private static SpiderFlowThreadPoolExecutor executor;
private static final String ATOMIC_DEAD_CYCLE = "__atomic_dead_cycle";
@PostConstruct
private void init() {
executorMap = executors.stream().collect(Collectors.toMap(ShapeExecutor::supportShape, v -> v));
@ -96,15 +98,17 @@ public class Spider {
private void executeRoot(SpiderNode root, SpiderContext context, Map<String, Object> variables) {
int nThreads = NumberUtils.toInt(root.getStringJsonValue(ShapeExecutor.THREAD_COUNT), defaultThreads);
SubThreadPoolExecutor pool = executor.createSubThreadPoolExecutor(nThreads);
SubThreadPoolExecutor flowPool = executor.createSubThreadPoolExecutor(nThreads);
CountableThreadPool taskPool = new CountableThreadPool(nThreads);
context.setRootNode(root);
context.setThreadPool(pool);
context.setFlowPool(flowPool);
context.setTaskPool(taskPool);
if (listeners != null) {
listeners.forEach(listener -> listener.beforeStart(context));
}
try {
executeNode(pool, null, root, context, variables);
pool.awaitTermination();
executeNode(null, root, context, variables);
flowPool.awaitTermination();
} finally {
if (listeners != null) {
listeners.forEach(listener -> listener.afterEnd(context));
@ -113,25 +117,26 @@ public class Spider {
}
public void execute(int nThreads, SpiderNode fromNode, SpiderNode node, SpiderContext context, Map<String, Object> variables) {
SubThreadPoolExecutor pool = executor.createSubThreadPoolExecutor(nThreads);
context.setThreadPool(pool);
executeNode(pool, fromNode, node, context, variables);
pool.awaitTermination();
SubThreadPoolExecutor flowPool = executor.createSubThreadPoolExecutor(nThreads);
CountableThreadPool taskPool = new CountableThreadPool(nThreads);
context.setFlowPool(flowPool);
context.setTaskPool(taskPool);
executeNode(fromNode, node, context, variables);
}
private void executeNextNodes(SubThreadPoolExecutor pool, SpiderNode node, SpiderContext context, Map<String, Object> variables) {
private void executeNextNodes(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
List<SpiderNode> nextNodes = node.getNextNodes();
if (nextNodes != null) {
for (SpiderNode nextNode : nextNodes) {
executeNode(pool, node, nextNode, context, variables);
executeNode(node, nextNode, context, variables);
}
}
}
public void executeNode(SubThreadPoolExecutor pool, SpiderNode fromNode, SpiderNode node, SpiderContext context, Map<String, Object> variables) {
public void executeNode(SpiderNode fromNode, SpiderNode node, SpiderContext context, Map<String, Object> variables) {
String shape = node.getStringJsonValue("shape");
if (StringUtils.isBlank(shape)) {
executeNextNodes(pool, node, context, variables);
executeNextNodes(node, context, variables);
return;
}
//判断条件，如果不成立则不执行
@ -196,24 +201,32 @@ public class Spider {
nVariables.put("ex", t);
context.error("执行节点[{}:{}]出错,异常信息：{}", node.getNodeName(), node.getNodeId(), t);
} finally {
synchronized (context){
//设置当前线程为已完成状态
runnableNode.setState(RunnableNode.State.DONE);
//判断是否允许执行后续节点
if (executor.allowExecuteNext(node, context, nVariables)) {
context.debug("执行节点[{}:{}]完毕", node.getNodeName(), node.getNodeId());
// 递归执行下一级节点
executeNextNodes(pool, node, context, nVariables);
} else {
context.debug("执行节点[{}:{}]完毕，忽略执行下一节点", node.getNodeName(), node.getNodeId());
}
if(node.isSync()){
context.lock();
}
//设置当前线程为已完成状态
runnableNode.setState(RunnableNode.State.DONE);
//判断是否允许执行后续节点
if (executor.allowExecuteNext(node, context, nVariables)) {
context.debug("执行节点[{}:{}]完毕", node.getNodeName(), node.getNodeId());
// 递归执行下一级节点
executeNextNodes(node, context, nVariables);
} else {
context.debug("执行节点[{}:{}]完毕，忽略执行下一节点", node.getNodeName(), node.getNodeId());
}
if(node.isSync()){
context.unlock();
}
}
}
});
}
}
runnables.forEach(executor.isThread() ? pool::submit : Runnable::run);
if (node.getNextNodes() == null || node.getNextNodes().isEmpty()) {
runnables.forEach(executor.isThread() ? context.getTaskPool()::execute : Runnable::run);
} else {
runnables.forEach(executor.isThread() ? context.getFlowPool()::submit : Runnable::run);
}
}
}

2
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java
View File

@ -92,7 +92,7 @@ public class FileFunctionExecutor implements FunctionExecutor{
}
@Comment("下载Url资源")
@Example("${file.write('e:/result.html',urls)}")
@Example("${file.write('e:/downloadPath',urls)}")
public static void write(String path, List<String> urls) throws IOException{
if(!CollectionUtils.isEmpty(urls)) {
for (String url : urls) {

2
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/ThreadFunctionExecutor.java
View File

@ -24,7 +24,7 @@ public class ThreadFunctionExecutor implements FunctionExecutor {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

8
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ElementFunctionExtension.java
View File

@ -22,16 +22,16 @@ public class ElementFunctionExtension implements FunctionExtension{
@Comment("根据xpath提取内容")
@Example("${elementVar.xpath('//title/text()')}")
@Return({Element.class,String.class})
public static Object xpath(Element element,String xpath){
return ExtractUtils.getObjectValueByXPath(element, xpath);
public static String xpath(Element element,String xpath){
return ExtractUtils.getValueByXPath(element, xpath);
}
@Comment("根据xpath提取内容")
@Example("${elementVar.xpaths('//h2/text()')}")
@Return({Element.class,String.class})
public static List<Object> xpaths(Element element,String xpath){
return ExtractUtils.getObjectValuesByXPath(element, xpath);
public static List<String> xpaths(Element element,String xpath){
return ExtractUtils.getValuesByXPath(element, xpath);
}
@Comment("根据正则表达式提取内容")

8
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ElementsFunctionExtension.java
View File

@ -22,15 +22,15 @@ public class ElementsFunctionExtension implements FunctionExtension{
@Comment("根据xpath提取内容")
@Example("${elementsVar.xpath('//title/text()')}")
@Return({Element.class,String.class})
public static Object xpath(Elements elements,String xpath){
return ExtractUtils.getObjectValueByXPath(elements, xpath);
public static String xpath(Elements elements,String xpath){
return ExtractUtils.getValueByXPath(elements, xpath);
}
@Comment("根据xpath提取内容")
@Example("${elementsVar.xpaths('//h2/text()')}")
@Return({Element.class,String.class})
public static List<Object> xpaths(Elements elements,String xpath){
return ExtractUtils.getObjectValuesByXPath(elements, xpath);
public static List<String> xpaths(Elements elements,String xpath){
return ExtractUtils.getValuesByXPath(elements, xpath);
}
@Comment("根据正则表达式提取内容")

6
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ListFunctionExtension.java
View File

@ -33,7 +33,11 @@ public class ListFunctionExtension implements FunctionExtension{
@Comment("将list用separator拼接起来")
@Example("${listVar.join('-')}")
public static String join(List<?> list,String separator){
return StringUtils.join(list.toArray(),separator);
if(list.size() == 1){
return list.get(0).toString();
}else{
return StringUtils.join(list.toArray(),separator);
}
}
@Comment("将list打乱顺序")

21
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/ResponseFunctionExtension.java
View File

@ -33,14 +33,14 @@ public class ResponseFunctionExtension implements FunctionExtension {
@Comment("根据xpath在请求结果中查找")
@Example("${resp.xpath('//title/text()')}")
@Return({Element.class, String.class})
public static Object xpath(SpiderResponse response, String xpath) {
return ExtractUtils.getObjectValueByXPath(element(response), xpath);
public static String xpath(SpiderResponse response, String xpath) {
return ExtractUtils.getValueByXPath(element(response), xpath);
}
@Comment("根据xpath在请求结果中查找")
@Example("${resp.xpaths('//a/@href')}")
public static List<Object> xpaths(SpiderResponse response, String xpath) {
return ExtractUtils.getObjectValuesByXPath(element(response), xpath);
public static List<String> xpaths(SpiderResponse response, String xpath) {
return ExtractUtils.getValuesByXPath(element(response), xpath);
}
@Comment("根据正则表达式提取请求结果中的内容")
@ -119,9 +119,20 @@ public class ResponseFunctionExtension implements FunctionExtension {
@Comment("获取当前页面所有图片链接")
@Example("${resp.images()}")
public static List<String> images(SpiderResponse response) {
return ExtractUtils.getAttrBySelector(element(response), "img", "abs:src")
return ExtractUtils.getAttrBySelector(element(response), "img", "src")
.stream()
.filter(link -> StringUtils.isNotBlank(link))
.collect(Collectors.toList());
}
@Comment("暂停当前线程")
@Example("${resp.sleep(1000L)}")
public static SpiderResponse sleep(SpiderResponse response,Long sleep) {
try {
Thread.sleep(sleep);
} catch (InterruptedException e) {
e.printStackTrace();
}
return response;
}
}

10
spider-flow-core/src/main/java/org/spiderflow/core/executor/function/extension/StringFunctionExtension.java
View File

@ -65,14 +65,14 @@ public class StringFunctionExtension implements FunctionExtension{
@Comment("根据xpath在String变量中查找")
@Example("${strVar.xpath('//title/text()')}")
@Return({Element.class,String.class})
public static Object xpath(String source,String xpath){
return ExtractUtils.getObjectValueByXPath(Jsoup.parse(source), xpath);
public static String xpath(String source,String xpath){
return ExtractUtils.getValueByXPath(Jsoup.parse(source), xpath);
}
@Comment("根据xpath在String变量中查找")
@Example("${resp.xpaths('//a/@href')}")
public static List<Object> xpaths(String source,String xpath){
return ExtractUtils.getObjectValuesByXPath(Jsoup.parse(source), xpath);
public static List<String> xpaths(String source,String xpath){
return ExtractUtils.getValuesByXPath(Jsoup.parse(source), xpath);
}
@Comment("将String变量转为Element对象")
@ -92,7 +92,7 @@ public class StringFunctionExtension implements FunctionExtension{
public static Elements selectors(String source,String cssQuery){
return element(source).select(cssQuery);
}
@Comment("将string转为json对象")
@Example("${strVar.json()}")
public static Object json(String source){

2
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopJoinExecutor.java
View File

@ -19,7 +19,7 @@ import java.util.stream.Collectors;
@Component
public class LoopJoinExecutor implements ShapeExecutor {
private static final String JOIN_NODE_ID = "joinNode";
public static final String JOIN_NODE_ID = "joinNode";
public static final String VARIABLE_CONTEXT = "__variable_context";

3
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/ProcessExecutor.java
View File

@ -35,11 +35,10 @@ public class ProcessExecutor implements ShapeExecutor{
if(spiderFlow != null){
context.debug("执行子流程:{}", spiderFlow.getName());
SpiderNode root = SpiderFlowUtils.loadXMLFromString(spiderFlow.getXml());
spider.executeNode(context.getThreadPool(),null,root,context,variables);
spider.executeNode(null,root,context,variables);
}else{
context.debug("执行子流程:{}", flowId);
}
}
@Override

235
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Ast.java
View File

@ -61,12 +61,13 @@ public abstract class Ast {
CharacterStream stream = new CharacterStream(unescapedValue);
while (stream.hasMore()) {
if (stream.match("\\{", true))
if (stream.match("\\{", true)) {
builder.append('{');
else if (stream.match("\\}", true))
} else if (stream.match("\\}", true)) {
builder.append('}');
else
} else {
builder.append(stream.consume());
}
}
content = builder.toString();
}
@ -97,9 +98,15 @@ public abstract class Ast {
Not, Negate, Positive;
public static UnaryOperator getOperator (Token op) {
if (op.getType() == TokenType.Not) return UnaryOperator.Not;
if (op.getType() == TokenType.Plus) return UnaryOperator.Positive;
if (op.getType() == TokenType.Minus) return UnaryOperator.Negate;
if (op.getType() == TokenType.Not) {
return UnaryOperator.Not;
}
if (op.getType() == TokenType.Plus) {
return UnaryOperator.Positive;
}
if (op.getType() == TokenType.Minus) {
return UnaryOperator.Negate;
}
ExpressionError.error("Unknown unary operator " + op + ".", op.getSpan());
return null; // not reached
}
@ -127,24 +134,26 @@ public abstract class Ast {
Object operand = getOperand().evaluate(template, context);
if (getOperator() == UnaryOperator.Negate) {
if (operand instanceof Integer)
if (operand instanceof Integer) {
return -(Integer)operand;
else if (operand instanceof Float)
} else if (operand instanceof Float) {
return -(Float)operand;
else if (operand instanceof Double)
} else if (operand instanceof Double) {
return -(Double)operand;
else if (operand instanceof Byte)
} else if (operand instanceof Byte) {
return -(Byte)operand;
else if (operand instanceof Short)
} else if (operand instanceof Short) {
return -(Short)operand;
else if (operand instanceof Long)
} else if (operand instanceof Long) {
return -(Long)operand;
else {
} else {
ExpressionError.error("Operand of operator '" + getOperator().name() + "' must be a number, got " + operand, getSpan());
return null; // never reached
}
} else if (getOperator() == UnaryOperator.Not) {
if (!(operand instanceof Boolean)) ExpressionError.error("Operand of operator '" + getOperator().name() + "' must be a boolean", getSpan());
if (!(operand instanceof Boolean)) {
ExpressionError.error("Operand of operator '" + getOperator().name() + "' must be a boolean", getSpan());
}
return !(Boolean)operand;
} else {
return operand;
@ -160,21 +169,51 @@ public abstract class Ast {
Addition, Subtraction, Multiplication, Division, Modulo, Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual, And, Or, Xor, Assignment;
public static BinaryOperator getOperator (Token op) {
if (op.getType() == TokenType.Plus) return BinaryOperator.Addition;
if (op.getType() == TokenType.Minus) return BinaryOperator.Subtraction;
if (op.getType() == TokenType.Asterisk) return BinaryOperator.Multiplication;
if (op.getType() == TokenType.ForwardSlash) return BinaryOperator.Division;
if (op.getType() == TokenType.Percentage) return BinaryOperator.Modulo;
if (op.getType() == TokenType.Equal) return BinaryOperator.Equal;
if (op.getType() == TokenType.NotEqual) return BinaryOperator.NotEqual;
if (op.getType() == TokenType.Less) return BinaryOperator.Less;
if (op.getType() == TokenType.LessEqual) return BinaryOperator.LessEqual;
if (op.getType() == TokenType.Greater) return BinaryOperator.Greater;
if (op.getType() == TokenType.GreaterEqual) return BinaryOperator.GreaterEqual;
if (op.getType() == TokenType.And) return BinaryOperator.And;
if (op.getType() == TokenType.Or) return BinaryOperator.Or;
if (op.getType() == TokenType.Xor) return BinaryOperator.Xor;
if (op.getType() == TokenType.Assignment) return BinaryOperator.Assignment;
if (op.getType() == TokenType.Plus) {
return BinaryOperator.Addition;
}
if (op.getType() == TokenType.Minus) {
return BinaryOperator.Subtraction;
}
if (op.getType() == TokenType.Asterisk) {
return BinaryOperator.Multiplication;
}
if (op.getType() == TokenType.ForwardSlash) {
return BinaryOperator.Division;
}
if (op.getType() == TokenType.Percentage) {
return BinaryOperator.Modulo;
}
if (op.getType() == TokenType.Equal) {
return BinaryOperator.Equal;
}
if (op.getType() == TokenType.NotEqual) {
return BinaryOperator.NotEqual;
}
if (op.getType() == TokenType.Less) {
return BinaryOperator.Less;
}
if (op.getType() == TokenType.LessEqual) {
return BinaryOperator.LessEqual;
}
if (op.getType() == TokenType.Greater) {
return BinaryOperator.Greater;
}
if (op.getType() == TokenType.GreaterEqual) {
return BinaryOperator.GreaterEqual;
}
if (op.getType() == TokenType.And) {
return BinaryOperator.And;
}
if (op.getType() == TokenType.Or) {
return BinaryOperator.Or;
}
if (op.getType() == TokenType.Xor) {
return BinaryOperator.Xor;
}
if (op.getType() == TokenType.Assignment) {
return BinaryOperator.Assignment;
}
ExpressionError.error("Unknown binary operator " + op + ".", op.getSpan());
return null; // not reached
}
@ -204,13 +243,27 @@ public abstract class Ast {
}
private Object evaluateAddition (Object left, Object right) {
if (left instanceof String || right instanceof String) return left.toString() + right.toString();
if (left instanceof Double || right instanceof Double) return ((Number)left).doubleValue() + ((Number)right).doubleValue();
if (left instanceof Float || right instanceof Float) return ((Number)left).floatValue() + ((Number)right).floatValue();
if (left instanceof Long || right instanceof Long) return ((Number)left).longValue() + ((Number)right).longValue();
if (left instanceof Integer || right instanceof Integer) return ((Number)left).intValue() + ((Number)right).intValue();
if (left instanceof Short || right instanceof Short) return ((Number)left).shortValue() + ((Number)right).shortValue();
if (left instanceof Byte || right instanceof Byte) return ((Number)left).byteValue() + ((Number)right).byteValue();
if (left instanceof String || right instanceof String) {
return left.toString() + right.toString();
}
if (left instanceof Double || right instanceof Double) {
return ((Number)left).doubleValue() + ((Number)right).doubleValue();
}
if (left instanceof Float || right instanceof Float) {
return ((Number)left).floatValue() + ((Number)right).floatValue();
}
if (left instanceof Long || right instanceof Long) {
return ((Number)left).longValue() + ((Number)right).longValue();
}
if (left instanceof Integer || right instanceof Integer) {
return ((Number)left).intValue() + ((Number)right).intValue();
}
if (left instanceof Short || right instanceof Short) {
return ((Number)left).shortValue() + ((Number)right).shortValue();
}
if (left instanceof Byte || right instanceof Byte) {
return ((Number)left).byteValue() + ((Number)right).byteValue();
}
ExpressionError.error("Operands for addition operator must be numbers or strings, got " + left + ", " + right + ".", getSpan());
return null; // never reached
@ -369,30 +422,50 @@ public abstract class Ast {
}
private Object evaluateAnd (Object left, ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
if (!(left instanceof Boolean)) ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
if (!(Boolean)left) return false;
if (!(left instanceof Boolean)) {
ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
}
if (!(Boolean)left) {
return false;
}
Object right = getRightOperand().evaluate(template, context);
if (!(right instanceof Boolean)) ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
if (!(right instanceof Boolean)) {
ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
}
return (Boolean)left && (Boolean)right;
}
private Object evaluateOr (Object left, ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
if (!(left instanceof Boolean)) ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
if ((Boolean)left) return true;
if (!(left instanceof Boolean)) {
ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
}
if ((Boolean)left) {
return true;
}
Object right = getRightOperand().evaluate(template, context);
if (!(right instanceof Boolean)) ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
if (!(right instanceof Boolean)) {
ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
}
return (Boolean)left || (Boolean)right;
}
private Object evaluateXor (Object left, Object right) {
if (!(left instanceof Boolean)) ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
if (!(right instanceof Boolean)) ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
if (!(left instanceof Boolean)) {
ExpressionError.error("Left operand must be a boolean, got " + left + ".", getLeftOperand().getSpan());
}
if (!(right instanceof Boolean)) {
ExpressionError.error("Right operand must be a boolean, got " + right + ".", getRightOperand().getSpan());
}
return (Boolean)left ^ (Boolean)right;
}
private Object evaluateEqual (Object left, Object right) {
if (left != null) return left.equals(right);
if (right != null) return right.equals(left);
if (left != null) {
return left.equals(right);
}
if (right != null) {
return right.equals(left);
}
return true;
}
@ -403,7 +476,9 @@ public abstract class Ast {
@Override
public Object evaluate (ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
if (getOperator() == BinaryOperator.Assignment) {
if (!(getLeftOperand() instanceof VariableAccess)) ExpressionError.error("Can only assign to top-level variables in context.", getLeftOperand().getSpan());
if (!(getLeftOperand() instanceof VariableAccess)) {
ExpressionError.error("Can only assign to top-level variables in context.", getLeftOperand().getSpan());
}
Object value = getRightOperand().evaluate(template, context);
context.set(((VariableAccess)getLeftOperand()).getVariableName().getText(), value);
return null;
@ -476,7 +551,9 @@ public abstract class Ast {
@Override
public Object evaluate (ExpressionTemplate template, ExpressionTemplateContext context) throws IOException {
Object condition = getCondition().evaluate(template, context);
if (!(condition instanceof Boolean)) ExpressionError.error("Condition of ternary operator must be a boolean, got " + condition + ".", getSpan());
if (!(condition instanceof Boolean)) {
ExpressionError.error("Condition of ternary operator must be a boolean, got " + condition + ".", getSpan());
}
return ((Boolean)condition) ? getTrueExpression().evaluate(template, context) : getFalseExpression().evaluate(template, context);
}
}
@ -538,7 +615,9 @@ public abstract class Ast {
public FloatLiteral (Span literal) {
super(literal);
String text = literal.getText();
if (text.charAt(text.length() - 1) == 'f') text = text.substring(0, text.length() - 1);
if (text.charAt(text.length() - 1) == 'f') {
text = text.substring(0, text.length() - 1);
}
this.value = Float.parseFloat(text);
}
@ -637,17 +716,17 @@ public abstract class Ast {
String text = literal.getText();
if (text.length() > 3) {
if (text.charAt(2) == 'n')
if (text.charAt(2) == 'n') {
value = '\n';
else if (text.charAt(2) == 'r')
} else if (text.charAt(2) == 'r') {
value = '\r';
else if (text.charAt(2) == 't')
} else if (text.charAt(2) == 't') {
value = '\t';
else if (text.charAt(2) == '\\')
} else if (text.charAt(2) == '\\') {
value = '\\';
else if (text.charAt(2) == '\'')
} else if (text.charAt(2) == '\'') {
value = '\'';
else {
} else {
ExpressionError.error("Unknown escape sequence '" + literal.getText() + "'.", literal);
value = 0; // never reached
}
@ -678,18 +757,19 @@ public abstract class Ast {
CharacterStream stream = new CharacterStream(unescapedValue);
while (stream.hasMore()) {
if (stream.match("\\\\", true))
if (stream.match("\\\\", true)) {
builder.append('\\');
else if (stream.match("\\n", true))
} else if (stream.match("\\n", true)) {
builder.append('\n');
else if (stream.match("\\r", true))
} else if (stream.match("\\r", true)) {
builder.append('\r');
else if (stream.match("\\t", true))
} else if (stream.match("\\t", true)) {
builder.append('\t');
else if (stream.match("\\\"", true))
} else if (stream.match("\\\"", true)) {
builder.append('"');
else
} else {
builder.append(stream.consume());
}
}
value = builder.toString();
}
@ -772,26 +852,27 @@ public abstract class Ast {
ExpressionError.error("Array index must be an integer, but was " + keyOrIndex.getClass().getSimpleName(), getKeyOrIndex().getSpan());
}
int index = ((Number)keyOrIndex).intValue();
if (mapOrArray instanceof int[])
if (mapOrArray instanceof int[]) {
return ((int[])mapOrArray)[index];
else if (mapOrArray instanceof float[])
} else if (mapOrArray instanceof float[]) {
return ((float[])mapOrArray)[index];
else if (mapOrArray instanceof double[])
} else if (mapOrArray instanceof double[]) {
return ((double[])mapOrArray)[index];
else if (mapOrArray instanceof boolean[])
} else if (mapOrArray instanceof boolean[]) {
return ((boolean[])mapOrArray)[index];
else if (mapOrArray instanceof char[])
} else if (mapOrArray instanceof char[]) {
return ((char[])mapOrArray)[index];
else if (mapOrArray instanceof short[])
} else if (mapOrArray instanceof short[]) {
return ((short[])mapOrArray)[index];
else if (mapOrArray instanceof long[])
} else if (mapOrArray instanceof long[]) {
return ((long[])mapOrArray)[index];
else if (mapOrArray instanceof byte[])
} else if (mapOrArray instanceof byte[]) {
return ((byte[])mapOrArray)[index];
else if (mapOrArray instanceof String)
return "" + ((String)mapOrArray).charAt(index);
else
} else if (mapOrArray instanceof String) {
return Character.toString(((String)mapOrArray).charAt(index));
} else {
return ((Object[])mapOrArray)[index];
}
}
}
}
@ -956,8 +1037,9 @@ public abstract class Ast {
/** Must be invoked when this node is done evaluating so we don't leak memory **/
public void clearCachedArguments () {
Object[] args = getCachedArguments();
for (int i = 0; i < args.length; i++)
for (int i = 0; i < args.length; i++) {
args[i] = null;
}
}
@Override
@ -993,7 +1075,9 @@ public abstract class Ast {
}
}
method = Reflection.getInstance().getMethod(function, null, argumentValues);
if (method == null) ExpressionError.error("Couldn't find function.", getSpan());
if (method == null) {
ExpressionError.error("Couldn't find function.", getSpan());
}
setCachedFunction(method);
try {
return Reflection.getInstance().callMethod(function, method, argumentValues);
@ -1067,8 +1151,9 @@ public abstract class Ast {
/** Must be invoked when this node is done evaluating so we don't leak memory **/
public void clearCachedArguments () {
Object[] args = getCachedArguments();
for (int i = 0; i < args.length; i++)
for (int i = 0; i < args.length; i++) {
args[i] = null;
}
}
@Override

63
spider-flow-core/src/main/java/org/spiderflow/core/utils/ExtractUtils.java
View File

@ -1,5 +1,6 @@
package org.spiderflow.core.utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -8,12 +9,15 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import com.alibaba.fastjson.JSONPath;
import us.codecraft.xsoup.Xsoup;
/**
* 抽取数据工具类
* @author jmxd
@ -90,7 +94,7 @@ public class ExtractUtils {
element = getFirstElement(element,selector);
return element == null ? null : element.html();
}
public static String getFirstOuterHTMLBySelector(Element element,String selector){
element = getFirstElement(element,selector);
return element == null ? null : element.outerHtml();
@ -155,55 +159,26 @@ public class ExtractUtils {
}
public static List<String> getValuesByXPath(Element element,String xpath){
JXDocument jXdocument = JXDocument.create(new Elements(element));
List<JXNode> nodes = jXdocument.selN(xpath);
if(nodes != null){
List<String> result = new ArrayList<>();
for (JXNode node : nodes) {
result.add(node.asString());
}
return result;
}
return Collections.emptyList();
return Xsoup.select(element,xpath).list();
}
public static List<String> getValuesByXPath(Elements elements,String xpath){
return Xsoup.select(elements.html(),xpath).list();
}
public static String getValueByXPath(Element element,String xpath){
JXDocument jXdocument = JXDocument.create(new Elements(element));
JXNode node = jXdocument.selNOne(xpath);
if(node != null){
return node.asString();
}
return null;
return Xsoup.select(element,xpath).get();
}
public static String getValueByXPath(Elements elements,String xpath){
return Xsoup.select(elements.html(),xpath).get();
}
public static String getElementByXPath(Element element,String xpath){
JXDocument jXdocument = JXDocument.create(new Elements(element));
JXNode node = jXdocument.selNOne(xpath);
if(node != null){
return node.asString();
}
return null;
return Xsoup.select(element,xpath).get();
}
public static Object getObjectValueByXPath(Element element,String xpath){
return getObjectValueByXPath(new Elements(element),xpath);
}
public static List<Object> getObjectValuesByXPath(Element element,String xpath){
return getObjectValuesByXPath(new Elements(element),xpath);
}
public static Object getObjectValueByXPath(Elements elements,String xpath){
JXDocument jXdocument = JXDocument.create(elements);
return jXdocument.selOne(xpath);
}
public static List<Object> getObjectValuesByXPath(Elements elements,String xpath){
JXDocument jXdocument = JXDocument.create(elements);
return jXdocument.sel(xpath);
}
public static boolean isNumber(String str) {
public static boolean isNumber(String str) {
return compile("^(\\-|\\+)?\\d+(\\.\\d+)?$").matcher(str).matches();
}

11
spider-flow-core/src/main/java/org/spiderflow/core/utils/SpiderFlowUtils.java
View File

@ -10,6 +10,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.spiderflow.core.executor.shape.LoopJoinExecutor;
import org.spiderflow.model.SpiderNode;
import org.springframework.util.CollectionUtils;
@ -43,10 +44,14 @@ public class SpiderFlowUtils {
node.setNodeId(nodeId);
nodeMap.put(nodeId, node);
if(element.hasAttr("edge")){ //判断是否是连线
edgeMap.put(nodeId, Collections.singletonMap(element.attr("source"),element.attr("target")));
}else if(jsonProperty != null && node.getStringJsonValue("shape") != null){
if("start".equals(node.getStringJsonValue("shape"))){
edgeMap.put(nodeId, Collections.singletonMap(element.attr("source"), element.attr("target")));
} else if (jsonProperty != null && node.getStringJsonValue("shape") != null) {
if ("start".equals(node.getStringJsonValue("shape"))) {
root = node;
} else if ("loopJoin".equals(node.getStringJsonValue("shape"))) {
String joinNodeId = node.getStringJsonValue(LoopJoinExecutor.JOIN_NODE_ID);
node.setSync(true);
nodeMap.get(joinNodeId).setSync(true);
}
}
if("0".equals(nodeId)){

4
spider-flow-web/src/main/resources/application.properties
View File

@ -28,7 +28,7 @@ spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.mongo.M
#selenium 插件配置
#chrome驱动位置
#设置chrome的WebDriver驱动路径，下载地址：http://chromedriver.storage.googleapis.com/index.html，注意版本问题
selenium.driver.chrome=E:/driver/chromedriver.exe
#firefox驱动位置
#设置fireFox的WebDriver驱动路径，下载地址：https://github.com/mozilla/geckodriver/releases
selenium.driver.firefox=E:/driver/geckodriver.exe

3
spider-flow-web/src/main/resources/static/css/editor.css
View File

@ -224,4 +224,7 @@ html,body{
.hint-grammer .hint-example{
padding:2px 5px;
color : #000;
}
.layui-layer-tips{
word-break: break-all;
}

19
spider-flow-web/src/main/resources/static/js/editor.js
View File

@ -478,14 +478,18 @@ $(function(){
name : 'process',
image : 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACMklEQVRYR+2WwXHTQBSGv+fMkNwIB7BvMRVgV0DoIFRAUoGlCpJUsKICnApCB4QKcCrAvslwwEebGfyYp2g1QgmytBmPOaCjtLv/t+/9+neFHT+yY31aAxw6PdwXzgVOgH5lA1OEcTqSy6Ybaw3QdZqIMKoTUOX9PJaoCUQIwEKEp7+U4fdYJmWR504He8IXYJpG8tK+9RJVP0ZhIkKSjuTKv9sOgDJLY8naUwYog6yUN4tYFq0Bek4vEM5ry6tcprFclMeYd57AcQcShCOrxjySYWsAWzTzgZlQOPoDRJkB46p4FeQAJtlc5SwIoIm56sa8cHrSEa4zTzx2sZD51o4D4YfN3QlAz2kf4evOAIoWKLetKxCShD2np2ksY9txXn7Lin6QCdsmoYkjfFgqz/Y7vBYlMXFVbuexDB6sQNdphPBOYOBNlkaSje06bZyEXrxqVBNfwfG9IMrL+6ksHASQJ2EVwIQFEt+OeybsOrWsfoUyW0P0E26MsryDtknoIXwFq9UoWmBlF8GZ+BIGVeHyxLZJWDZhHUC2+7Xy9lssH0MCJmROUQF/aplb63YfIlI3598B8AbcWQsKE8J0qQx9G3pO7Xg99WUMScJGLchD5u43hKkK0WrN5/zUGqeRnBV3gW3dCbPdwU0OUQXPINokYRPDPhjF9t8qWC5YNe4e5cpa0QigdCfcBLHxNMx+z1zcFmubhI8HqJiw8EHgnfCvSbiJdFvfN7ZgW8J+3f8AvwFIqFFyobd5IgAAAABJRU5ErkJggg==',
title : '子流程'
}]
}];
var addShape = function(shape){
var image = new Image();
image.src = shape.image;
image.title = shape.title;
if(shape.hidden){
}else{
image.id = shape.name;
image.onclick = function (ev) {
if(shape.desc){
layer.tips("(" + shape.name + ")" + shape.title + "<hr/>" + shape.desc, '#' + shape.name);
}
}
if(!shape.hidden){
container.appendChild(image);
}
@ -838,12 +842,7 @@ function onCanvasViewerClick(e,source){
}
function createWebSocket(options){
options = options || {};
var socket;
if(location.host === 'demo.spiderflow.org'){
socket = new WebSocket(options.url || 'ws://49.233.182.130:8088/ws');
}else{
socket = new WebSocket(options.url || (location.origin.replace("http",'ws') + '/ws'));
}
var socket = new WebSocket(options.url || (location.origin.replace("http",'ws') + '/ws'));
socket.onopen = options.onopen;
socket.onmessage = options.onmessage;
socket.onerror = options.onerror || function(){

18
spider-flow-web/src/main/resources/static/js/mxgraph/css/explorer.css Normal file
View File

@ -0,0 +1,18 @@
div.mxTooltip {
filter:progid:DXImageTransform.Microsoft.DropShadow(OffX=4, OffY=4,
Color='#A2A2A2', Positive='true');
}
div.mxPopupMenu {
filter:progid:DXImageTransform.Microsoft.DropShadow(OffX=4, OffY=4,
Color='#C0C0C0', Positive='true');
}
div.mxWindow {
_filter:progid:DXImageTransform.Microsoft.DropShadow(OffX=4, OffY=4,
Color='#C0C0C0', Positive='true');
}
td.mxWindowTitle {
_height: 23px;
}
.mxDisabled {
filter:alpha(opacity=20) !important;
}

1
spider-flow-web/src/main/resources/static/resources/templates/function.html
View File

@ -36,7 +36,6 @@
<button class="layui-btn function-add" type="button">添加一个函数</button>
</div>
</div>
<hr>
</form>
</div>
</div>

2
spider-flow-web/src/main/resources/static/resources/templates/root.html
View File

@ -15,7 +15,7 @@
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">线程数</label>
<label class="layui-form-label">最大线程数</label>
<div class="layui-input-block">
<input type="text" name="threadCount" placeholder="请输入线程数" autocomplete="off" class="layui-input" value="{{=d.data.object.threadCount}}">
</div>