diff --git a/spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java b/spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java index f23e6ea..448de38 100644 --- a/spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java +++ b/spider-flow-api/src/main/java/org/spiderflow/model/SpiderNode.java @@ -3,6 +3,7 @@ package org.spiderflow.model; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import com.alibaba.fastjson.JSONArray; @@ -84,6 +85,11 @@ public class SpiderNode { } return value; } + + public String getStringJsonValue(String key,String defaultValue){ + String value = getStringJsonValue(key); + return StringUtils.isNotBlank(value) ? value : defaultValue; + } public List> getListJsonValue(String ... keys){ List arrays = new ArrayList<>(); diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/Spider.java b/spider-flow-core/src/main/java/org/spiderflow/core/Spider.java index c20a17d..345eb23 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/Spider.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/Spider.java @@ -9,6 +9,7 @@ import org.spiderflow.concurrent.*; import org.spiderflow.concurrent.SpiderFlowThreadPoolExecutor.SubThreadPoolExecutor; import org.spiderflow.context.SpiderContext; import org.spiderflow.context.SpiderContextHolder; +import org.spiderflow.core.executor.shape.LoopExecutor; import org.spiderflow.core.model.SpiderFlow; import org.spiderflow.core.service.FlowNoticeService; import org.spiderflow.core.utils.ExecutorsUtils; @@ -24,6 +25,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.lang.reflect.Array; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -162,7 +164,9 @@ public class Spider { } //睡眠1ms,让出cpu Thread.sleep(1); - } catch (InterruptedException | ExecutionException ignored) { + } catch (InterruptedException ignored) { + } catch (Throwable t){ + logger.error("程序发生异常",t); } } //等待线程池结束 @@ -211,15 +215,37 @@ public class Spider { logger.error("执行失败,找不到对应的执行器:{}", shape); context.setRunning(false); } - //循环次数默认为1,如果节点有循环属性且填了循环次数,则取出循环次数 - int loopCount = 1; + int loopCount = 1; //循环次数默认为1,如果节点有循环属性且填了循环次数/集合,则取出循环次数 + int loopStart = 0; //循环起始位置 + int loopEnd = 1; //循环结束位置 String loopCountStr = node.getStringJsonValue(ShapeExecutor.LOOP_COUNT); - if (StringUtils.isNotBlank(loopCountStr)) { + Object loopArray = null; + boolean isLoop = false; + if (isLoop = StringUtils.isNotBlank(loopCountStr)) { try { - Object result = ExpressionUtils.execute(loopCountStr, variables); - result = result == null ? 0 : result; - logger.info("获取循环次数{}={}", loopCountStr, result); - loopCount = Integer.parseInt(result.toString()); + loopArray = ExpressionUtils.execute(loopCountStr, variables); + if(loopArray == null){ + loopCount = 0; + }else if(loopArray instanceof Collection){ + loopCount = ((Collection)loopArray).size(); + loopArray = ((Collection)loopArray).toArray(); + }else if(loopArray.getClass().isArray()){ + loopCount = Array.getLength(loopArray); + }else{ + loopCount = NumberUtils.toInt(loopArray.toString(),0); + loopArray = null; + } + loopEnd = loopCount; + if(loopCount > 0){ + loopStart = Math.max(NumberUtils.toInt(node.getStringJsonValue(LoopExecutor.LOOP_START), 0),0); + int end = NumberUtils.toInt(node.getStringJsonValue(LoopExecutor.LOOP_END), -1); + if(end >=0){ + loopEnd = Math.min(end,loopEnd); + }else{ + loopEnd = Math.max(loopEnd + end + 1,0); + } + } + logger.info("获取循环次数{}={}", loopCountStr, loopCount); } catch (Throwable t) { loopCount = 0; logger.error("获取循环次数失败,异常信息：{}", t); @@ -228,8 +254,9 @@ public class Spider { if (loopCount > 0) { //获取循环下标的变量名称 String loopVariableName = node.getStringJsonValue(ShapeExecutor.LOOP_VARIABLE_NAME); + String loopItem = node.getStringJsonValue(LoopExecutor.LOOP_ITEM,"item"); List tasks = new ArrayList<>(); - for (int i = 0; i < loopCount; i++) { + for (int i = loopStart; i < loopEnd; i++) { node.increment(); //节点执行次数+1(后续Join节点使用) if (context.isRunning()) { Map nVariables = new HashMap<>(); @@ -237,9 +264,13 @@ public class Spider { if(fromNode == null || node.isTransmitVariable(fromNode.getNodeId())){ nVariables.putAll(variables); } - // 存入下标变量 - if (!StringUtils.isBlank(loopVariableName)) { - nVariables.put(loopVariableName, i); + if(isLoop){ + // 存入下标变量 + if (!StringUtils.isBlank(loopVariableName)) { + nVariables.put(loopVariableName, i); + } + // 存入item + nVariables.put(loopItem,loopArray == null ? i : Array.get(loopArray, i)); } tasks.add(new SpiderTask(TtlRunnable.get(() -> { if (context.isRunning()) { diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopExecutor.java b/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopExecutor.java index 6dbaabe..a3657dd 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopExecutor.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/LoopExecutor.java @@ -15,9 +15,11 @@ import org.springframework.stereotype.Component; @Component public class LoopExecutor implements ShapeExecutor{ - public static final String LOOP_NODE_KEY = "__loop_node_"; + public static final String LOOP_ITEM = "loopItem"; - public static final String BEFORE_LOOP_VARIABLE = "__loop_before_variable_"; + public static final String LOOP_START = "loopStart"; + + public static final String LOOP_END = "loopEnd"; @Override public void execute(SpiderNode node, SpiderContext context, Map variables) { diff --git a/spider-flow-web/src/main/resources/static/resources/templates/loop.html b/spider-flow-web/src/main/resources/static/resources/templates/loop.html index 54e4522..7ac8e0c 100644 --- a/spider-flow-web/src/main/resources/static/resources/templates/loop.html +++ b/spider-flow-web/src/main/resources/static/resources/templates/loop.html @@ -5,28 +5,52 @@
-
- -
- -
-
-
- -
- -
-
- -
- -
-
+
+
+
+ +
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+ +
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+