增加循环功能

2019-07-20 15:23:40 +08:00 · 2019-07-20 15:23:40 +08:00 · 238cd2f0f1
commit 238cd2f0f1
parent cf3cd44cc4
18 changed files with 402 additions and 76 deletions
--- a/db/db.mv.db
+++ b/db/db.mv.db
--- a/db/db.trace.db
+++ b/db/db.trace.db
@ -882,3 +882,213 @@ ALTER TABLE SP_FLOW ADD COLUMN EXECUTE_COUNT INT(9) NULL "; SQL statement:
 select * from sp_flow
 alter table sp_flow drop column execute_count 
 alter table sp_flow add column execute_count int(9) null [42000-197]
+2019-07-20 14:15:52 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
+2019-07-20 14:15:52 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
+2019-07-20 14:15:53 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
+2019-07-20 14:15:54 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
+2019-07-20 14:15:54 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
+2019-07-20 14:15:55 database: flush
+org.h2.message.DbException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.get(DbException.java:168)
+	at org.h2.message.DbException.convert(DbException.java:307)
+	at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:95)
+	at org.h2.mvstore.MVStore.handleException(MVStore.java:2506)
+	at org.h2.mvstore.MVStore.panic(MVStore.java:383)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:364)
+	at org.h2.mvstore.MVStore$Builder.open(MVStore.java:2934)
+	at org.h2.mvstore.db.MVTableEngine$Store.open(MVTableEngine.java:155)
+	at org.h2.mvstore.db.MVTableEngine.init(MVTableEngine.java:100)
+	at org.h2.engine.Database.getPageStore(Database.java:2538)
+	at org.h2.engine.Database.open(Database.java:709)
+	at org.h2.engine.Database.openDatabase(Database.java:286)
+	at org.h2.engine.Database.<init>(Database.java:280)
+	at org.h2.engine.Engine.openSession(Engine.java:66)
+	at org.h2.engine.Engine.openSession(Engine.java:179)
+	at org.h2.engine.Engine.createSessionAndValidate(Engine.java:157)
+	at org.h2.engine.Engine.createSession(Engine.java:140)
+	at org.h2.engine.Engine.createSession(Engine.java:28)
+	at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:351)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:124)
+	at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:103)
+	at org.h2.Driver.connect(Driver.java:69)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1596)
+	at com.alibaba.druid.pool.DruidAbstractDataSource.createPhysicalConnection(DruidAbstractDataSource.java:1662)
+	at com.alibaba.druid.pool.DruidDataSource$CreateConnectionThread.run(DruidDataSource.java:2601)
+Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]" [50000-197]
+	at org.h2.message.DbException.getJdbcSQLException(DbException.java:357)
+	... 25 more
+Caused by: java.lang.IllegalStateException: The file is locked: nio:C:/Users/Administrator/git/spider-flow/db/db.mv.db [1.4.197/7]
+	at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:870)
+	at org.h2.mvstore.FileStore.open(FileStore.java:173)
+	at org.h2.mvstore.MVStore.<init>(MVStore.java:350)
+	... 19 more
--- a/pom.xml
+++ b/pom.xml
@ -56,10 +56,9 @@
 		    <artifactId>spring-boot-starter-data-jpa</artifactId>
 		</dependency>
 		<dependency>
-		    <groupId>org.h2database</groupId>
+		    <groupId>com.h2database</groupId>
 		    <artifactId>h2</artifactId>
 		    <scope>runtime</scope>
-		    <version>1.4.197</version>
 		</dependency>
 		<dependency>
 			<groupId>org.freemarker</groupId>
--- a/src/main/java/com/mxd/spider/core/Spider.java
+++ b/src/main/java/com/mxd/spider/core/Spider.java
@ -1,9 +1,15 @@
 package com.mxd.spider.core;

+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
-import java.util.concurrent.CountDownLatch;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;

 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@ -30,55 +36,86 @@ public class Spider {
 	
 	public List<SpiderOutput> runWithTest(SpiderNode root,SpiderContext context){
 		//开始不允许设置任何东西
-		ExecutorService threadPool = Executors.newCachedThreadPool();
-		execute(threadPool,root, context);
+		int threadPoolSize = 8;
+		ThreadPoolExecutor pool = new ThreadPoolExecutor(threadPoolSize,threadPoolSize,5000,TimeUnit.MILLISECONDS,new LinkedBlockingQueue<Runnable>());
+		execute(pool,root, context,new HashMap<>());
 		context.log("测试完毕！");
 		return context.getOutputs();
 	}
 	
-	private void execute(ExecutorService threadPool,SpiderNode node,SpiderContext context){
+	private void execute(ExecutorService threadPool,SpiderNode node,SpiderContext context,Map<String,Object> variables){
 		if(logger.isDebugEnabled()){
 			logger.debug("执行节点[{}:{}]",node.getNodeName(),node.getNodeId());
 		}
 		context.log(String.format("执行节点[%s:%s]", node.getNodeName(),node.getNodeId()));
+		if(executeCondition(node,context,variables)){
+			executeNode(threadPool,node,context,variables);
+			if(logger.isDebugEnabled()){
+				logger.debug("执行节点[{}:{}]完毕",node.getNodeName(),node.getNodeId());
+			}
+			context.log(String.format("执行节点[%s:%s]完毕", node.getNodeName(),node.getNodeId()));
+		}
+	}
+	
+	private void executeaNextNodes(ExecutorService threadPool,SpiderNode node,SpiderContext context,Map<String,Object> variables){
+		List<SpiderNode> nextNodes = node.getNextNodes();
+		if(nextNodes != null){
+			for (SpiderNode nextNode : nextNodes) {
+				execute(threadPool,nextNode,context,variables);
+			}
+		}
+	}
+	
+	private void executeNode(ExecutorService pool,SpiderNode node,SpiderContext context,Map<String,Object> variables){
+		int loopCount = 1;
+		if(node.getLoopCount() != null){
+			Object result = engine.execute(node.getLoopCount(), variables);
+			if(result != null){
+				if(logger.isDebugEnabled()){
+					logger.debug("获取循环次数{}={}",node.getLoopCount(),result);
+				}
+				context.log(String.format("获取循环次数%s=%s",node.getLoopCount(),result));
+				loopCount = ((Long)result).intValue();
+			}
+		}
+		List<Future<Map<String,Object>>> futures = new ArrayList<>();
+		if(loopCount > 0){
+			for (Executor executor : executors) {
+				if(executor.supportShape().equals(node.getShape())){
+					for (int i = 0; i < loopCount; i++) {
+						//存入下标变量
+						Map<String, Object> nVariables = Maps.add(variables, node.getLoopVariableName(), i);
+						futures.add(pool.submit(()->{
+							executor.execute(node, context,nVariables);
+							return nVariables;
+						}));
+					}
+				}
+			}
+		}
+		//递归执行下一级
+		for (Future<Map<String,Object>> future : futures) {
+			try {
+				executeaNextNodes(pool, node, context, future.get());
+			} catch (InterruptedException | ExecutionException e) {
+				
+			}
+			
+		}
+	}
+	
+	private boolean executeCondition(SpiderNode node,SpiderContext context,Map<String,Object> variables){
 		if(node.getCondition() != null){	//判断是否有条件
-			Object result = engine.execute(node.getCondition(), Maps.add(context, "resp", node.getLastResponse()));
+			Object result = engine.execute(node.getCondition(), variables);
 			if(result != null){
 				boolean isContinue = "true".equals(result);
 				if(logger.isDebugEnabled()){
 					logger.debug("判断{}={}",node.getCondition(),isContinue);
 				}
 				context.log(String.format("判断%s=%s",node.getCondition(),isContinue));
-				if(!isContinue){
-					return;
-				}
+				return isContinue;
 			}
 		}
-		CountDownLatch latch = new CountDownLatch(this.executors.size());
-		for (Executor executor : executors) {
-			threadPool.submit(()->{
-					try {
-						if(executor.supportShape().equals(node.getShape())){
-							executor.execute(node, context);
-						}
-					} finally{
-						latch.countDown();
-					}
-				
-			});
-		}
-		try {
-			latch.await();
-		} catch (InterruptedException e) {
-		}
-		if(logger.isDebugEnabled()){
-			logger.debug("执行节点[{}:{}]完毕",node.getNodeName(),node.getNodeId());
-		}
-		context.log(String.format("执行节点[%s:%s]完毕", node.getNodeName(),node.getNodeId()));
-		//递归执行下一级
-		List<SpiderNode> nodes = node.getNextNodes();
-		for (SpiderNode nextNode : nodes) {
-			execute(threadPool,nextNode,context);
-		}
+		return true;
 	}
 }
--- a/src/main/java/com/mxd/spider/core/executor/DataSourceExecutor.java
+++ b/src/main/java/com/mxd/spider/core/executor/DataSourceExecutor.java
@ -1,5 +1,7 @@
 package com.mxd.spider.core.executor;

+import java.util.Map;
+
 import javax.sql.DataSource;

 import org.slf4j.Logger;
@ -17,7 +19,7 @@ public class DataSourceExecutor implements Executor{
 	private static Logger logger = LoggerFactory.getLogger(DataSourceExecutor.class);

 	@Override
-	public void execute(SpiderNode node, SpiderContext context) {
+	public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
 		SpiderJsonProperty property = node.getJsonProperty();
 		if(property != null){
 			if(property.getDatasourceType() == null){
--- a/src/main/java/com/mxd/spider/core/executor/ExecuteSQLExecutor.java
+++ b/src/main/java/com/mxd/spider/core/executor/ExecuteSQLExecutor.java
@ -17,7 +17,6 @@ import com.mxd.spider.core.freemarker.FreeMarkerEngine;
 import com.mxd.spider.core.model.SpiderJsonProperty;
 import com.mxd.spider.core.model.SpiderNode;
 import com.mxd.spider.core.utils.ExtractUtils;
-import com.mxd.spider.core.utils.Maps;

@Component
 public class ExecuteSQLExecutor implements Executor{
@ -28,7 +27,7 @@ public class ExecuteSQLExecutor implements Executor{
 	private FreeMarkerEngine engine;

 	@Override
-	public void execute(SpiderNode node, SpiderContext context) {
+	public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
 		SpiderJsonProperty property = node.getJsonProperty();
 		if(property != null){
 			if(StringUtils.isEmpty(property.getDatasourceId())){
@ -46,7 +45,6 @@ public class ExecuteSQLExecutor implements Executor{
 				//把变量替换成占位符
 				String sql = property.getSql().replaceAll("#(.*?)#", "?");
 				List<String> parameters = ExtractUtils.getMatchers(property.getSql(), "#(.*?)#", true);
-				Map<String, Object> variables = Maps.add(context, "resp", node.getLastResponse());
 				int size = parameters.size();
 				Object[] params = new Object[size];
 				for (int i = 0;i<size;i++) {
--- a/src/main/java/com/mxd/spider/core/executor/Executor.java
+++ b/src/main/java/com/mxd/spider/core/executor/Executor.java
@ -1,5 +1,7 @@
 package com.mxd.spider.core.executor;

+import java.util.Map;
+
 import com.mxd.spider.core.context.SpiderContext;
 import com.mxd.spider.core.model.SpiderNode;

@ -7,5 +9,5 @@ public interface Executor {
 	
 	public String supportShape();

-	public void execute(SpiderNode node,SpiderContext context);
+	public void execute(SpiderNode node,SpiderContext context,Map<String,Object> variables);
 }
--- a/src/main/java/com/mxd/spider/core/executor/OutputExecutor.java
+++ b/src/main/java/com/mxd/spider/core/executor/OutputExecutor.java
@ -14,7 +14,6 @@ import com.mxd.spider.core.model.SpiderJsonProperty;
 import com.mxd.spider.core.model.SpiderNameValue;
 import com.mxd.spider.core.model.SpiderNode;
 import com.mxd.spider.core.model.SpiderOutput;
-import com.mxd.spider.core.utils.Maps;

@Component
 public class OutputExecutor implements Executor{
@ -25,13 +24,12 @@ public class OutputExecutor implements Executor{
 	private static Logger logger = LoggerFactory.getLogger(OutputExecutor.class);

 	@Override
-	public void execute(SpiderNode node, SpiderContext context) {
+	public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
 		SpiderJsonProperty property = node.getJsonProperty();
 		SpiderOutput output = new SpiderOutput();
 		output.setNodeName(node.getNodeName());
 		output.setNodeId(node.getNodeId());
 		if(property != null){
-			Map<String, Object> variables = Maps.add(context, "resp", node.getLastResponse());
 			for (SpiderNameValue nameValue : property.getOutputs()) {
 				Object value = null;
 				try {
--- a/src/main/java/com/mxd/spider/core/executor/RequestExecutor.java
+++ b/src/main/java/com/mxd/spider/core/executor/RequestExecutor.java
@ -18,7 +18,6 @@ import com.mxd.spider.core.io.HttpResponse;
 import com.mxd.spider.core.model.SpiderJsonProperty;
 import com.mxd.spider.core.model.SpiderNameValue;
 import com.mxd.spider.core.model.SpiderNode;
-import com.mxd.spider.core.utils.Maps;

@Component
 public class RequestExecutor implements Executor{
@ -34,11 +33,10 @@ public class RequestExecutor implements Executor{
 	}

 	@Override
-	public void execute(SpiderNode node, SpiderContext context) {
+	public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
 		HttpRequest request = HttpRequest.create();
 		SpiderJsonProperty property = node.getJsonProperty();
 		List<SpiderNameValue> parameters = property.getParameters();
-		Map<String, Object> variables = Maps.add(context, "resp", node.getLastResponse());
 		//设置请求url
 		String url = null;
 		try {
@ -97,12 +95,8 @@ public class RequestExecutor implements Executor{
 		}
 		try {
 			HttpResponse response = request.execute();
-			List<SpiderNode> nodes = node.getNextNodes();
-			if(nodes != null){
-				for (SpiderNode nextNode : nodes) {
-					nextNode.setLastResponse(response);
-				}
-			}
+			//结果存入变量
+			variables.put("resp", response);
 		} catch (IOException e) {
 			logger.error("请求{}出错",url,e);
 			context.log(String.format("请求%s出错,异常信息:%s", url,ExceptionUtils.getStackTrace(e)));
--- a/src/main/java/com/mxd/spider/core/executor/VariableExecutor.java
+++ b/src/main/java/com/mxd/spider/core/executor/VariableExecutor.java
@ -13,7 +13,6 @@ import com.mxd.spider.core.freemarker.FreeMarkerEngine;
 import com.mxd.spider.core.model.SpiderJsonProperty;
 import com.mxd.spider.core.model.SpiderNameValue;
 import com.mxd.spider.core.model.SpiderNode;
-import com.mxd.spider.core.utils.Maps;

@Component
 public class VariableExecutor implements Executor{
@ -24,10 +23,9 @@ public class VariableExecutor implements Executor{
 	private FreeMarkerEngine engine;

 	@Override
-	public void execute(SpiderNode node, SpiderContext context) {
+	public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
 		SpiderJsonProperty property = node.getJsonProperty();
 		if(property != null){
-			Map<String, Object> variables = Maps.add(context, "resp", node.getLastResponse());
 			for (SpiderNameValue nameValue : property.getVariables()) {
 				Object value = null;
 				try {
@ -41,7 +39,7 @@ public class VariableExecutor implements Executor{
 					context.log(String.format("设置变量%s出错,异常信息：%s", nameValue.getName(),ExceptionUtils.getStackTrace(e)));
 					ExceptionUtils.wrapAndThrow(e);
 				}
-				context.put(nameValue.getName(), value);
+				variables.put(nameValue.getName(), value);
 			}
 		}
 	}
--- a/src/main/java/com/mxd/spider/core/model/SpiderJsonProperty.java
+++ b/src/main/java/com/mxd/spider/core/model/SpiderJsonProperty.java
@ -11,7 +11,20 @@ public class SpiderJsonProperty{
 	
 	private List<SpiderNameValue> outputs;
 	
+	/**
+	 * 条件判断表达式
+	 */
 	private String condition;
+
+	/**
+	 * 循环次数
+	 */
+	private String loopCount;
+	
+	/**
+	 * 循环变量
+	 */
+	private String loopVariableName;
 	
 	/*爬取参数--start*/
 	
@ -168,13 +181,29 @@ public class SpiderJsonProperty{
 		this.sql = sql;
 	}

+	public String getLoopCount() {
+		return loopCount;
+	}
+
+	public void setLoopCount(String loopCount) {
+		this.loopCount = loopCount;
+	}
+	
+	public String getLoopVariableName() {
+		return loopVariableName;
+	}
+
+	public void setLoopVariableName(String loopVariableName) {
+		this.loopVariableName = loopVariableName;
+	}
+
 	@Override
 	public String toString() {
 		return "SpiderJsonProperty [shape=" + shape + ", variables=" + variables + ", outputs=" + outputs
-				+ ", condition=" + condition + ", url=" + url + ", method=" + method + ", headers=" + headers
-				+ ", parameters=" + parameters + ", datasourceType=" + datasourceType + ", datasourceUrl="
-				+ datasourceUrl + ", datasourceUsername=" + datasourceUsername + ", datasourcePassword="
-				+ datasourcePassword + ", datasourceId=" + datasourceId + ", statementType=" + statementType + ", sql="
-				+ sql + "]";
+				+ ", condition=" + condition + ", loopCount=" + loopCount + ", loopVariableName=" + loopVariableName
+				+ ", url=" + url + ", method=" + method + ", headers=" + headers + ", parameters=" + parameters
+				+ ", datasourceType=" + datasourceType + ", datasourceUrl=" + datasourceUrl + ", datasourceUsername="
+				+ datasourceUsername + ", datasourcePassword=" + datasourcePassword + ", datasourceId=" + datasourceId
+				+ ", statementType=" + statementType + ", sql=" + sql + "]";
 	}
 }
--- a/src/main/java/com/mxd/spider/core/model/SpiderNode.java
+++ b/src/main/java/com/mxd/spider/core/model/SpiderNode.java
@ -3,8 +3,6 @@ package com.mxd.spider.core.model;
 import java.util.ArrayList;
 import java.util.List;

-import com.mxd.spider.core.io.HttpResponse;
-

 public class SpiderNode {
 	
@ -14,12 +12,14 @@ public class SpiderNode {
 	
 	private String condition;
 	
-	private HttpResponse lastResponse;
-	
 	private String nodeName;
 	
 	private String nodeId;
 	
+	private String loopCount;
+	
+	private String loopVariableName;
+	
 	public String getNodeId() {
 		return nodeId;
 	}
@ -68,18 +68,29 @@ public class SpiderNode {
 		this.condition = condition;
 	}
 	
-	public HttpResponse getLastResponse() {
-		return lastResponse;
+	public String getLoopCount() {
+		return loopCount;
 	}

-	public void setLastResponse(HttpResponse lastResponse) {
-		this.lastResponse = lastResponse;
+	public void setLoopCount(String loopCount) {
+		this.loopCount = loopCount;
+	}
+
+	public String getLoopVariableName() {
+		return loopVariableName;
+	}
+	
+	
+
+	public void setLoopVariableName(String loopVariableName) {
+		this.loopVariableName = loopVariableName;
 	}

 	@Override
 	public String toString() {
 		return "SpiderNode [jsonProperty=" + jsonProperty + ", nextNodes=" + nextNodes + ", condition=" + condition
-				+ ", lastResponse=" + lastResponse + ", nodeName=" + nodeName + ", nodeId=" + nodeId + "]";
+				+ ", nodeName=" + nodeName + ", nodeId=" + nodeId + ", loopCount=" + loopCount + ", loopVariableName="
+				+ loopVariableName + "]";
 	}
 	
 }
--- a/src/main/java/com/mxd/spider/core/utils/Maps.java
+++ b/src/main/java/com/mxd/spider/core/utils/Maps.java
@ -1,6 +1,7 @@
 package com.mxd.spider.core.utils;

 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;

 public class Maps {
@ -10,4 +11,15 @@ public class Maps {
 		destMap.put(k, v);
 		return destMap;
 	}
+	
+	public static <K,V> Map<K,V> add(Map<K,V> srcMap,List<K> ks,List<V> vs){
+		HashMap<K, V> destMap = new HashMap<>(srcMap);
+		if(ks != null && vs != null && ks.size() == vs.size()){
+			int size = ks.size();
+			for (int i = 0; i < size; i++) {
+				destMap.put(ks.get(0), vs.get(0));
+			}
+		}
+		return destMap;
+	}
 }
--- a/src/main/java/com/mxd/spider/core/utils/SpiderFlowUtils.java
+++ b/src/main/java/com/mxd/spider/core/utils/SpiderFlowUtils.java
@ -39,6 +39,10 @@ public class SpiderFlowUtils {
 				if("start".equals(node.getJsonProperty().getShape())){
 					root = node;
 				}
+				if(jsonProperty.getLoopCount() != null){
+					node.setLoopCount(jsonProperty.getLoopCount());
+					node.setLoopVariableName(jsonProperty.getLoopVariableName());
+				}
 			}
 		}
 		//处理连线
--- a/src/main/resources/static/css/editor.css
+++ b/src/main/resources/static/css/editor.css
@ -85,11 +85,11 @@ html,body{
 }

 .test-window-container .output-container{
-	height: 300px;
+	height: 320px;
    overflow: auto;
 }
 .test-window-container .log-container{
-	height : 170px;
+	height : 100px;
 }
 .test-window-container .log-container textarea{
 	width : 100%;
--- a/src/main/resources/static/js/editor.js
+++ b/src/main/resources/static/js/editor.js
@ -123,6 +123,12 @@ $(function(){
 				cell.data = cell.data || new JsonProperty();
 				cell.data.set('spiderName',$(this).val())
 			}
+		}).on("keyup",".editor-form-node .layui-form-item input[name=loopCount]",function(){
+			var cell = graph.getSelectionCell();
+			if(cell != null){
+				cell.data = cell.data || new JsonProperty();
+				cell.data.set('loopCount',$(this).val())
+			}
 		}).on("keyup",".editor-form-node .layui-form-item input[name^=variable-]",function(){	//变量操作
 			resetFormArray(graph,'variable','variables');
 		}).on("click",".editor-form-node .variable-remove",function(){	//移除单个变量
@ -135,7 +141,6 @@ $(function(){
 			$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close variable-remove"></i><label class="layui-form-label">变量名</label><div class="layui-input-block"><input type="text" name="variable-name" placeholder="变量名称" autocomplete="off" class="layui-input"></div></div><div class="layui-form-item"><label class="layui-form-label">变量值</label><div class="layui-input-block"><input type="text" name="variable-value" placeholder="请输入变量值" autocomplete="off" class="layui-input"></div></div><hr>');
 		}).on("keyup",".editor-form-node .layui-form-item input[name^=header-]",function(e){	//Header操作
 			resetFormArray(graph,'header','headers');
-			console.log('header-input');
 			e.stopPropagation();
 		}).on("click",".editor-form-node .header-remove",function(){	//移除单个Header
 			var $dom = $(this).parent();
@ -494,7 +499,10 @@ $(function(){
 									tableMap[tableId].instance = layui.table.render({
 										elem : '#' + tableId,
 										cols : tableMap[tableId].cols,
-										data : tableMap[tableId].data
+										data : tableMap[tableId].data,
+										page : true,
+										limit : 5,
+										limits : [5]
 									})
 								}
 							}else if(eventType == 'log'){
--- a/src/main/resources/static/resources/templates/output.html
+++ b/src/main/resources/static/resources/templates/output.html
@ -11,6 +11,18 @@
    				<input type="text" name="nodeName" placeholder="请输入节点名称" value="{{d.value}}" autocomplete="off" class="layui-input">
    			</div>
    		</div>
+    		<div class="layui-form-item">
+    			<label class="layui-form-label">循环变量</label>
+    			<div class="layui-input-block">
+    				<input type="text" name="loopVariableName" placeholder="请输入循环变量" autocomplete="off" class="layui-input" value="{{d.data.object.loopVariableName || ''}}">
+    			</div>
+    		</div>
+    		<div class="layui-form-item">
+    			<label class="layui-form-label">循环次数</label>
+    			<div class="layui-input-block">
+    				<input type="text" name="loopCount" placeholder="请输入循环次数" autocomplete="off" class="layui-input" value="{{d.data.object.loopCount || ''}}">
+    			</div>
+    		</div>
    		{{# layui.each(d.data.object.outputs,function(index,output){ }}
    			<hr>
 	    		<div class="layui-form-item layui-form-relative">
--- a/src/main/resources/static/resources/templates/rectangle.html
+++ b/src/main/resources/static/resources/templates/rectangle.html
@ -11,6 +11,18 @@
    				<input type="text" name="value" placeholder="请输入节点名称" value="{{d.value}}" autocomplete="off" class="layui-input">
    			</div>
    		</div>
+    		<div class="layui-form-item">
+    			<label class="layui-form-label">循环变量</label>
+    			<div class="layui-input-block">
+    				<input type="text" name="loopVariableName" placeholder="请输入循环变量" autocomplete="off" class="layui-input" value="{{d.data.object.loopVariableName || ''}}">
+    			</div>
+    		</div>
+    		<div class="layui-form-item">
+    			<label class="layui-form-label">循环次数</label>
+    			<div class="layui-input-block">
+    				<input type="text" name="loopCount" placeholder="请输入循环次数" autocomplete="off" class="layui-input" value="{{d.data.object.loopCount || ''}}">
+    			</div>
+    		</div>
    		<div class="layui-form-item">
    			<label class="layui-form-label">起始URL</label>
    			<div class="layui-input-block">