Compare commits

..

1 Commits

Author SHA1 Message Date
dependabot[bot]
51ca39898c
Bump commons-io from 2.6 to 2.7
Bumps commons-io from 2.6 to 2.7.

Signed-off-by: dependabot[bot] <support@github.com>
2021-04-26 20:09:52 +00:00
6 changed files with 45 additions and 77 deletions

View File

@ -56,4 +56,4 @@
## 免责声明
请勿将`spider-flow`应用到任何可能会违反法律规定和道德约束的工作中请友善使用`spider-flow`,遵守蜘蛛协议,不要将`spider-flow`用于任何非法用途。如您选择使用`spider-flow`即代表您遵守此协议,作者不承担任何由于您违反此协议带来任何的法律风险和损失,一切后果由您承担。
请勿将`spider-flow`应用到任何可能会违反法律规定和道德约束的工作中,请友善使用`spider-flow`,遵守蜘蛛协议,不要将`spider-flow`用于任何非法用途。如您选择使用`spider-flow`即代表您遵守此协议,作者不承担任何由于您违反此协议带来任何的法律风险和损失,一切后果由您承担。

View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.spiderflow</groupId>
<artifactId>spider-flow</artifactId>
@ -16,7 +18,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<spider-flow.version>${project.version}</spider-flow.version>
<alibaba.fastjson.version>1.2.83</alibaba.fastjson.version>
<alibaba.fastjson.version>1.2.58</alibaba.fastjson.version>
<alibaba.druid.version>1.1.16</alibaba.druid.version>
<alibaba.transmittable.version>2.11.5</alibaba.transmittable.version>
<mybatis.plus.version>3.1.0</mybatis.plus.version>
@ -169,4 +171,4 @@
<module>spider-flow-core</module>
<module>spider-flow-web</module>
</modules>
</project>
</project>

View File

@ -35,13 +35,13 @@ import java.util.stream.Collectors;
*/
@Service
public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow> {
@Autowired
private SpiderFlowMapper sfMapper;
@Autowired
private SpiderJobManager spiderJobManager;
@Autowired
private FlowNoticeMapper flowNoticeMapper;
@ -73,15 +73,15 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
public IPage<SpiderFlow> selectSpiderPage(Page<SpiderFlow> page, String name){
return sfMapper.selectSpiderPage(page,name);
}
public int executeCountIncrement(String id, Date lastExecuteTime, Date nextExecuteTime){
if(nextExecuteTime == null){
return sfMapper.executeCountIncrement(id, lastExecuteTime);
}
return sfMapper.executeCountIncrementAndExecuteTime(id, lastExecuteTime, nextExecuteTime);
}
/**
* 重置定时任务
* @param id 爬虫的ID
@ -130,20 +130,13 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
}
return true;
}
public void stop(String id){
sfMapper.resetSpiderStatus(id,"0");
sfMapper.resetNextExecuteTime(id);
spiderJobManager.remove(id);
}
public void copy(String id){
// 复制ID
SpiderFlow spiderFlow = sfMapper.selectById(id);
String new_id = UUID.randomUUID().toString().replace("-", "");
sfMapper.insertSpiderFlow(new_id, spiderFlow.getName() + "-copy", spiderFlow.getXml());
}
public void start(String id){
spiderJobManager.remove(id);
SpiderFlow spiderFlow = getById(id);
@ -154,11 +147,11 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
sfMapper.resetSpiderStatus(id, "1");
}
}
public void run(String id){
spiderJobManager.run(id);
}
public void resetExecuteCount(String id){
sfMapper.resetExecuteCount(id);
}
@ -167,11 +160,11 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
spiderJobManager.remove(id);
flowNoticeMapper.deleteById(id);
}
public List<SpiderFlow> selectOtherFlows(String id){
return sfMapper.selectOtherFlows(id);
}
public List<SpiderFlow> selectFlows(){
return sfMapper.selectFlows();
}

View File

@ -48,29 +48,29 @@ import java.util.stream.Collectors;
@RestController
@RequestMapping("/spider")
public class SpiderFlowController {
@Autowired
private List<FunctionExecutor> functionExecutors;
@Autowired
private List<FunctionExtension> functionExtensions;
@Autowired
private List<Grammerable> grammerables;
@Autowired
private SpiderFlowService spiderFlowService;
@Autowired(required = false)
private List<PluginConfig> pluginConfigs;
@Value("${spider.workspace}")
private String workspace;
private final List<Grammer> grammers = new ArrayList<Grammer>();
private static Logger logger = LoggerFactory.getLogger(SpiderFlowController.class);
@PostConstruct
private void init(){
for (FunctionExecutor executor : functionExecutors) {
@ -84,7 +84,7 @@ public class SpiderFlowController {
grammer.setFunction(function);
grammers.add(grammer);
}
for (FunctionExtension extension : functionExtensions) {
String owner = extension.support().getSimpleName();
grammers.addAll(Grammer.findGrammers(extension.getClass(),null,owner,true));
@ -93,7 +93,7 @@ public class SpiderFlowController {
grammers.addAll(grammerable.grammers());
}
}
/**
* 爬虫列表
* @param page 页数
@ -104,7 +104,7 @@ public class SpiderFlowController {
public IPage<SpiderFlow> list(@RequestParam(name = "page", defaultValue = "1") Integer page, @RequestParam(name = "limit", defaultValue = "1") Integer size, @RequestParam(name = "name", defaultValue = "") String name) {
return spiderFlowService.selectSpiderPage(new Page<>(page, size), name);
}
@RequestMapping("/save")
public String save(SpiderFlow spiderFlow){
spiderFlowService.save(spiderFlow);
@ -119,12 +119,12 @@ public class SpiderFlowController {
return new JsonBean<>(spiderFlowService.historyList(id));
}
}
@RequestMapping("/get")
public SpiderFlow get(String id){
return spiderFlowService.getById(id);
}
@RequestMapping("/other")
public List<SpiderFlow> other(String id){
if(StringUtils.isBlank(id)){
@ -132,37 +132,32 @@ public class SpiderFlowController {
}
return spiderFlowService.selectOtherFlows(id);
}
@RequestMapping("/remove")
public void remove(String id){
spiderFlowService.remove(id);
}
@RequestMapping("/start")
public void start(String id){
spiderFlowService.start(id);
}
@RequestMapping("/stop")
public void stop(String id){
spiderFlowService.stop(id);
}
@RequestMapping("/copy")
public void copy(String id){
spiderFlowService.copy(id);
}
@RequestMapping("/run")
public void run(String id){
spiderFlowService.run(id);
}
@RequestMapping("/cron")
public void cron(String id,String cron){
spiderFlowService.resetCornExpression(id, cron);
}
@RequestMapping("/xml")
public String xml(String id){
return spiderFlowService.getById(id).getXml();
@ -197,17 +192,17 @@ public class SpiderFlowController {
return new JsonBean<>(-1,"读取日志文件出错");
}
}
@RequestMapping("/shapes")
public List<Shape> shapes(){
return ExecutorsUtils.shapes();
}
@RequestMapping("/pluginConfigs")
public List<Plugin> pluginConfigs(){
return null == pluginConfigs ? Collections.emptyList() : pluginConfigs.stream().filter(e-> e.plugin() != null).map(plugin -> plugin.plugin()).collect(Collectors.toList());
}
@RequestMapping("/grammers")
public JsonBean<List<Grammer>> grammers(){
return new JsonBean<>(this.grammers);

View File

@ -227,7 +227,7 @@ li {
周期从 <input class="numberspinner" style="width: 60px;"
data-options="min:0,max:23" value="0" id="hourStart_0">
- <input class="numberspinner" style="width: 60px;"
data-options="min:2,max:23" value="2" id="hourEnd_0"> 小时
data-options="min:2,max:23" value="2" id="hourEnd_1"> 小时
</div>
<div class="line">
<input type="radio" name="hour" onclick="startOn(this)">
@ -530,4 +530,4 @@ li {
});
</script>
</body>
</html>
</html>

View File

@ -84,7 +84,7 @@
align : 'center'
},{
title : '操作',
width : 250,
width : 195,
align : 'center',
templet : '#buttons'
}]]
@ -121,28 +121,6 @@
}
$("body").on('click','.btn-search',function(){
reloadTable();
}).on('click','.btn-copy',function(){
var id = $(this).data('id');
layui.layer.confirm('您确定要复制此爬虫吗',{
title : '复制'
},function(index){
$table.reload();
$.ajax({
url : 'spider/copy',
data : {
id : id
},
success : function(){
layui.layer.msg('复制成功',{time : 500},function(){
$table.reload();
})
},
error : function(){
layui.layer.msg('复制失败')
}
})
layui.layer.close(index);
})
}).on('click','.btn-remove',function(){
var id = $(this).data('id');
layui.layer.confirm('您确定要删除此爬虫吗',{
@ -227,4 +205,4 @@
<a class="layui-btn layui-btn-sm btn-remove" data-id="{{d.id}}" title="删除"><i class="layui-icon">&#xe640;</i></a>
</script>
</body>
</html>
</html>