1、调整日志
2、新增resp获取title，url，新增resp.images() 3、新增url批量下载，不用再先爬取再下载 4、增加List<String>过滤方法 5、新增string.lastIndexOf 6、新增多函数执行 7、新增ThreadFunctionExecutor 8、调整RequestExecutor中延迟时间对第一次执行也生效 9、隐藏数据源中密码显示 10、日志查看增加动态滚动
This commit is contained in:
td_zhangyu
parent
5b460f618a
commit
c217f8f1d5
@ -94,6 +94,10 @@ public class SpiderContext extends HashMap<String, Object>{
|
||||
public void debug(String message,Object ... variables){
|
||||
log("debug",message,variables);
|
||||
}
|
||||
|
||||
public void warn(String message, Object... variables) {
|
||||
log("warn", message, variables);
|
||||
}
|
||||
|
||||
public void error(String message,Object ... variables){
|
||||
log("error",message,variables);
|
||||
@ -104,6 +108,8 @@ public class SpiderContext extends HashMap<String, Object>{
|
||||
log.info(message,variables);
|
||||
}else if("debug".equals(level) && log.isDebugEnabled()){
|
||||
log.debug(message,variables);
|
||||
}else if("warn".equals(level)){
|
||||
log.warn(message, variables);
|
||||
}else if("error".equals(level)){
|
||||
log.error(message, variables);
|
||||
}
|
||||
|
||||
@ -14,6 +14,10 @@ public interface SpiderResponse {
|
||||
@Example("${resp.statusCode}")
|
||||
int getStatusCode();
|
||||
|
||||
@Comment("获取网页标题")
|
||||
@Example("${resp.title}")
|
||||
String getTitle();
|
||||
|
||||
@Comment("获取网页html")
|
||||
@Example("${resp.html}")
|
||||
String getHtml();
|
||||
@ -41,9 +45,7 @@ public interface SpiderResponse {
|
||||
|
||||
@Comment("获取当前url")
|
||||
@Example("${resp.url}")
|
||||
default String getUrl(){
|
||||
return null;
|
||||
}
|
||||
String getUrl();
|
||||
|
||||
@Example("${resp.setCharset('UTF-8')}")
|
||||
default void setCharset(String charset){
|
||||
|
||||
@ -2,12 +2,15 @@ package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.core.utils.FileUtils;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.DigestUtils;
|
||||
|
||||
/**
|
||||
@ -87,6 +90,16 @@ public class FileFunctionExecutor implements FunctionExecutor{
|
||||
public static void write(String path,byte[] bytes) throws IOException{
|
||||
write(path,bytes,false);
|
||||
}
|
||||
|
||||
@Comment("下载Url资源")
|
||||
@Example("${file.write('e:/result.html',urls)}")
|
||||
public static void write(String path, List<String> urls) throws IOException{
|
||||
if(!CollectionUtils.isEmpty(urls)) {
|
||||
for (String url : urls) {
|
||||
FileUtils.downloadFile(path, url, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Comment("读取文件")
|
||||
@Example("${file.bytes('e:/result.html')}")
|
||||
|
||||
@ -2,6 +2,7 @@ package org.spiderflow.core.executor.function;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
@ -55,5 +56,20 @@ public class ListFunctionExecutor implements FunctionExecutor{
|
||||
public static List<?> sublist(List<?> list,int fromIndex,int toIndex){
|
||||
return list!= null ? list.subList(fromIndex, toIndex) : new ArrayList<>();
|
||||
}
|
||||
|
||||
@Comment("过滤字符串list元素")
|
||||
@Example("${listVar.filterStr(pattern)}")
|
||||
public static List<String> filterStr(List<String> list, String pattern) {
|
||||
if (list == null || list.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
List<String> result = new ArrayList<>(list.size());
|
||||
for (String item : list) {
|
||||
if (Pattern.matches(pattern, item)) {
|
||||
result.add(item);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -55,6 +55,12 @@ public class StringFunctionExecutor implements FunctionExecutor{
|
||||
return content != null ? content.indexOf(str) : -1;
|
||||
}
|
||||
|
||||
@Comment("查找指定字符在字符串中最后出现的位置")
|
||||
@Example("${string.lastIndexOf(content,str)}")
|
||||
public static int lastIndexOf(String content, String str) {
|
||||
return content != null ? content.lastIndexOf(str) : -1;
|
||||
}
|
||||
|
||||
@Comment("查找指定字符在字符串在中的位置")
|
||||
@Example("${string.indexOf(content,str,fromIndex)}")
|
||||
public static int indexOf(String content, String str, int fromIndex) {
|
||||
@ -166,5 +172,5 @@ public class StringFunctionExecutor implements FunctionExecutor{
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,30 @@
|
||||
package org.spiderflow.core.executor.function;
|
||||
|
||||
import org.spiderflow.annotation.Comment;
|
||||
import org.spiderflow.annotation.Example;
|
||||
import org.spiderflow.executor.FunctionExecutor;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Created on 2019-12-06
|
||||
*
|
||||
* @author YZ255027 - Octopus
|
||||
*/
|
||||
@Component
|
||||
@Comment("thread常用方法")
|
||||
public class ThreadFunctionExecutor implements FunctionExecutor {
|
||||
@Override
|
||||
public String getFunctionPrefix() {
|
||||
return "thread";
|
||||
}
|
||||
|
||||
@Comment("线程休眠")
|
||||
@Example("${thread.sleep(1000L)}")
|
||||
public static void sleep(Long sleepTime){
|
||||
try {
|
||||
Thread.sleep(sleepTime);
|
||||
} catch (InterruptedException e) {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -115,4 +115,13 @@ public class ResponseFunctionExtension implements FunctionExtension {
|
||||
.filter(link -> pattern.matcher(link).matches())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Comment("获取当前页面所有图片链接")
|
||||
@Example("${resp.images()}")
|
||||
public static List<String> images(SpiderResponse response) {
|
||||
return ExtractUtils.getAttrBySelector(element(response), "img", "abs:src")
|
||||
.stream()
|
||||
.filter(link -> StringUtils.isNotBlank(link))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,9 +52,9 @@ public class ExecuteSQLExecutor implements ShapeExecutor, Grammerable {
|
||||
String dsId = node.getStringJsonValue(DATASOURCE_ID);
|
||||
String sql = node.getStringJsonValue(SQL);
|
||||
if (!StringUtils.isNotBlank(dsId)) {
|
||||
context.debug("数据源ID为空！");
|
||||
context.warn("数据源ID为空！");
|
||||
} else if (!StringUtils.isNotBlank(sql)) {
|
||||
context.debug("sql为空！");
|
||||
context.warn("sql为空！");
|
||||
} else {
|
||||
JdbcTemplate template = new JdbcTemplate(DataSourceUtils.getDataSource(dsId));
|
||||
//把变量替换成占位符
|
||||
@ -63,7 +63,7 @@ public class ExecuteSQLExecutor implements ShapeExecutor, Grammerable {
|
||||
try {
|
||||
Object sqlObject = engine.execute(sql, variables);
|
||||
if(sqlObject == null){
|
||||
context.debug("获取的sql为空！");
|
||||
context.warn("获取的sql为空！");
|
||||
return;
|
||||
}
|
||||
sql = sqlObject.toString();
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package org.spiderflow.core.executor.shape;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
@ -26,13 +27,16 @@ public class FunctionExecutor implements ShapeExecutor{
|
||||
|
||||
@Override
|
||||
public void execute(SpiderNode node, SpiderContext context, Map<String,Object> variables) {
|
||||
String function = node.getStringJsonValue(FUNCTION);
|
||||
if(StringUtils.isNotBlank(function)){
|
||||
try {
|
||||
engine.execute(function, variables);
|
||||
} catch (Exception e) {
|
||||
context.error("执行函数{}失败,异常信息:{}",function,e);
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
List<Map<String, String>> functions = node.getListJsonValue(FUNCTION);
|
||||
for (Map<String, String> item : functions) {
|
||||
String function = item.get(FUNCTION);
|
||||
if(StringUtils.isNotBlank(function)){
|
||||
try {
|
||||
engine.execute(function, variables);
|
||||
} catch (Exception e) {
|
||||
context.error("执行函数{}失败,异常信息:{}",function,e);
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -68,7 +68,7 @@ public class LoopJoinExecutor implements ShapeExecutor {
|
||||
}
|
||||
return isDone;
|
||||
} else {
|
||||
context.error("找不到等待节点：{}" + node.getStringJsonValue(JOIN_NODE_ID));
|
||||
context.warn("找不到等待节点：{}" + node.getStringJsonValue(JOIN_NODE_ID));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -100,14 +100,17 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
Object value = engine.execute(sleepCondition, variables);
|
||||
if(value != null){
|
||||
long sleepTime = NumberUtils.toLong(value.toString(), 0L);
|
||||
synchronized (node.getNodeId().intern()){
|
||||
synchronized (node.getNodeId().intern()) {
|
||||
//实际等待时间 = 上次执行时间 + 睡眠时间 - 当前时间
|
||||
sleepTime = context.get(LAST_EXECUTE_TIME + node.getNodeId(), 0L) + sleepTime - System.currentTimeMillis();
|
||||
if(sleepTime > 0){
|
||||
Long lastExecuteTime = context.get(LAST_EXECUTE_TIME + node.getNodeId(), 0L);
|
||||
if (lastExecuteTime != 0) {
|
||||
sleepTime = lastExecuteTime + sleepTime - System.currentTimeMillis();
|
||||
}
|
||||
if (sleepTime > 0) {
|
||||
context.info("设置延迟时间:{}ms", sleepTime);
|
||||
Thread.sleep(sleepTime);
|
||||
}
|
||||
context.put(LAST_EXECUTE_TIME + node.getNodeId(),System.currentTimeMillis());
|
||||
context.put(LAST_EXECUTE_TIME + node.getNodeId(), System.currentTimeMillis());
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
@ -220,7 +223,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
//结果存入变量
|
||||
variables.put("resp", response);
|
||||
} catch (IOException e) {
|
||||
context.debug("请求{}出错,异常信息:{}",url,e);
|
||||
context.error("请求{}出错,异常信息:{}",url,e);
|
||||
ExceptionUtils.wrapAndThrow(e);
|
||||
} finally{
|
||||
if(streams != null){
|
||||
@ -261,7 +264,7 @@ public class RequestExecutor implements ShapeExecutor,Grammerable{
|
||||
request.data(parameterName, parameterFilename, stream);
|
||||
context.debug("设置请求参数：{}={}",parameterName,parameterFilename);
|
||||
}else{
|
||||
context.debug("设置请求参数：{}失败，无二进制内容",parameterName);
|
||||
context.warn("设置请求参数：{}失败，无二进制内容",parameterName);
|
||||
}
|
||||
}else{
|
||||
request.data(parameterName, value);
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package org.spiderflow.core.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import org.jsoup.Connection.Response;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.spiderflow.io.SpiderResponse;
|
||||
|
||||
@ -16,27 +16,46 @@ import com.alibaba.fastjson.JSON;
|
||||
*
|
||||
*/
|
||||
public class HttpResponse implements SpiderResponse{
|
||||
|
||||
|
||||
private Response response;
|
||||
|
||||
private int statusCode;
|
||||
|
||||
private String urlLink;
|
||||
|
||||
private Document document;
|
||||
|
||||
private String htmlValue;
|
||||
|
||||
private String titleName;
|
||||
|
||||
private Object jsonValue;
|
||||
|
||||
public HttpResponse(Response response) {
|
||||
public HttpResponse(Response response) throws IOException {
|
||||
super();
|
||||
this.response = response;
|
||||
this.statusCode = response.statusCode();
|
||||
this.urlLink = response.url().toExternalForm();
|
||||
document = response.parse();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getStatusCode(){
|
||||
return response.statusCode();
|
||||
return statusCode;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getTitle() {
|
||||
if (titleName == null) {
|
||||
titleName = document.title();
|
||||
}
|
||||
return titleName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHtml(){
|
||||
if(htmlValue == null){
|
||||
htmlValue = response.body();
|
||||
htmlValue = document.body().html();
|
||||
}
|
||||
return htmlValue;
|
||||
}
|
||||
@ -76,7 +95,7 @@ public class HttpResponse implements SpiderResponse{
|
||||
|
||||
@Override
|
||||
public String getUrl() {
|
||||
return response.url().toExternalForm();
|
||||
return urlLink;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -81,7 +81,8 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
|
||||
spiderJobManager.addJob(spiderFlow);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean save(SpiderFlow spiderFlow){
|
||||
//解析corn,获取并设置任务的开始时间
|
||||
if(StringUtils.isNotEmpty(spiderFlow.getCron())){
|
||||
@ -163,5 +164,4 @@ public class SpiderFlowService extends ServiceImpl<SpiderFlowMapper, SpiderFlow>
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,249 @@
|
||||
package org.spiderflow.core.utils;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.io.*;
|
||||
import java.net.*;
|
||||
|
||||
/**
|
||||
* 文件处理工具类
|
||||
*
|
||||
* @author ruoyi
|
||||
*/
|
||||
public class FileUtils
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(FileUtils.class);
|
||||
|
||||
public static String FILENAME_PATTERN = "[a-zA-Z0-9_\\-\\|\\.\\u4e00-\\u9fa5]+";
|
||||
|
||||
/**
|
||||
* 输出指定文件的byte数组
|
||||
*
|
||||
* @param filePath 文件路径
|
||||
* @param os 输出流
|
||||
* @return
|
||||
*/
|
||||
public static void writeBytes(String filePath, OutputStream os) throws IOException
|
||||
{
|
||||
FileInputStream fis = null;
|
||||
try
|
||||
{
|
||||
File file = new File(filePath);
|
||||
if (!file.exists())
|
||||
{
|
||||
throw new FileNotFoundException(filePath);
|
||||
}
|
||||
fis = new FileInputStream(file);
|
||||
byte[] b = new byte[1024];
|
||||
int length;
|
||||
while ((length = fis.read(b)) > 0)
|
||||
{
|
||||
os.write(b, 0, length);
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw e;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (os != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
os.close();
|
||||
}
|
||||
catch (IOException e1)
|
||||
{
|
||||
e1.printStackTrace();
|
||||
}
|
||||
}
|
||||
if (fis != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
fis.close();
|
||||
}
|
||||
catch (IOException e1)
|
||||
{
|
||||
e1.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文件
|
||||
*
|
||||
* @param filePath 文件
|
||||
* @return
|
||||
*/
|
||||
public static boolean deleteFile(String filePath)
|
||||
{
|
||||
boolean flag = false;
|
||||
File file = new File(filePath);
|
||||
// 路径为文件且不为空则进行删除
|
||||
if (file.isFile() && file.exists())
|
||||
{
|
||||
file.delete();
|
||||
flag = true;
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件名称验证
|
||||
*
|
||||
* @param filename 文件名称
|
||||
* @return true 正常 false 非法
|
||||
*/
|
||||
public static boolean isValidFilename(String filename)
|
||||
{
|
||||
return filename.matches(FILENAME_PATTERN);
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载文件名重新编码
|
||||
*
|
||||
* @param request 请求对象
|
||||
* @param fileName 文件名
|
||||
* @return 编码后的文件名
|
||||
*/
|
||||
public static String setFileDownloadHeader(HttpServletRequest request, String fileName)
|
||||
throws UnsupportedEncodingException
|
||||
{
|
||||
final String agent = request.getHeader("USER-AGENT");
|
||||
String filename = fileName;
|
||||
if (agent.contains("MSIE"))
|
||||
{
|
||||
// IE浏览器
|
||||
filename = URLEncoder.encode(filename, "utf-8");
|
||||
filename = filename.replace("+", " ");
|
||||
}
|
||||
else if (agent.contains("Firefox"))
|
||||
{
|
||||
// 火狐浏览器
|
||||
filename = new String(fileName.getBytes(), "ISO8859-1");
|
||||
}
|
||||
else if (agent.contains("Chrome"))
|
||||
{
|
||||
// google浏览器
|
||||
filename = URLEncoder.encode(filename, "utf-8");
|
||||
}
|
||||
else
|
||||
{
|
||||
// 其它浏览器
|
||||
filename = URLEncoder.encode(filename, "utf-8");
|
||||
}
|
||||
return filename;
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件下载状态
|
||||
*/
|
||||
public enum DownloadStatus {
|
||||
URL_ERROR(1, "URL错误"),
|
||||
FILE_EXIST(2,"文件存在"),
|
||||
TIME_OUT(3,"连接超时"),
|
||||
DOWNLOAD_FAIL(4,"下载失败"),
|
||||
DOWNLOAD_SUCCESS(5,"下载成功");
|
||||
|
||||
private int code;
|
||||
|
||||
private String name;
|
||||
|
||||
DownloadStatus(int code, String name){
|
||||
this.code = code;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public int getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public void setCode(int code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
||||
public static DownloadStatus downloadFile(String savePath, String fileUrl, boolean downNew) {
|
||||
URL urlfile = null;
|
||||
HttpURLConnection httpUrl = null;
|
||||
BufferedInputStream bis = null;
|
||||
BufferedOutputStream bos = null;
|
||||
if (fileUrl.startsWith("//")) {
|
||||
fileUrl = "http:" + fileUrl;
|
||||
}
|
||||
String fileName;
|
||||
try {
|
||||
urlfile = new URL(fileUrl);
|
||||
String urlPath = urlfile.getPath();
|
||||
fileName = urlPath.substring(urlPath.lastIndexOf("/") + 1);
|
||||
} catch (MalformedURLException e) {
|
||||
logger.error("URL异常", e);
|
||||
return DownloadStatus.URL_ERROR;
|
||||
}
|
||||
File path = new File(savePath);
|
||||
if (!path.exists()) {
|
||||
path.mkdirs();
|
||||
}
|
||||
File file = new File(savePath + File.separator + fileName);
|
||||
if (file.exists()) {
|
||||
if (downNew) {
|
||||
file.delete();
|
||||
} else {
|
||||
logger.info("文件已存在不重新下载！");
|
||||
return DownloadStatus.FILE_EXIST;
|
||||
}
|
||||
}
|
||||
try {
|
||||
httpUrl = (HttpURLConnection) urlfile.openConnection();
|
||||
httpUrl.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0");
|
||||
//读取超时时间
|
||||
httpUrl.setReadTimeout(60000);
|
||||
//连接超时时间
|
||||
httpUrl.setConnectTimeout(60000);
|
||||
httpUrl.connect();
|
||||
bis = new BufferedInputStream(httpUrl.getInputStream());
|
||||
bos = new BufferedOutputStream(new FileOutputStream(file));
|
||||
int len = 2048;
|
||||
byte[] b = new byte[len];
|
||||
long readLen = 0;
|
||||
while ((len = bis.read(b)) != -1) {
|
||||
bos.write(b, 0, len);
|
||||
}
|
||||
logger.info("远程文件下载成功:" + fileUrl);
|
||||
bos.flush();
|
||||
bis.close();
|
||||
httpUrl.disconnect();
|
||||
return DownloadStatus.DOWNLOAD_SUCCESS;
|
||||
} catch (SocketTimeoutException e) {
|
||||
logger.error("读取文件超时", e);
|
||||
return DownloadStatus.TIME_OUT;
|
||||
} catch (Exception e) {
|
||||
logger.error("远程文件下载失败", e);
|
||||
return DownloadStatus.DOWNLOAD_FAIL;
|
||||
} finally {
|
||||
try {
|
||||
if (bis != null) {
|
||||
bis.close();
|
||||
}
|
||||
if (bos != null) {
|
||||
bos.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("下载出错", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -27,7 +27,7 @@ public class SpiderApplication implements ServletContextInitializer{
|
||||
@Override
|
||||
public void onStartup(ServletContext servletContext) throws ServletException {
|
||||
//设置文本缓存1M
|
||||
servletContext.setInitParameter("org.apache.tomcat.websocket.textBufferSize", "" + (1024 * 1024));
|
||||
servletContext.setInitParameter("org.apache.tomcat.websocket.textBufferSize", Integer.toString((1024 * 1024)));
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@ -46,7 +46,9 @@ public class DataSourceController {
|
||||
|
||||
@RequestMapping("/get")
|
||||
public DataSource get(String id){
|
||||
return dataSourceService.getById(id);
|
||||
DataSource dataSource = dataSourceService.getById(id);
|
||||
dataSource.setPassword(null);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
@RequestMapping("/remove")
|
||||
|
||||
@ -90,6 +90,10 @@ html,body{
|
||||
top : 30px;
|
||||
color : #333;
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
}
|
||||
.properties-container .editor-form-node .layui-form-relative .layui-icon-close.function-remove{
|
||||
top : 5px;
|
||||
}
|
||||
|
||||
.toolbar-container ul li{
|
||||
|
||||
@ -55,7 +55,7 @@
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">密码</label>
|
||||
<div class="layui-input-block">
|
||||
<input type="password" name="password" placeholder="请输入密码" autocomplete="off" class="layui-input" />
|
||||
<input type="password" name="password" placeholder="请输入密码" autocomplete="off" class="layui-input" lay-verify="required"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="btns-submit">
|
||||
@ -73,10 +73,10 @@
|
||||
data : {
|
||||
id : dsId
|
||||
},
|
||||
success : function(data){
|
||||
layui.form.val('form',data)
|
||||
success : function(data) {
|
||||
layui.form.val('form', data);
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
layui.form.on('submit(save)',function(){
|
||||
$.ajax({
|
||||
@ -103,7 +103,7 @@
|
||||
})
|
||||
return false;
|
||||
})
|
||||
layui.form.on('submit(test)',function(){
|
||||
layui.form.on('submit(test)', function () {
|
||||
sf.ajax({
|
||||
url : 'datasource/test',
|
||||
type : 'post',
|
||||
@ -119,7 +119,7 @@
|
||||
error : function(){
|
||||
layui.layer.msg('请求失败');
|
||||
}
|
||||
})
|
||||
});
|
||||
return false;
|
||||
})
|
||||
</script>
|
||||
|
||||
@ -25,7 +25,7 @@
|
||||
</div>
|
||||
<div class="layui-body">
|
||||
<div class="layui-tab" lay-filter="admin-tab" lay-allowClose="true">
|
||||
<ul class="layui-tab-title admin-unselect" unselectable="on" lay-allowClose="false">
|
||||
<ul id="layuiTab" class="layui-tab-title admin-unselect" unselectable="on" lay-allowClose="false">
|
||||
<li class="layui-this layui-tab-hidden-close" lay-id="welcome">爬虫列表</li>
|
||||
</ul>
|
||||
<div class="layui-tab-content">
|
||||
|
||||
@ -213,6 +213,10 @@ $(function(){
|
||||
$dom.next().remove();
|
||||
$dom.remove();
|
||||
serializeForm();
|
||||
}).on("click",".editor-form-node .function-remove",function () {
|
||||
var $dom = $(this).parent();
|
||||
$dom.remove();
|
||||
serializeForm();
|
||||
}).on("click",".editor-form-node .header-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close header-remove"></i><label class="layui-form-label">header名</label><div class="layui-input-block"><input type="text" name="header-name" placeholder="header key" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">header值</label><div class="layui-input-block array" codemirror="header-value" placeholder="请输入header value"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
@ -223,7 +227,10 @@ $(function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close cookie-remove"></i><label class="layui-form-label">Cookie名</label><div class="layui-input-block"><input type="text" name="cookie-name" placeholder="请输入Cookie名" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">Cookie值</label><div class="layui-input-block array" codemirror="cookie-value" placeholder="请输入Cookie值"></div></div><hr>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".editor-form-node .output-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close output-remove"></i><label class="layui-form-label">输出项</label><div class="layui-input-block"><input type="text" name="output-name" placeholder="请输入输出项" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">输出值</label><div class="layui-input-block array" codemirror="output-value" placeholder="请输入输出值"></div></div><hr>');
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close output-remove"></i><label class="layui-form-label">输出项</label><div class="layui-input-block"><input type="text" name="output-name" placeholder="请输入输出项" autocomplete="off" class="layui-input array"></div></div><div class="layui-form-item"><label class="layui-form-label">输出值</label><div class="layui-input-block array" codemirror="output-value" placeholder="请输入输出值"></div></div>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".editor-form-node .function-add",function(){
|
||||
$(this).parent().parent().before('<div class="layui-form-item layui-form-relative"><i class="layui-icon layui-icon-close function-remove"></i><label class="layui-form-label">执行函数</label><div class="layui-input-block array" codemirror="function" placeholder="执行函数"></div></div>');
|
||||
renderCodeMirror();
|
||||
}).on("click",".parameter-form-add",function(){
|
||||
var html = '';
|
||||
@ -324,14 +331,11 @@ $(function(){
|
||||
if(cell.data.shape != 'start'){
|
||||
loadTemplate(cell.data.object.shape,cell,graph);
|
||||
}
|
||||
|
||||
}
|
||||
}else{
|
||||
loadTemplate('root',graph.getModel().getRoot(),graph);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 重置已设表单array（参数、变量、Headers）
|
||||
*/
|
||||
@ -719,6 +723,8 @@ function bindToolbarClickAction(editor){
|
||||
Save();
|
||||
})
|
||||
}
|
||||
//最近点击打开的弹窗
|
||||
var index;
|
||||
function onCanvasViewerClick(e,source){
|
||||
var msg = e.text;
|
||||
var json;
|
||||
@ -736,7 +742,8 @@ function onCanvasViewerClick(e,source){
|
||||
msg = temp.innerHTML;
|
||||
temp = null;
|
||||
}
|
||||
layer.open({
|
||||
layer.close(index);
|
||||
index = layer.open({
|
||||
type : 1,
|
||||
title : source +'内容',
|
||||
content: '<div class="message-content" style="padding:10px;'+(json ? '':'font-weight: bold;font-family:Consolas;font-size:12px;')+'">'+(json ? '' : msg.replace(/\n/g,'<br>')).replace(/ /g,' ').replace(/\t/g,' ')+'</div>',
|
||||
@ -755,12 +762,7 @@ function onCanvasViewerClick(e,source){
|
||||
}
|
||||
function createWebSocket(options){
|
||||
options = options || {};
|
||||
var socket;
|
||||
if(location.host === 'demo.spiderflow.org'){
|
||||
socket = new WebSocket(options.url || 'ws://49.233.182.130:8088/ws');
|
||||
}else{
|
||||
socket = new WebSocket(options.url || (location.origin.replace("http",'ws') + '/ws'));
|
||||
}
|
||||
var socket = new WebSocket(options.url || (location.origin.replace("http",'ws') + '/ws'));
|
||||
socket.onopen = options.onopen;
|
||||
socket.onmessage = options.onmessage;
|
||||
socket.onerror = options.onerror || function(){
|
||||
|
||||
@ -1,21 +1,33 @@
|
||||
var $ = layui.$;
|
||||
function openTab(title,href){
|
||||
if($(".layui-tab[lay-filter=admin-tab]").find("[lay-id='"+title+"']").length > 0){ //判断是否已打开
|
||||
function openTab(title,id,href){
|
||||
if($(".layui-tab[lay-filter=admin-tab]").find("[lay-id="+id+"]").length > 0){ //判断是否已打开
|
||||
var $dom = $(".layui-tab[lay-filter=admin-tab]");
|
||||
var index = $dom.find("[lay-id='"+title+"']").index();
|
||||
var index = $dom.find("[lay-id="+id+"]").index();
|
||||
$dom.find(".layui-tab-content .layui-tab-item").eq(index).find("iframe").attr("src",href);
|
||||
}else{
|
||||
var html = '<iframe src="'+href+'" width="100%" height="100%" scrolling="yes" frameborder="0"></iframe>';
|
||||
layui.element.tabAdd('admin-tab',{
|
||||
title:title,
|
||||
content:html,
|
||||
id:title,
|
||||
id:id,
|
||||
});
|
||||
}
|
||||
layui.element.tabChange("admin-tab",title);
|
||||
layui.element.tabChange("admin-tab",id);
|
||||
}
|
||||
$(function(){
|
||||
layui.element.init();
|
||||
$.ajax({
|
||||
url:'spider/pluginConfigs',
|
||||
success:function(data){
|
||||
for(var i =0;i<data.length;i++){
|
||||
$(".menu-list .layui-nav-tree").append('<li class="layui-nav-item layui-nav-itemed"><a data-link="'+data[i].url+'" title="'+data[i].name+'">'+data[i].name+'</a></li>');
|
||||
}
|
||||
layui.element.init();
|
||||
initMenu();
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
function initMenu() {
|
||||
$("body").on('click','.menu-list li a',function(){
|
||||
$(this).parents("ul").siblings().find("li.layui-this,dd.layui-this").removeClass('layui-this')
|
||||
}).on('click','.menu-list > ul',function(){
|
||||
@ -24,16 +36,8 @@ $(function(){
|
||||
var href = $(this).data('link');
|
||||
if(href){
|
||||
var title = $(this).html();
|
||||
openTab(title,href);
|
||||
openTab(title, title, href);
|
||||
return false;
|
||||
}
|
||||
})
|
||||
$.ajax({
|
||||
url:'spider/pluginConfigs',
|
||||
success:function(data){
|
||||
for(var i =0;i<data.length;i++){
|
||||
$(".menu-list .layui-nav-tree").append('<li class="layui-nav-item layui-nav-itemed"><a data-link="'+data[i].url+'" title="'+data[i].name+'">'+data[i].name+'</a></li>');
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -169,10 +169,11 @@
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
$(function(){
|
||||
var logId = getQueryString('id');
|
||||
var viewer = new LogViewer({
|
||||
url: 'spider/log',
|
||||
maxLines : parseInt(($('.log-container').height() - 8) / 14),
|
||||
logId : getQueryString('id'),
|
||||
logId : logId,
|
||||
element : $('.log-container'),
|
||||
onSearchFinish : function(hasData){
|
||||
if(hasData){
|
||||
@ -186,13 +187,14 @@
|
||||
layui.layer.alert('日志文件不存在');
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
var setOptions = function(){
|
||||
viewer.setOptions('keywords',$('.toolbox-container .input-text').val());
|
||||
viewer.setOptions('matchcase',$('#matchcase').is(':checked'));
|
||||
viewer.setOptions('regx',$('#regx').is(':checked'));
|
||||
viewer.setOptions('reversed',$('#reversed').is(':checked'));
|
||||
}
|
||||
var time;
|
||||
$('.toolbox-container').on('keydown','.input-text',function(e){
|
||||
setOptions();
|
||||
if(e.keyCode === 13){
|
||||
@ -223,8 +225,17 @@
|
||||
viewer.setOptions('reversed',false);
|
||||
viewer.init();
|
||||
}).on('click','.btn-page-end',function(){
|
||||
viewer.setOptions('reversed',true);
|
||||
viewer.init();
|
||||
if (time == null) {
|
||||
time = window.setInterval(function () {
|
||||
if ($('#layuiTab', parent.document).find("li[lay-id='" + logId + "']").attr('class').indexOf("layui-this") >= 0) {
|
||||
viewer.setOptions('reversed',true);
|
||||
viewer.init();
|
||||
}
|
||||
}, 5000);
|
||||
} else {
|
||||
window.clearInterval(time);
|
||||
time = null;
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
@ -21,10 +21,20 @@
|
||||
<label class="layui-form-label">循环次数</label>
|
||||
<div class="layui-input-block" codemirror="loopCount" placeholder="请输入循环次数" data-value="{{=d.data.object.loopCount}}"></div>
|
||||
</div>
|
||||
<div class="layui-form-item">
|
||||
<label class="layui-form-label">执行函数</label>
|
||||
<div class="layui-input-block" codemirror="function" placeholder="执行函数" data-value="{{=d.data.object.function}}"></div>
|
||||
</div>
|
||||
<hr>
|
||||
{{# layui.each(d.data.object['function'],function(index,item){ }}
|
||||
<div class="layui-form-item layui-form-relative">
|
||||
<i class="layui-icon layui-icon-close function-remove"></i>
|
||||
<label class="layui-form-label">执行函数</label>
|
||||
<div class="layui-input-block array" codemirror="function" placeholder="执行函数" data-value="{{=d.data.object['function'][index]}}"></div>
|
||||
</div>
|
||||
{{# }) }}
|
||||
<div class="layui-form-item">
|
||||
<div class="layui-input-block">
|
||||
<button class="layui-btn function-add" type="button">添加一个函数</button>
|
||||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -50,7 +50,7 @@
|
||||
},{
|
||||
title : 'cron表达式',
|
||||
field : 'cron',
|
||||
width : 100,
|
||||
width : 120,
|
||||
templet : function(row){
|
||||
return '<a class="layui-btn layui-btn-sm btn-edit-cron" data-id="'+row.id+'" data-cron="'+row.cron+'">'+(row.cron || '编辑cron')+'</a>';
|
||||
}
|
||||
@ -72,11 +72,10 @@
|
||||
field : 'lastExecuteTime',
|
||||
align : 'center'
|
||||
},{
|
||||
title : '运行中/已完成',
|
||||
width : 120,
|
||||
title : '已执行',
|
||||
width : 80,
|
||||
field : 'executeCount',
|
||||
align : 'center',
|
||||
templet : '#execute-count'
|
||||
align : 'center'
|
||||
},{
|
||||
title : '下次执行时间',
|
||||
width : 160,
|
||||
@ -165,10 +164,8 @@
|
||||
})
|
||||
layui.layer.close(index);
|
||||
})
|
||||
}).on('click','.btn-task',function(){
|
||||
parent.openTab($(this).data('name') + '-任务详情','task.html?id=' + $(this).data('id') + '&name=' + encodeURIComponent(encodeURIComponent($(this).data('name'))));
|
||||
}).on('click','.btn-log',function(){
|
||||
parent.openTab($(this).data('name') + '-日志','log.html?id=' + $(this).data('id'));
|
||||
parent.openTab($(this).data('name') + '-日志',$(this).data('id'),'log.html?id=' + $(this).data('id'));
|
||||
}).on('click','.btn-edit-cron',function(){
|
||||
var id = $(this).data('id');
|
||||
var value = $(this).data('cron') || '';
|
||||
@ -182,9 +179,6 @@
|
||||
});
|
||||
})
|
||||
</script>
|
||||
<script type="text/html" id="execute-count">
|
||||
<a class="layui-btn layui-btn-sm btn-task" data-name="{{d.name}}" data-id="{{d.id}}">{{d.running || 0}}/{{d.executeCount || 0}}</a>
|
||||
</script>
|
||||
<script type="text/html" id="buttons">
|
||||
<a class="layui-btn layui-btn-sm btn-run" data-id="{{d.id}}">手动运行</a>
|
||||
<a class="layui-btn layui-btn-sm btn-log" data-name="{{d.name}}" data-id="{{d.id}}">查看日志</a>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user