diff --git a/spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java b/spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java index 716bc5a..433dbf9 100644 --- a/spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java +++ b/spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java @@ -1,5 +1,6 @@ package org.spiderflow.io; +import java.io.InputStream; import java.util.Map; import org.spiderflow.annotation.Comment; @@ -48,4 +49,8 @@ public interface SpiderResponse { default void setCharset(String charset){ } + @Example("${resp.stream}") + default InputStream getStream(){ + return null; + } } diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java b/spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java index 81fe705..ffb34b3 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/executor/function/FileFunctionExecutor.java @@ -1,102 +1,111 @@ -package org.spiderflow.core.executor.function; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.charset.Charset; - -import org.apache.commons.io.IOUtils; -import org.spiderflow.annotation.Comment; -import org.spiderflow.annotation.Example; -import org.spiderflow.executor.FunctionExecutor; -import org.springframework.stereotype.Component; - -/** - * 文件读写 工具类 防止NPE - * @author Administrator - * - */ -@Component -@Comment("file常用方法") -public class FileFunctionExecutor implements FunctionExecutor{ - - @Override - public String getFunctionPrefix() { - return "file"; - } - - /** - * - * @param path 文件路径/名 - * @param createDirectory 是否需要创建 - * @return File 文件 - */ - private static File getFile(String path,boolean createDirectory){ - File f = new File(path); - if(createDirectory&&!f.getParentFile().exists()){ - f.getParentFile().mkdirs(); - } - return f; - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.html,false)}") - public static void write(String path,String content,boolean append) throws IOException{ - write(path,content,Charset.defaultCharset().name(),append); - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.html,'UTF-8',false)}") - public static void write(String path,String content,String charset,boolean append) throws IOException{ - write(path,StringFunctionExecutor.bytes(content, charset),append); - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.bytes,false)}") - public static void write(String path,byte[] bytes,boolean append) throws IOException{ - try(FileOutputStream fos = new FileOutputStream(getFile(path,true),append)){ - fos.write(bytes); - fos.flush(); - } - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.html)}") - public static void write(String path,String content) throws IOException{ - write(path,content,false); - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.html,'UTF-8')}") - public static void write(String path,String content,String charset) throws IOException{ - write(path,content,charset,false); - } - - @Comment("写出文件") - @Example("${file.write('e:/result.html',resp.bytes)}") - public static void write(String path,byte[] bytes) throws IOException{ - write(path,bytes,false); - } - - @Comment("读取文件") - @Example("${file.bytes('e:/result.html')}") - public static byte[] bytes(String path) throws IOException{ - try(FileInputStream fis = new FileInputStream(getFile(path, false))){ - return IOUtils.toByteArray(fis); - } - } - - @Comment("读取文件") - @Example("${file.string('e:/result.html','UTF-8')}") - public static String string(String path,String charset) throws IOException{ - return StringFunctionExecutor.newString(bytes(path), charset); - } - - @Comment("读取文件") - @Example("${file.string('e:/result.html')}") - public static String string(String path) throws IOException{ - return StringFunctionExecutor.newString(bytes(path), Charset.defaultCharset().name()); - } - -} +package org.spiderflow.core.executor.function; + +import java.io.*; +import java.nio.charset.Charset; + +import org.apache.commons.io.IOUtils; +import org.spiderflow.annotation.Comment; +import org.spiderflow.annotation.Example; +import org.spiderflow.executor.FunctionExecutor; +import org.springframework.stereotype.Component; +import org.springframework.util.DigestUtils; + +/** + * 文件读写 工具类 防止NPE + * @author Administrator + * + */ +@Component +@Comment("file常用方法") +public class FileFunctionExecutor implements FunctionExecutor{ + + @Override + public String getFunctionPrefix() { + return "file"; + } + + /** + * + * @param path 文件路径/名 + * @param createDirectory 是否需要创建 + * @return File 文件 + */ + private static File getFile(String path,boolean createDirectory){ + File f = new File(path); + if(createDirectory&&!f.getParentFile().exists()){ + f.getParentFile().mkdirs(); + } + return f; + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.html,false)}") + public static void write(String path,String content,boolean append) throws IOException{ + write(path,content,Charset.defaultCharset().name(),append); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.html,'UTF-8',false)}") + public static void write(String path,String content,String charset,boolean append) throws IOException{ + write(path,StringFunctionExecutor.bytes(content, charset),append); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.bytes,false)}") + public static void write(String path,byte[] bytes,boolean append) throws IOException{ + write(path, new ByteArrayInputStream(bytes),append); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.stream,false)}") + public static void write(String path, InputStream stream, boolean append) throws IOException { + try(FileOutputStream fos = new FileOutputStream(getFile(path,true),append)){ + IOUtils.copyLarge(stream, fos); + } + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.bytes,false)}") + public static void write(String path, InputStream stream) throws IOException { + write(path, stream,false); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.html)}") + public static void write(String path,String content) throws IOException{ + write(path,content,false); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.html,'UTF-8')}") + public static void write(String path,String content,String charset) throws IOException{ + write(path,content,charset,false); + } + + @Comment("写出文件") + @Example("${file.write('e:/result.html',resp.bytes)}") + public static void write(String path,byte[] bytes) throws IOException{ + write(path,bytes,false); + } + + @Comment("读取文件") + @Example("${file.bytes('e:/result.html')}") + public static byte[] bytes(String path) throws IOException{ + try(FileInputStream fis = new FileInputStream(getFile(path, false))){ + return IOUtils.toByteArray(fis); + } + } + + @Comment("读取文件") + @Example("${file.string('e:/result.html','UTF-8')}") + public static String string(String path,String charset) throws IOException{ + return StringFunctionExecutor.newString(bytes(path), charset); + } + + @Comment("读取文件") + @Example("${file.string('e:/result.html')}") + public static String string(String path) throws IOException{ + return StringFunctionExecutor.newString(bytes(path), Charset.defaultCharset().name()); + } + +} diff --git a/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java b/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java index c98f9a4..1d1b59b 100644 --- a/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java +++ b/spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java @@ -1,80 +1,85 @@ -package org.spiderflow.core.io; - -import java.util.Map; - -import org.jsoup.Connection.Response; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.spiderflow.io.SpiderResponse; - -import com.alibaba.fastjson.JSON; - -/** - * 响应对象包装类 - * @author Administrator - * - */ -public class HttpResponse implements SpiderResponse{ - - private Response response; - - - public HttpResponse() { - super(); - } - - public HttpResponse(Response response) { - super(); - this.response = response; - } - - @Override - public int getStatusCode(){ - return response.statusCode(); - } - - @Override - public String getHtml(){ - return response.body(); - } - - @Override - public Object getJson(){ - return JSON.parse(getHtml()); - } - - public Document getDocument(){ - return Jsoup.parse(getHtml()); - } - - @Override - public Map getCookies(){ - return response.cookies(); - } - - @Override - public Map getHeaders(){ - return response.headers(); - } - - @Override - public byte[] getBytes(){ - return response.bodyAsBytes(); - } - - @Override - public String getContentType(){ - return response.contentType(); - } - - @Override - public void setCharset(String charset) { - this.response.charset(charset); - } - - @Override - public String getUrl() { - return response.url().toExternalForm(); - } - -} +package org.spiderflow.core.io; + +import java.io.InputStream; +import java.util.Map; + +import org.jsoup.Connection.Response; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.spiderflow.io.SpiderResponse; + +import com.alibaba.fastjson.JSON; + +/** + * 响应对象包装类 + * @author Administrator + * + */ +public class HttpResponse implements SpiderResponse{ + + private Response response; + + + public HttpResponse() { + super(); + } + + public HttpResponse(Response response) { + super(); + this.response = response; + } + + @Override + public int getStatusCode(){ + return response.statusCode(); + } + + @Override + public String getHtml(){ + return response.body(); + } + + @Override + public Object getJson(){ + return JSON.parse(getHtml()); + } + + public Document getDocument(){ + return Jsoup.parse(getHtml()); + } + + @Override + public Map getCookies(){ + return response.cookies(); + } + + @Override + public Map getHeaders(){ + return response.headers(); + } + + @Override + public byte[] getBytes(){ + return response.bodyAsBytes(); + } + + @Override + public String getContentType(){ + return response.contentType(); + } + + @Override + public void setCharset(String charset) { + this.response.charset(charset); + } + + @Override + public String getUrl() { + return response.url().toExternalForm(); + } + + @Override + public InputStream getStream() { + return response.bodyStream(); + } +}