Spider/spider-flow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

页面编码格式设置

Browse Source
This commit is contained in:
mxd 2019-09-03 10:25:45 +08:00
parent 0fcced25de
commit bcd17dcf68
4 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spider-flow-api/src/main/java/org/spiderflow/io/SpiderResponse.java
View File

@ -22,5 +22,7 @@ public interface SpiderResponse {
public String getContentType();
default void setCharset(String charset){
}
}

10
spider-flow-core/src/main/java/org/spiderflow/core/executor/shape/RequestExecutor.java
View File

@ -65,6 +65,8 @@ public class RequestExecutor implements ShapeExecutor,Grammer{
public static final String TIMEOUT = "timeout";
public static final String RESPONSE_CHARSET = "response-charset";
private static Logger logger = LoggerFactory.getLogger(RequestExecutor.class);
@Autowired
@ -165,6 +167,14 @@ public class RequestExecutor implements ShapeExecutor,Grammer{
}
try {
HttpResponse response = request.execute();
String charset = node.getStringJsonValue(RESPONSE_CHARSET);
if(StringUtils.isNotBlank(charset)){
response.setCharset(charset);
if(logger.isDebugEnabled()){
logger.debug("设置response charset:{}" + charset);
}
context.log(String.format("设置response charset:%s", charset));
}
//结果存入变量
variables.put("resp", response);
} catch (IOException e) {

5
spider-flow-core/src/main/java/org/spiderflow/core/io/HttpResponse.java
View File

@ -61,4 +61,9 @@ public class HttpResponse implements SpiderResponse{
return response.contentType();
}
@Override
public void setCharset(String charset) {
this.response.charset(charset);
}
}

6
spider-flow-web/src/main/resources/static/resources/templates/request.html
View File

@ -40,6 +40,12 @@
<label class="layui-form-label">URL</label>
<div class="layui-input-block" codemirror="url" data-value="{{=d.data.object.url}}"></div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">页面编码</label>
<div class="layui-input-block">
<input type="text" name="response-charset" placeholder="请输入页面编码" autocomplete="off" class="layui-input input-default" value="{{=d.data.object['response-charset']}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">代理</label>
<div class="layui-input-block" placeholder="host:port" codemirror="proxy" data-value="{{=d.data.object.proxy}}"></div>