mongodb插件

This commit is contained in:
xuanxiaoli 2019-09-23 16:22:30 +08:00
parent 9dd4f5b974
commit 51059e4cb6
13 changed files with 647 additions and 0 deletions

View File

@ -38,5 +38,6 @@
<module>spider-flow-redis</module>
<module>spider-flow-oss</module>
<module>spider-flow-ocr</module>
<module>spider-flow-mongodb</module>
</modules>
</project>

View File

@ -0,0 +1,37 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.spiderflow</groupId>
<artifactId>spider-flow-mongodb</artifactId>
<version>0.0.1</version>
<name>spider-flow-mongodb</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.spiderflow</groupId>
<artifactId>spider-flow-api</artifactId>
<version>0.0.1</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.10.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-data-mongodb -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
<version>2.1.7.RELEASE</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,102 @@
package org.spiderflow.mongodb.executor;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spiderflow.context.SpiderContext;
import org.spiderflow.executor.ShapeExecutor;
import org.spiderflow.model.Shape;
import org.spiderflow.model.SpiderNode;
import org.spiderflow.mongodb.utils.MongoDBUtils;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.util.JSON;
/**
* 命令
* @author xuan
*
*/
@Component
public class MongoDBCommandExecutor implements ShapeExecutor {
private static Logger logger = LoggerFactory.getLogger(MongoDBCommandExecutor.class);
public static final String DATASOURCE_ID = "datasourceId";
public static final String MONGODB_SQL = "sql";
public static final String MONGODB_OPERATION_TYPE = "operationType";
public static final String MONGODB_COMMAND_VAR = "___mongoDB";
public static final String OPERATION_INSERT = "insert";
public static final String OPERATION_DELETE = "delete";
public static final String OPERATION_UPDATE = "update";
public static final String OPERATION_SELECT = "select";
DBCollection dbCollection = null;
@Override
public String supportShape() {
// TODO Auto-generated method stub
return "mongodbcommand";
}
@Override
public Shape shape() {
Shape shape = new Shape();
shape.setImage("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACkAAAAgCAYAAACPb1E+AAAC9UlEQVRYR+2YP0zUUBjAf9+RKJPgQK6THptxERPiZOTUxEUj/tlMjCS62xoT3YARl5bJTXByBGcHISQmxhjQ6A6TPRg8JrnE62de6Z0Htne9XhMw4W1N3/u+X7//rwJgzeokygRQMs+HYSksBsr0liNrYrk6gTB3GMBiGNZ9W4al6OqiCOOqfFGYKkD1oIEVRkRwQw5l2FhyCWEMZdp3ZOqgARv6LU/1CDIPbxxZMg8rhmUxj5gcdHWw6khiJSh6+lVgAGXMd2S90/79H9czpOXpd+CsKk7FEe8fBab2whwCdeF6n/ISOFUPuLT1RFbSWLsnSMvTOVUmREJVM74tz1uVGov1C6tR9/rs2zJqefoL6FfYDpSy6SKdQDNDGkAI2yeqvKk4ci/GilMIk40aZ1w95OpIQVgWOKFQDZTLnUAzQbYCorz2HQlhW5flakmFVYFBVWYrjtiN9yEoLIkwkAa0a0jL0x8m4SLrxAKGGenqPMIDVbZrUNqfWE1Qk1AFqAc83HLkVZzrs0AGgKhQDwJG41xluVpGeB99SGKLtTx9i3LTJFUAC5u23MkFcsjTZwWYFjie5CrLUwNYRtnwHYkd+fbENHyr/eZG9als5AJphLSLqdZxL1BubzqyGFP3mkmXFNN74jtrMW8FRdgB7u4EfGiWHGXZd6QcA7iCcBEz1yQkXa7F3ID2wUeEY0HATJ9wRoVxo6SunI+N18gqCu8qtlzrVCNzaYtFVx8BVwJ4UYB5Ec4pfKrYciEOoOiqjXC1ptxv10pzcXccQBQC5VqdhaQkSGO5XN2dRWGWM13XySxKej1zBNmrBRvnYy25fyDIS1kWOdG49zNqseG92xPhcSRsqWuhgoXSn3jOFH3F71Kuaatha91RTpp7d0lhzYxQXQra3W66yO4AHL86vW+nNOpOofgI9JaZAzOAmt80p9tAmgFivlu5Aaw1ZoB2Nkglt/kHJNmSsf08lfBoU2pIy9W/14JuNCTs9e3olpRC1n8B+Qd0Dhp9ddQMugAAAABJRU5ErkJggg==");
shape.setLabel("mongodbcommand");
shape.setName("mongodbcommand");
shape.setTitle("mongoDB执行语句");
return shape;
}
@Override
public void execute(SpiderNode node, SpiderContext context, Map<String, Object> variables) {
String datasourceId = node.getStringJsonValue(DATASOURCE_ID);
String sql = node.getStringJsonValue(MONGODB_SQL);
String operationType = node.getStringJsonValue(MONGODB_OPERATION_TYPE);
if(StringUtils.isBlank(datasourceId)){
context.debug("mongodb数据源ID为空");
}else if(StringUtils.isBlank(sql)){
context.debug("mongodb命令为空");
} else if(StringUtils.isBlank(operationType)){
context.debug("mongodb命令类型为空");
} else {
dbCollection = (DBCollection) context.get(MongoDBExecutor.MONGODB_CONTEXT_KEY + datasourceId);
DBCursor db = mongoDBOperation(sql,operationType);
MongoDBUtils.analysisJSON(variables,db);
}
}
public DBCursor mongoDBOperation(String sql,String operateType){
DBObject user_json = null;
if(!operateType.equals(OPERATION_UPDATE))
user_json = (DBObject)JSON.parse(sql);
switch(operateType){
case OPERATION_INSERT :
dbCollection.insert(user_json);
break;
case OPERATION_DELETE :
dbCollection.remove(user_json);
break;
case OPERATION_UPDATE :
String[] strArr = sql.split("\\}\\,\\{");
dbCollection.update((DBObject)JSON.parse(strArr[0]+"}"),(DBObject)JSON.parse("{"+strArr[1]));
break;
case OPERATION_SELECT :
return dbCollection.find(user_json);
}
return null;
}
}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,50 @@
package org.spiderflow.mongodb.executor;
import java.util.ArrayList;
import java.util.List;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoBulkWriteException;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.WriteResult;
import com.mongodb.client.MongoDatabase;
import com.mongodb.util.JSON;
public class MongoDBJDBC{
static DB mongoConn = null;
public static void main( String args[] ){
try{
List<MongoCredential> credentials = new ArrayList<MongoCredential>();
//连接到MongoDB服务 如果是远程连接可以替换localhost为服务器所在IP地址
//ServerAddress()两个参数分别为 服务器地址 端口
ServerAddress serverAddress = new ServerAddress("localhost",27017);
List<ServerAddress> addrs = new ArrayList<ServerAddress>();
addrs.add(serverAddress);
//MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码 xuanmi
MongoCredential credential = MongoCredential.createScramSha1Credential("admin", "admin", "xuanmi123".toCharArray());
credentials.add(credential);
//通过连接认证获取MongoDB连接
MongoClient mongoClient = new MongoClient(addrs,credentials);
DB db = mongoClient.getDB("xuanxiaoli");
DBCollection con = db.getCollection("xuanxiaoli");
//查询条件sql
@SuppressWarnings("deprecation")//{"name":"C1"}
DBObject user_json = (DBObject)JSON.parse("{\"name\":\"AA12\"}");//"db.xuanxiaoli.insert({name:'C2'})"
//WriteResult insert = con.insert(user_json);
DBCursor query = con.find(user_json);
while(query.hasNext()){
System.out.println(query.next().toString());
}
System.out.println("Connect to database successfully");
} catch (MongoBulkWriteException e){
System.err.println( e.getClass().getName() + ": " + e.getMessage() );
} finally {
}
}
}

View File

@ -0,0 +1,62 @@
package org.spiderflow.mongodb.model;
public class MongoDto {
private static String userName;//用户名
private static String pwd;//密码
private static String[] host;//主机地址
private static int[] port;//端口地址
private static String dbName;//数据库名
private static int connectionsPerHost = 20;//每台主机最大连接数
private static int threadsAllowedToBlockForConnectionMultiplier = 10;//线程队列数
private static boolean authentication = false;//是否需要身份验证
public static String getUserName() {
return userName;
}
public static void setUserName(String userName) {
MongoDto.userName = userName;
}
public static String getPwd() {
return pwd;
}
public static void setPwd(String pwd) {
MongoDto.pwd = pwd;
}
public static String[] getHost() {
return host;
}
public static void setHost(String[] host) {
MongoDto.host = host;
}
public static int[] getPort() {
return port;
}
public static void setPort(int[] port) {
MongoDto.port = port;
}
public static String getDbName() {
return dbName;
}
public static void setDbName(String dbName) {
MongoDto.dbName = dbName;
}
public static int getConnectionsPerHost() {
return connectionsPerHost;
}
public static void setConnectionsPerHost(int connectionsPerHost) {
MongoDto.connectionsPerHost = connectionsPerHost;
}
public static int getThreadsAllowedToBlockForConnectionMultiplier() {
return threadsAllowedToBlockForConnectionMultiplier;
}
public static void setThreadsAllowedToBlockForConnectionMultiplier(
int threadsAllowedToBlockForConnectionMultiplier) {
MongoDto.threadsAllowedToBlockForConnectionMultiplier = threadsAllowedToBlockForConnectionMultiplier;
}
public static boolean isAuthentication() {
return authentication;
}
public static void setAuthentication(boolean authentication) {
MongoDto.authentication = authentication;
}
}

View File

@ -0,0 +1,28 @@
package org.spiderflow.mongodb.utils;
import java.util.Map;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
public class FastJsonUtil {
public static String ObjectToString(Object o) {
String s = JSON.toJSONString(o);
return TrimDoubleQuote(s);
}
private static String TrimDoubleQuote(String s) {
if (s.startsWith("\"") && s.endsWith("\"")) {
s = s.substring(1, s.length());
s = s.substring(0, s.length() - 1);
}
return s;
}
public static void main(String[] args) {
String str = "{\"_id\": {\"$oid\": \"5d6f5e1a76ab0c15d8901786\"}, \"name\": \"BB123\", \"age\": \"20\"}";
JSONObject jsonObject1=JSON.parseObject(str);//将json字符串转换为json对象
for (Map.Entry<String, Object> entry : jsonObject1.entrySet()) {
System.out.println(entry.getKey() + ":" + entry.getValue());
}
}
}

View File

@ -0,0 +1,56 @@
package org.spiderflow.mongodb.utils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
public class MongoDBUtils {
public static DBCollection createMongoDBTemplate(String host,int port,String database,String table,String adminName,String password){
List<MongoCredential> credentials = new ArrayList<MongoCredential>();
ServerAddress serverAddress = new ServerAddress(host,port);
List<ServerAddress> addrs = new ArrayList<ServerAddress>();
addrs.add(serverAddress);
//MongoCredential.createScramSha1Credential()
MongoCredential credential = MongoCredential.createScramSha1Credential(adminName, database, password.toCharArray());
credentials.add(credential);
//通过连接认证获取MongoDB连接
MongoClient mongoClient = new MongoClient(addrs,credentials);
DB db = mongoClient.getDB(table);
DBCollection con = db.getCollection(table);
return con;
/*//连接到数据库
@SuppressWarnings("deprecation")
DBObject user_json = (DBObject)JSON.parse("{'$or' : [{\"name\":\"AA12\"},{\"name\":\"BB123\"}]}");//,\"age\":\"20\"
//con.insert(user_json);
DBCursor query = con.find(user_json);*/
}
public static void analysisJSON(Map<String, Object> variables,DBCursor db){
while(db.hasNext()){
JSONObject jsonObject1=JSON.parseObject(db.next().toString());//将json字符串转换为json对象
for (Map.Entry<String, Object> entry : jsonObject1.entrySet()) {
System.out.println(entry.getKey()+"------"+entry.getValue());
if(variables.containsKey(entry.getKey())){
List<String> rsStr = (List<String>) variables.get(entry.getKey());
rsStr.add(entry.getValue().toString());
variables.put(entry.getKey(),rsStr);
} else {
List<String> str = new ArrayList<String>();
str.add(entry.getValue().toString());
variables.put(entry.getKey(), str);
}
}
}
}
}

View File

@ -0,0 +1,78 @@
package org.spiderflow.mongodb.web;
import java.util.ArrayList;
import java.util.List;
import org.bson.Document;
import org.spiderflow.model.JsonBean;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.mongodb.MongoBulkWriteException;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
@RestController
@RequestMapping("/mongo")
public class MongoDBController {
/**
*
* @param host 地址
* @param port 端口
* @param adminName 用户名
* @param database 数据库名
* @param password 密码
* @return
*/
@RequestMapping("/connTest")
public JsonBean<String> test(String host,Integer port,String adminName,String table,String database,String password){
MongoClient mongoClient = new MongoClient();
try{
List<MongoCredential> credentials = new ArrayList<MongoCredential>();
ServerAddress serverAddress = new ServerAddress(host,port);
List<ServerAddress> addrs = new ArrayList<ServerAddress>();
addrs.add(serverAddress);
//MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码
MongoCredential credential = MongoCredential.createScramSha1Credential(adminName, database, password.toCharArray());
credentials.add(credential);
MongoClientOptions options = setBuild();
//通过连接认证获取MongoDB连接
mongoClient = new MongoClient(addrs,credentials,options);
//连接到数据库
MongoDatabase mongoDatabase = mongoClient.getDatabase(table);
//数据传输查询是否连接成功
MongoCollection<Document> doc = mongoDatabase.getCollection(table);
try {
doc.count() ;
} catch (Exception e) {
return new JsonBean<String>(0,e.getMessage());
}
return new JsonBean<String>(1,"测试成功");
} catch (MongoBulkWriteException e){
return new JsonBean<String>(0,e.getMessage());
} finally {
mongoClient.close();
}
}
public MongoClientOptions setBuild(){
MongoClientOptions.Builder build = new MongoClientOptions.Builder();
build.connectionsPerHost(1000);
build.threadsAllowedToBlockForConnectionMultiplier(20);
build.connectTimeout(1000);
build.maxWaitTime(1000);
build.socketTimeout(1000);
return build.build();
}
}

View File

@ -0,0 +1,13 @@
#控制系统在发生连接错误时是否重试 默以为false --boolean
mongo.options.autoConnectRetry=false
#每个主机答应的连接数(每个主机的连接池大小),当连接池被用光时,会被阻塞住 默以为10 --int
mongo.options.connectionsPerHost=10
#multiplier for connectionsPerHost for # of threads that can block if connectionsPerHost is 10, and threadsAllowedToBlockForConnectionMultiplier is 5, then 50 threads can block more than that and an exception will be throw --int
mongo.options.threadsAllowedToBlockForConnectionMultiplier=5
#被阻塞线程从连接池获取连接的最长等待时间ms --int
mongo.options.maxWaitTime =1000
#在建立打开套接字连接时的超时时间ms默以为0无穷 --int
mongo.options.connectTimeout="100"
#套接字超时时间;该值会被传递给Socket.setSoTimeout(int)。默以为0无穷 --int
mongo.options.socketTimeout="100"
server.selection.timeout="100"

View File

@ -0,0 +1,94 @@
<div class="layui-tab layui-tab-fixed layui-tab-brief">
<ul class="layui-tab-title">
<li class="layui-this">配置</li>
</ul>
<div class="layui-tab-content">
<div class="layui-tab-item layui-show">
<form class="layui-form editor-form-node">
<div class="layui-form-item">
<label class="layui-form-label">节点名称</label>
<div class="layui-input-block">
<input type="text" name="value" placeholder="请输入节点名称" value="{{=d.value}}" autocomplete="off" class="layui-input">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">IP地址</label>
<div class="layui-input-block">
<input type="text" name="host" placeholder="请输入IP地址" autocomplete="off" class="layui-input" value="{{=d.data.object.host}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">端口</label>
<div class="layui-input-block">
<input type="number" name="port" placeholder="请输入端口" autocomplete="off" class="layui-input" value="{{=d.data.object.port}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">数据库名</label>
<div class="layui-input-block">
<input type="text" name="database" placeholder="请输入数据库名称" autocomplete="off" class="layui-input" value="{{=d.data.object.database}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label"></label>
<div class="layui-input-block">
<input type="text" name="table" placeholder="请输入表名称" autocomplete="off" class="layui-input" value="{{=d.data.object.table}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">用户名</label>
<div class="layui-input-block">
<input type="text" name="adminName" placeholder="请输入数据库用户名" autocomplete="off" class="layui-input" value="{{=d.data.object.adminName}}">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">密码</label>
<div class="layui-input-block">
<input type="password" name="password" placeholder="请输入数据库密码" autocomplete="off" class="layui-input" value="{{=d.data.object.password}}">
</div>
</div>
<div class="layui-form-item">
<div class="layui-input-block">
<button class="layui-btn btn-mongodb-datasource-test" type="button">测试连接</button>
</div>
</div>
</form>
</div>
</div>
</div>
<script>
$('.layui-form').on('click','.btn-mongodb-datasource-test',function(){
var $form = $('.layui-form');
var host = $form.find('input[name=host]').val();
var port = $form.find('input[name=port]').val();
var database = $form.find('input[name=database]').val();
var table = $form.find('input[name=table]').val();
var adminName = $form.find('input[name=adminName]').val();
var password = $form.find('input[name=password]').val();
$.ajax({
url : 'mongo/connTest',
data : {
host : host || 'localhost',
port : port || 27017,
database : database,
table : table,
adminName : adminName,
password : password
},
type : 'post',
dataType : 'json',
success : function(json){
if(json.code == 1){
layui.layer.msg('测试成功');
}else{
layui.layer.alert('测试失败,' + json.message || '',{
icon : 2
})
}
},
error : function(){
layui.layer.msg('测试失败');
}
})
});
</script>

View File

@ -0,0 +1,68 @@
<div class="layui-tab layui-tab-fixed layui-tab-brief">
<ul class="layui-tab-title">
<li class="layui-this">配置</li>
</ul>
<div class="layui-tab-content">
<div class="layui-tab-item layui-show">
<form class="layui-form editor-form-node">
<div class="layui-form-item">
<label class="layui-form-label">节点名称</label>
<div class="layui-input-block">
<input type="text" name="value" placeholder="请输入节点名称" value="{{=d.value}}" autocomplete="off" class="layui-input">
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">数据源</label>
<div class="layui-input-block">
<select name="datasourceId">
{{# for(var datasourceIndex in d.model.cells){ }}
{{# var cell = d.model.cells[datasourceIndex] }}
{{# if(cell.data&&cell.data.get('shape') == 'mongodb'){ }}
<option value="{{=datasourceIndex}}" {{datasourceIndex == d.data.object.datasourceId ? 'selected': ''}}>{{cell.value}}</option>
{{# } }}
{{# } }}
</select>
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">语句类型</label>
<div class="layui-input-block">
<select name="operationType" lay-filter="otf">
<option value="select" {{d.data.object.operationType == 'select' ? 'selected':''}}>select</option>
<option value="insert" {{d.data.object.operationType == 'insert' ? 'selected':''}}>insert</option>
<option value="delete" {{d.data.object.operationType == 'delete' ? 'selected':''}}>delete</option>
<option value="update" {{d.data.object.operationType == 'update' ? 'selected':''}}>update</option>
</select>
</div>
</div>
<div class="layui-form-item">
<label class="layui-form-label">SQL语句</label>
<textarea class="layui-input" style="height:200px" name="sql">{{=d.data.object.sql}}</textarea>
</div>
<!-- <div class="update" style="display:none;">
<div class="layui-form-item" >
<label class="layui-form-label">SQL语句</label>
<textarea class="layui-input" style="height:80px" name="selectsql">{{=d.data.object.selectsql}}</textarea>
</div>
<div class="layui-form-item" >
<label class="layui-form-label">SQL语句</label>
<textarea class="layui-input" style="height:80px" name="updatesql">{{=d.data.object.updatesql}}</textarea>
</div>
</div> -->
</form>
</div>
</div>
</div>
<script>
/* $(function(){
var operationType = ".editor-form-node [name=operationType]";
layui.form.on('select(otf)',function(data){
$(".noup").show();
$(".update").hide();
if(data.value === "update"){
$(".noup").hide();
$("."+data.value).show();
}
})
}) */
</script>

View File

@ -47,6 +47,11 @@
<artifactId>spider-flow-ocr</artifactId>
<version>0.0.1</version>
</dependency>
<dependency>
<groupId>org.spiderflow</groupId>
<artifactId>spider-flow-mongodb</artifactId>
<version>0.0.1</version>
</dependency>
</dependencies>
<build>
<plugins>