Add a mongdb data source that supports simple and aggregated queries.…

data-providers/document-data-provider/pom.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>datart-data-provider</artifactId>
+        <groupId>datart</groupId>
+        <version>1.0.0-rc.2</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>datart-document-data-provider</artifactId>
+
+    <packaging>jar</packaging>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>datart</groupId>
+            <artifactId>datart-data-provider-base</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>datart</groupId>
+            <artifactId>datart-jdbc-data-provider</artifactId>
+            <version>1.0.0-rc.2</version>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+    </build>
+
+</project>

data-providers/document-data-provider/src/main/java/datart/data/provider/DocumentDataProvider.java

@@ -0,0 +1,116 @@
+package datart.data.provider;
+
+import datart.core.base.exception.Exceptions;
+import datart.core.common.MessageResolver;
+import datart.core.data.provider.*;
+import datart.data.provider.base.DocumentClientFactory;
+import datart.data.provider.base.DocumentDbConfig;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.DigestUtils;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * query json command.
+ * e.g.
+ * {"find":"test","filter":{"age":{"$gte":1}},"limit":2, "skip":0,"sort":{"name":1}}
+ * {"aggregate":"test","pipeline":[{"$group":{"_id":null,"count":{"$sum":1}}}],"cursor":{}}
+ */
+@Slf4j
+public class DocumentDataProvider extends DataProvider {
+
+    private static final String I18N_PREFIX = "config.template.document.";
+
+    public static final String DB_TYPE = "dbType";
+
+    public static final String USER = "user";
+
+    public static final String PASSWORD = "password";
+
+    public static final String URL = "url";
+
+    public static final String DRIVER_CLASS = "driverClass";
+
+    @Override
+    public Object test(DataProviderSource source) throws Exception {
+        try {
+            String jsonCommand = "{\"find\":\"test\",\"limit\":1}}";
+            DocumentClientFactory.getClient(source).execute(jsonCommand);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+            Exceptions.e(e);
+        }
+        return true;
+    }
+
+    @Override
+    public Set<String> readAllDatabases(DataProviderSource source) throws SQLException {
+        return null;
+    }
+
+    @Override
+    public Set<String> readTables(DataProviderSource source, String database) throws SQLException {
+        return null;
+    }
+
+    @Override
+    public Set<Column> readTableColumns(DataProviderSource source, String schema, String table) throws SQLException {
+        return null;
+    }
+
+    @Override
+    public String getConfigDisplayName(String name) {
+        return MessageResolver.getMessage(I18N_PREFIX + name);
+    }
+
+    @Override
+    public String getConfigDescription(String name) {
+        return null;
+    }
+
+    @Override
+    public String getQueryKey(DataProviderSource config, QueryScript script, ExecuteParam executeParam) throws Exception {
+        return "Q" + DigestUtils.md5DigestAsHex(script.getScript().getBytes());
+    }
+
+    @Override
+    public Dataframe execute(DataProviderSource config, QueryScript script, ExecuteParam executeParam) throws Exception {
+        Dataframe execute = DocumentClientFactory.getClient(config).execute(script.getScript());
+        return execute;
+    }
+
+    @Override
+    public String getConfigFile() {
+        return "document-data-provider.json";
+    }
+
+    @Override
+    public boolean validateFunction(DataProviderSource source, String snippet) {
+        return false;
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    @Override
+    public DataProviderConfigTemplate getConfigTemplate() throws IOException {
+        DataProviderConfigTemplate configTemplate = super.getConfigTemplate();
+        for (DataProviderConfigTemplate.Attribute attribute : configTemplate.getAttributes()) {
+            attribute.setDisplayName(MessageResolver.getMessage("config.template.document." + attribute.getName()));
+            if (attribute.getName().equals("dbType")) {
+                List<Object> dbInfos = DocumentDbConfig.getInstance().getDocumentConfigInfos().entrySet().stream().map(item -> {
+                    Properties properties = new Properties();
+                    properties.setProperty(DB_TYPE, item.getValue().getDbType());
+                    properties.setProperty(URL, item.getValue().getUrlPrefix());
+                    return properties;
+                }).collect(Collectors.toList());
+                attribute.setOptions(dbInfos);
+            }
+        }
+        return configTemplate;
+    }
+}

data-providers/document-data-provider/src/main/java/datart/data/provider/base/DocumentClient.java

@@ -0,0 +1,9 @@
+package datart.data.provider.base;
+
+import datart.core.data.provider.Dataframe;
+
+public interface DocumentClient<T> {
+
+    public Dataframe execute(Object command);
+
+}

data-providers/document-data-provider/src/main/java/datart/data/provider/base/DocumentClientFactory.java

@@ -0,0 +1,27 @@
+package datart.data.provider.base;
+
+import datart.core.data.provider.DataProviderSource;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+public class DocumentClientFactory {
+
+    private static ConcurrentHashMap<String, DocumentClient> cache = new ConcurrentHashMap<>();
+
+    public static DocumentClient getClient(DataProviderSource source) {
+        String dbType = source.getProperties().get("dbType").toString().toUpperCase();
+        if (!cache.contains(dbType)) {
+            synchronized (DocumentClientFactory.class) {
+                if (!cache.contains(dbType)) {
+                    switch (dbType) {
+                        case "MONGODB":
+                            MongoDocumentClientImpl client = MongoDocumentClientImpl.builder().source(source)
+                                    .build().complete();
+                            cache.put(dbType, client);
+                    }
+                }
+            }
+        }
+        return cache.get(dbType);
+    }
+}

data-providers/document-data-provider/src/main/java/datart/data/provider/base/DocumentDbConfig.java

@@ -0,0 +1,67 @@
+package datart.data.provider.base;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.annotation.JSONField;
+import datart.core.base.exception.Exceptions;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * load document db config info.
+ * e.g. mongodb
+ */
+@Slf4j
+public class DocumentDbConfig {
+
+    private static ConcurrentHashMap<String, DocumentConfigInfo> cache = new ConcurrentHashMap<>();
+
+    private static DocumentDbConfig documentDbConfig;
+
+    private DocumentDbConfig() {
+        try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("document-db-driver.yml")) {
+            Yaml yaml = new Yaml();
+            Map<String, Map<String, String>> hashMap = yaml.loadAs(inputStream, HashMap.class);
+            hashMap.entrySet().stream().forEach(item -> {
+                DocumentConfigInfo info = JSON.parseObject(JSON.toJSONString(item.getValue()), DocumentConfigInfo.class);
+                cache.put(item.getKey(), info);
+            });
+        } catch (Exception e) {
+            Exceptions.e(e);
+        }
+    }
+
+    public static DocumentDbConfig getInstance() {
+        if (documentDbConfig == null) {
+            synchronized (DocumentDbConfig.class) {
+                if (documentDbConfig == null) {
+                    DocumentDbConfig.documentDbConfig = new DocumentDbConfig();
+                }
+            }
+        }
+        return DocumentDbConfig.documentDbConfig;
+    }
+
+    public DocumentConfigInfo getDocumentConfigInfo(String dbType) {
+        return cache.get(dbType);
+    }
+
+    public Map<String, DocumentConfigInfo> getDocumentConfigInfos() {
+        return new HashMap<>(cache);
+    }
+
+    @Data
+    public static class DocumentConfigInfo {
+        @JSONField(name = "db-type")
+        private String dbType;
+        private String name;
+        @JSONField(name = "url-prefix")
+        private String urlPrefix;
+    }
+
+}

data-providers/document-data-provider/src/main/java/datart/data/provider/base/MongoDocumentClientImpl.java

@@ -0,0 +1,75 @@
+package datart.data.provider.base;
+
+import datart.core.base.consts.ValueType;
+import datart.core.data.provider.Column;
+import datart.core.data.provider.DataProviderSource;
+import datart.core.data.provider.Dataframe;
+import lombok.Builder;
+import org.bson.Document;
+import org.bson.types.ObjectId;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.SimpleMongoClientDatabaseFactory;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Builder
+public class MongoDocumentClientImpl implements DocumentClient<MongoTemplate>{
+    private DataProviderSource source;
+    private MongoTemplate mongoTemplate;
+    private SimpleMongoClientDatabaseFactory simpleMongoClientDatabaseFactory;
+    public MongoDocumentClientImpl complete() {
+        simpleMongoClientDatabaseFactory =
+                new SimpleMongoClientDatabaseFactory(source.getProperties().get("url").toString());
+        mongoTemplate = new MongoTemplate(simpleMongoClientDatabaseFactory);
+        return this;
+    }
+    @Override
+    public Dataframe execute(Object command) {
+        Document document = mongoTemplate.executeCommand(command.toString());
+        Dataframe dataframe = new Dataframe();
+        List<Document> list = document.get("cursor", Document.class).getList("firstBatch", Document.class);
+
+        Map<String, Column> columnMap = new HashMap<>();
+        list.stream().forEach(item -> {
+            item.entrySet().forEach(kv -> {
+                if (!columnMap.containsKey(kv.getKey())) {
+                    if (kv.getValue() instanceof Date) {
+                        Column column = Column.of(ValueType.DATE, kv.getKey());
+                        columnMap.put(kv.getKey(), column);
+                    } else if (kv.getValue() instanceof Number) {
+                        Column column = Column.of(ValueType.NUMERIC, kv.getKey());
+                        columnMap.put(kv.getKey(), column);
+                    } else if (kv.getValue() instanceof Boolean) {
+                        Column column = Column.of(ValueType.BOOLEAN, kv.getKey());
+                        columnMap.put(kv.getKey(), column);
+                    } else {
+                        Column column = Column.of(ValueType.STRING, kv.getKey());
+                        columnMap.put(kv.getKey(), column);
+                    }
+                }
+            });
+        });
+        ArrayList<Column> columns = new ArrayList<>(columnMap.values());
+        List<List<Object>> rows = list.stream().map(item -> {
+            ArrayList<Object> row = new ArrayList<>();
+            columns.forEach(col -> {
+                Object value = item.get(col.getName()[0]);
+                if (value == null) {
+                    row.add(null);
+                } else if (value instanceof Document) {
+                    row.add(((Document) value).toJson());
+                } else if (value instanceof ObjectId) {
+                    row.add(value.toString());
+                } else {
+                    row.add(value);
+                }
+            });
+            return row;
+        }).collect(Collectors.toList());
+        dataframe.setColumns(columns);
+        dataframe.setRows(rows);
+        dataframe.setScript(command.toString());
+        return dataframe;
+    }
+}

data-providers/document-data-provider/src/main/resources/META-INF/services/datart.core.data.provider.DataProvider

@@ -0,0 +1 @@
+datart.data.provider.DocumentDataProvider

data-providers/document-data-provider/src/main/resources/application.properties

@@ -0,0 +1 @@
+

data-providers/document-data-provider/src/main/resources/document-data-provider.json

@@ -0,0 +1,25 @@
+{
+  "type": "DOCUMENT",
+  "name": "document-data-provider",
+  "attributes": [
+    {
+      "code": "dbType",
+      "name": "dbType",
+      "type": "string",
+      "required": true,
+      "defaultValue": ""
+    },
+    {
+      "name": "url",
+      "type": "string",
+      "required": true,
+      "defaultValue": ""
+    },
+    {
+      "name": "properties",
+      "type": "object",
+      "required": false,
+      "defaultValue": ""
+    }
+  ]
+}

data-providers/document-data-provider/src/main/resources/document-db-driver.yml

@@ -0,0 +1,4 @@
+MONGODB:
+  db-type: MONGODB
+  name: MONGODB
+  url-prefix: mongodb://

data-providers/pom.xml

@@ -16,6 +16,7 @@
         <module>jdbc-data-provider</module>
         <module>file-data-provider</module>
         <module>http-data-provider</module>
+        <module>document-data-provider</module>
     </modules>
 <!--    <dependencies>-->
 <!--        <dependency>-->
@@ -76,4 +77,4 @@
 <!--        </plugins>-->
 <!--    </build>-->
 
-</project>
+</project>