diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java index d7ce6c3e5a..3ba2f02ef0 100644 --- a/backend/src/main/java/io/dataease/config/CommonConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -39,7 +39,7 @@ public class CommonConfig { datasource.setId("doris"); datasource.setName("doris"); datasource.setDesc("doris"); - datasource.setType("mysql"); + datasource.setType("doris"); datasource.setConfiguration(jsonObject.toJSONString()); return datasource; } diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index 5b704e2a5b..51df39c55e 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -1,5 +1,5 @@ package io.dataease.datasource.constants; public enum DatasourceTypes { - mysql, sqlServer, excel + mysql, sqlServer, excel, doris } diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index d1d03efa9f..e2306f3d3f 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -20,6 +20,7 @@ public class JdbcProvider extends DatasourceProvider { private static Map jdbcConnection = new HashMap<>(); private static int initPoolSize = 5; private static int maxConnections = 200; + @Override public List getData(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); @@ -33,7 +34,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } return list; @@ -50,7 +51,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } } @@ -68,7 +69,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } } @@ -108,7 +109,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } } @@ -133,7 +134,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } } @@ -173,7 +174,7 @@ public class JdbcProvider extends DatasourceProvider { return tables; } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); - }finally { + } finally { con.close(); } } @@ -209,7 +210,7 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); - }finally { + } finally { connection.close(); } return list; @@ -227,7 +228,7 @@ public class JdbcProvider extends DatasourceProvider { ps.close(); } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); - }finally { + } finally { con.close(); } } @@ -236,20 +237,21 @@ public class JdbcProvider extends DatasourceProvider { public Long count(DatasourceRequest datasourceRequest) throws Exception { Connection con = null; try { - con = getConnectionFromPool(datasourceRequest); Statement ps = con.createStatement(); + con = getConnectionFromPool(datasourceRequest); + Statement ps = con.createStatement(); ResultSet resultSet = ps.executeQuery(datasourceRequest.getQuery()); while (resultSet.next()) { return resultSet.getLong(1); } } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); - }finally { + } finally { con.close(); } return 0L; } - private Connection getConnectionFromPool(DatasourceRequest datasourceRequest)throws Exception { + private Connection getConnectionFromPool(DatasourceRequest datasourceRequest) throws Exception { ComboPooledDataSource dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId()); if (dataSource == null) { initDataSource(datasourceRequest); @@ -260,7 +262,7 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public void initDataSource(DatasourceRequest datasourceRequest)throws Exception{ + public void initDataSource(DatasourceRequest datasourceRequest) throws Exception { ComboPooledDataSource dataSource = jdbcConnection.get(datasourceRequest.getDatasource().getId()); if (dataSource == null) { dataSource = new ComboPooledDataSource(); @@ -298,6 +300,13 @@ public class JdbcProvider extends DatasourceProvider { driver = mysqlConfigration.getDriver(); jdbcurl = mysqlConfigration.getJdbc(); break; + case doris: + MysqlConfigration dorisConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); + username = dorisConfigration.getUsername(); + password = dorisConfigration.getPassword(); + driver = dorisConfigration.getDriver(); + jdbcurl = dorisConfigration.getJdbc(); + break; case sqlServer: SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); username = sqlServerConfigration.getUsername(); @@ -319,26 +328,33 @@ public class JdbcProvider extends DatasourceProvider { } - private void setCredential(DatasourceRequest datasourceRequest, ComboPooledDataSource dataSource) throws PropertyVetoException { - DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); - switch (datasourceType) { - case mysql: - MysqlConfigration mysqlConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); - dataSource.setUser(mysqlConfigration.getUsername()); - dataSource.setDriverClass(mysqlConfigration.getDriver()); - dataSource.setPassword(mysqlConfigration.getPassword()); - dataSource.setJdbcUrl(mysqlConfigration.getJdbc()); - break; - case sqlServer: - SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); - dataSource.setUser(sqlServerConfigration.getUsername()); - dataSource.setDriverClass(sqlServerConfigration.getDriver()); - dataSource.setPassword(sqlServerConfigration.getPassword()); - dataSource.setJdbcUrl(sqlServerConfigration.getJdbc()); - break; - default: - break; - } + private void setCredential(DatasourceRequest datasourceRequest, ComboPooledDataSource dataSource) throws PropertyVetoException { + DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); + switch (datasourceType) { + case mysql: + MysqlConfigration mysqlConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); + dataSource.setUser(mysqlConfigration.getUsername()); + dataSource.setDriverClass(mysqlConfigration.getDriver()); + dataSource.setPassword(mysqlConfigration.getPassword()); + dataSource.setJdbcUrl(mysqlConfigration.getJdbc()); + break; + case doris: + MysqlConfigration dorisConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); + dataSource.setUser(dorisConfigration.getUsername()); + dataSource.setDriverClass(dorisConfigration.getDriver()); + dataSource.setPassword(dorisConfigration.getPassword()); + dataSource.setJdbcUrl(dorisConfigration.getJdbc()); + break; + case sqlServer: + SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); + dataSource.setUser(sqlServerConfigration.getUsername()); + dataSource.setDriverClass(sqlServerConfigration.getDriver()); + dataSource.setPassword(sqlServerConfigration.getPassword()); + dataSource.setJdbcUrl(sqlServerConfigration.getJdbc()); + break; + default: + break; + } } private String getDatabase(DatasourceRequest datasourceRequest) { @@ -347,6 +363,9 @@ public class JdbcProvider extends DatasourceProvider { case mysql: MysqlConfigration mysqlConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); return mysqlConfigration.getDataBase(); + case doris: + MysqlConfigration dorisConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigration.class); + return dorisConfigration.getDataBase(); case sqlServer: SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); return sqlServerConfigration.getDataBase(); @@ -360,6 +379,8 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: return "show tables;"; + case doris: + return "show tables;"; case sqlServer: return "SELECT TABLE_NAME FROM fit2cloud2.INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE';"; default: diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index 244a3ccde3..72f68b631d 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -1,13 +1,14 @@ package io.dataease.datasource.provider; import io.dataease.datasource.constants.DatasourceTypes; +import io.dataease.provider.DDLProvider; +import io.dataease.provider.QueryProvider; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; - @Component public class ProviderFactory implements ApplicationContextAware { @@ -18,18 +19,46 @@ public class ProviderFactory implements ApplicationContextAware { this.context = applicationContext; } - public static DatasourceProvider getProvider(String type){ + public static DatasourceProvider getProvider(String type) { DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); - switch (datasourceType){ + switch (datasourceType) { case mysql: return context.getBean("jdbc", DatasourceProvider.class); + case doris: + return context.getBean("jdbc", DatasourceProvider.class); case sqlServer: return context.getBean("jdbc", DatasourceProvider.class); default: return context.getBean("jdbc", DatasourceProvider.class); } - } + public static QueryProvider getQueryProvider(String type) { + DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); + switch (datasourceType) { + case mysql: + return context.getBean("mysqlQuery", QueryProvider.class); + case doris: + return context.getBean("dorisQuery", QueryProvider.class); + case sqlServer: + return context.getBean("mysqlQuery", QueryProvider.class); + default: + return context.getBean("mysqlQuery", QueryProvider.class); + } + } + + public static DDLProvider getDDLProvider(String type) { + DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); + switch (datasourceType) { + case mysql: + return context.getBean("mysqlDDL", DDLProvider.class); + case doris: + return context.getBean("dorisDDL", DDLProvider.class); + case sqlServer: + return context.getBean("mysqlDDL", DDLProvider.class); + default: + return context.getBean("mysqlDDL", DDLProvider.class); + } + } } diff --git a/backend/src/main/java/io/dataease/provider/DDLProvider.java b/backend/src/main/java/io/dataease/provider/DDLProvider.java new file mode 100644 index 0000000000..88db6f9871 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/DDLProvider.java @@ -0,0 +1,11 @@ +package io.dataease.provider; + +/** + * @Author gin + * @Date 2021/5/17 4:19 下午 + */ +public abstract class DDLProvider { + public abstract String createView(String name, String viewSQL); + + public abstract String dropTableOrView(String name); +} diff --git a/backend/src/main/java/io/dataease/provider/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java new file mode 100644 index 0000000000..bf05fecd1f --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java @@ -0,0 +1,33 @@ +package io.dataease.provider; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartViewFieldDTO; + +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 2:42 下午 + */ +public abstract class QueryProvider { + public abstract Integer transFieldType( String field); + + public abstract String createQueryCountSQL(String table); + + public abstract String createQueryCountSQLAsTmp(String sql); + + public abstract String createSQLPreview(String sql, String orderBy); + + public abstract String createQuerySQL(String table, List fields); + + public abstract String createQuerySQLAsTmp(String sql, List fields); + + public abstract String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize); + + public abstract String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize); + + public abstract String getSQL(String table, List xAxis, List yAxis, List extFilterRequestList); + + public abstract String getSQLAsTmp(String table, List xAxis, List yAxis, List extFilterRequestList); +} diff --git a/backend/src/main/java/io/dataease/provider/doris/DorisDDLProvider.java b/backend/src/main/java/io/dataease/provider/doris/DorisDDLProvider.java new file mode 100644 index 0000000000..a3448b6079 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/doris/DorisDDLProvider.java @@ -0,0 +1,21 @@ +package io.dataease.provider.doris; + +import io.dataease.provider.DDLProvider; +import org.springframework.stereotype.Service; + +/** + * @Author gin + * @Date 2021/5/17 4:27 下午 + */ +@Service("dorisDDL") +public class DorisDDLProvider extends DDLProvider { + @Override + public String createView(String name, String viewSQL) { + return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; + } + + @Override + public String dropTableOrView(String name) { + return "DROP TABLE IF EXISTS " + name; + } +} diff --git a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java new file mode 100644 index 0000000000..e5ab95407b --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java @@ -0,0 +1,289 @@ +package io.dataease.provider.doris; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.provider.QueryProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 4:11 下午 + */ +@Service("dorisQuery") +public class DorisQueryProvider extends QueryProvider { + @Override + public Integer transFieldType(String field) { + switch (field) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "ENUM": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createQueryCountSQL(String table) { + return MessageFormat.format("SELECT count(*) FROM {0}", table); + } + + @Override + public String createQueryCountSQLAsTmp(String sql) { + return createQueryCountSQL(" (" + sql + ") AS tmp "); + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sql + ") AS tmp ORDER BY " + orderBy + " LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } else { + if (f.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields) { + return createQuerySQL(" (" + sql + ") AS tmp ", fields); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQL(table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List extFilterRequestList) { + // 字段汇总 排序等 + String[] field = yAxis.stream().map(y -> { + StringBuilder f = new StringBuilder(); + if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { + f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")"); + } else { + f.append("ROUND(") + .append(y.getSummary()).append("(") + .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append("),2").append(")"); + } + f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + return f.toString(); + }).toArray(String[]::new); + String[] groupField = xAxis.stream().map(x -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } else { + if (x.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new); + String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new); + String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new); + String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); + System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); + + String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) + .map(x -> { + String[] s = x.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(x.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(x.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'" + f.getValue() + "'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + + String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", + StringUtils.join(groupField, ","), + StringUtils.join(field, ","), + table, + xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter + StringUtils.join(group, ","), + StringUtils.join(order, ",")); + if (sql.endsWith(",")) { + sql = sql.substring(0, sql.length() - 1); + } + // 如果是对结果字段过滤，则再包裹一层sql + String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) + .map(y -> { + String[] s = y.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + if (y.getDeType() == 1 && y.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(_") + .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'" + f.getValue() + "'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + if (resultFilter.length == 0) { + return sql; + } else { + String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}", + "(" + sql + ") AS tmp", + StringUtils.join(resultFilter, " ")); + return filterSql; + } + } + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List extFilterRequestList) { + return getSQL(" (" + sql + ") AS tmp ", xAxis, yAxis, extFilterRequestList); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + default: + return ""; + } + } + + public String transMysqlExtFilter(List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return ""; + } + StringBuilder filter = new StringBuilder(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + if (CollectionUtils.isEmpty(value)) { + continue; + } + DatasetTableField field = request.getDatasetTableField(); + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(field.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(field.getDataeaseName()); + } + filter.append(" ") + .append(transMysqlFilterTerm(request.getOperator())) + .append(" "); + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + filter.append("('").append(StringUtils.join(value, "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + filter.append("'%").append(value.get(0)).append("%'"); + } else { + filter.append("'").append(value.get(0)).append("'"); + } + } + return filter.toString(); + } +} diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlDDLProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlDDLProvider.java new file mode 100644 index 0000000000..5f51f5459f --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlDDLProvider.java @@ -0,0 +1,21 @@ +package io.dataease.provider.mysql; + +import io.dataease.provider.DDLProvider; +import org.springframework.stereotype.Service; + +/** + * @Author gin + * @Date 2021/5/17 4:27 下午 + */ +@Service("mysqlDDL") +public class MysqlDDLProvider extends DDLProvider { + @Override + public String createView(String name, String viewSQL) { + return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")"; + } + + @Override + public String dropTableOrView(String name) { + return "DROP TABLE IF EXISTS " + name; + } +} diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java new file mode 100644 index 0000000000..f59597becb --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java @@ -0,0 +1,296 @@ +package io.dataease.provider.mysql; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.provider.QueryProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/17 2:43 下午 + */ +@Service("mysqlQuery") +public class MysqlQueryProvider extends QueryProvider { + @Override + public Integer transFieldType(String field) { + switch (field) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "ENUM": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createQueryCountSQL(String table) { + return MessageFormat.format("SELECT count(*) FROM {0}", table); + } + + @Override + public String createQueryCountSQLAsTmp(String sql) { + return createQueryCountSQL(" (" + sql + ") AS tmp "); + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sql + ") AS tmp ORDER BY " + orderBy + " LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } else { + if (f.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName()); + } else { + stringBuilder.append(f.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields) { + return createQuerySQL(" (" + sql + ") AS tmp ", fields); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQL(table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize) { + return createQuerySQLAsTmp(sql, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List extFilterRequestList) { + // 字段汇总 排序等 + String[] field = yAxis.stream().map(y -> { + StringBuilder f = new StringBuilder(); + if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { + f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")"); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + f.append("CAST(") + .append(y.getSummary()).append("(") + .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(") AS DECIMAL(20,2)").append(")"); + } else { + f.append(y.getSummary()).append("(") + .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") + .append(")"); + } + } + f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + return f.toString(); + }).toArray(String[]::new); + String[] groupField = xAxis.stream().map(x -> { + StringBuilder stringBuilder = new StringBuilder(); + // 如果原始类型为时间 + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } else { + if (x.getDeType() == 1) { + stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName()); + } else { + stringBuilder.append(x.getDataeaseName()); + } + } + return stringBuilder.toString(); + }).toArray(String[]::new); + String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new); + String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new); + String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) + .map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new); + String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); + System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); + + String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) + .map(x -> { + String[] s = x.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(x.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(x.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + + String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", + StringUtils.join(groupField, ","), + StringUtils.join(field, ","), + table, + xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter + StringUtils.join(group, ","), + StringUtils.join(order, ",")); + if (sql.endsWith(",")) { + sql = sql.substring(0, sql.length() - 1); + } + // 如果是对结果字段过滤，则再包裹一层sql + String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) + .map(y -> { + String[] s = y.getFilter().stream().map(f -> { + StringBuilder filter = new StringBuilder(); + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (y.getDeType() == 1 && y.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(_") + .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); + } + filter.append(transMysqlFilterTerm(f.getTerm())); + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + filter.append("%").append(f.getValue()).append("%"); + } else { + filter.append("'").append(f.getValue()).append("'"); + } + return filter.toString(); + }).toArray(String[]::new); + return StringUtils.join(s, " "); + }).toArray(String[]::new); + if (resultFilter.length == 0) { + return sql; + } else { + String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}", + "(" + sql + ") AS tmp", + StringUtils.join(resultFilter, " ")); + return filterSql; + } + } + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List extFilterRequestList) { + return getSQL(" (" + sql + ") AS tmp ", xAxis, yAxis, extFilterRequestList); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + default: + return ""; + } + } + + public String transMysqlExtFilter(List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return ""; + } + StringBuilder filter = new StringBuilder(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + if (CollectionUtils.isEmpty(value)) { + continue; + } + DatasetTableField field = request.getDatasetTableField(); + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + filter.append(" AND FROM_UNIXTIME(cast(") + .append(field.getDataeaseName()) + .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + } else { + filter.append(" AND ").append(field.getDataeaseName()); + } + filter.append(" ") + .append(transMysqlFilterTerm(request.getOperator())) + .append(" "); + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + filter.append("('").append(StringUtils.join(value, "','")).append("')"); + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + filter.append("'%").append(value.get(0)).append("%'"); + } else { + filter.append("'").append(value.get(0)).append("'"); + } + } + return filter.toString(); + } +} diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index ef7b44a338..567b4b1239 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -1,6 +1,5 @@ package io.dataease.service.chart; -import com.alibaba.fastjson.JSONObject; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import io.dataease.base.domain.*; @@ -8,11 +7,9 @@ import io.dataease.base.mapper.ChartViewMapper; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.CommonBeanFactory; -import io.dataease.commons.utils.LogUtil; import io.dataease.controller.request.chart.ChartExtFilterRequest; import io.dataease.controller.request.chart.ChartExtRequest; import io.dataease.controller.request.chart.ChartViewRequest; -import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; @@ -21,6 +18,7 @@ import io.dataease.dto.chart.ChartViewDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.chart.Series; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.provider.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; import org.apache.commons.collections4.CollectionUtils; @@ -31,7 +29,6 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.math.BigDecimal; import java.math.RoundingMode; -import java.text.MessageFormat; import java.util.*; /** @@ -46,8 +43,6 @@ public class ChartViewService { private DataSetTableService dataSetTableService; @Resource private DatasourceService datasourceService; - // @Resource -// private SparkCalc sparkCalc; @Resource private DataSetTableFieldsService dataSetTableFieldsService; @@ -105,10 +100,6 @@ public class ChartViewService { BeanUtils.copyBean(dto, view); return dto; } -// List xIds = xAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList()); -// List yIds = yAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList()); -// List xList = dataSetTableFieldsService.getListByIds(xIds); -// List yList = dataSetTableFieldsService.getListByIds(yIds); // 过滤来自仪表盘的条件 List extFilterList = new ArrayList<>(); @@ -138,18 +129,15 @@ public class ChartViewService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); if (StringUtils.equalsIgnoreCase(table.getType(), "db")) { datasourceRequest.setTable(dataTableInfoDTO.getTable()); - datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xAxis, yAxis, extFilterList)); + datasourceRequest.setQuery(qp.getSQL(dataTableInfoDTO.getTable(), xAxis, yAxis, extFilterList)); } else if (StringUtils.equalsIgnoreCase(table.getType(), "sql")) { - datasourceRequest.setQuery(getSQL(ds.getType(), " (" + dataTableInfoDTO.getSql() + ") AS tmp ", xAxis, yAxis, extFilterList)); + datasourceRequest.setQuery(qp.getSQLAsTmp(dataTableInfoDTO.getSql(), xAxis, yAxis, extFilterList)); } data = datasourceProvider.getData(datasourceRequest); } else if (table.getMode() == 1) {// 抽取 - // 获取数据集de字段 -// List fields = dataSetTableFieldsService.getFieldsByTableId(table.getId()); -// data = sparkCalc.getData(table.getId(), fields, xAxis, yAxis, "tmp_" + view.getId().split("-")[0], extFilterList); - // 连接doris，构建doris数据源查询 Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); @@ -157,7 +145,8 @@ public class ChartViewService { datasourceRequest.setDatasource(ds); String tableName = "ds_" + table.getId().replaceAll("-", "_"); datasourceRequest.setTable(tableName); - datasourceRequest.setQuery(transDorisSQL(tableName, xAxis, yAxis, extFilterList)); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, extFilterList)); data = datasourceProvider.getData(datasourceRequest); } @@ -220,309 +209,6 @@ public class ChartViewService { return dto; } - public String transMysqlExtFilter(List requestList) { - if (CollectionUtils.isEmpty(requestList)) { - return ""; - } - StringBuilder filter = new StringBuilder(); - for (ChartExtFilterRequest request : requestList) { - List value = request.getValue(); - if (CollectionUtils.isEmpty(value)) { - continue; - } - DatasetTableField field = request.getDatasetTableField(); - if (field.getDeType() == 1 && field.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(") - .append(field.getDataeaseName()) - .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND ").append(field.getDataeaseName()); - } - filter.append(" ") - .append(transMysqlFilterTerm(request.getOperator())) - .append(" "); - if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { - filter.append("('").append(StringUtils.join(value, "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { - filter.append("'%").append(value.get(0)).append("%'"); - } else { - filter.append("'").append(value.get(0)).append("'"); - } - } - return filter.toString(); - } - - public String getSQL(String type, String table, List xAxis, List yAxis, List extFilterRequestList) { - DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); - switch (datasourceType) { - case mysql: - return transMysqlSQL(table, xAxis, yAxis, extFilterRequestList); - case sqlServer: - default: - return ""; - } - } - - public String transDorisSQL(String table, List xAxis, List yAxis, List extFilterRequestList) { - // 字段汇总 排序等 - String[] field = yAxis.stream().map(y -> { - StringBuilder f = new StringBuilder(); - if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { - f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")"); - } else { - f.append("ROUND(") - .append(y.getSummary()).append("(") - .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append("),2").append(")"); - } - f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); - return f.toString(); - }).toArray(String[]::new); - String[] groupField = xAxis.stream().map(x -> { - StringBuilder stringBuilder = new StringBuilder(); - // 如果原始类型为时间 - if (x.getDeExtractType() == 1) { - if (x.getDeType() == 2 || x.getDeType() == 3) { - stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName()); - } else { - stringBuilder.append(x.getDataeaseName()); - } - } else { - if (x.getDeType() == 1) { - stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName()); - } else { - stringBuilder.append(x.getDataeaseName()); - } - } - return stringBuilder.toString(); - }).toArray(String[]::new); - String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new); - String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new); - String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new); - String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); - System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); - - String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) - .map(x -> { - String[] s = x.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - if (x.getDeType() == 1 && x.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(") - .append(x.getDataeaseName()) - .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND ").append(x.getDataeaseName()); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("%").append(f.getValue()).append("%"); - } else { - filter.append("'" + f.getValue() + "'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - - String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", - StringUtils.join(groupField, ","), - StringUtils.join(field, ","), - table, - xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter - StringUtils.join(group, ","), - StringUtils.join(order, ",")); - if (sql.endsWith(",")) { - sql = sql.substring(0, sql.length() - 1); - } - // 如果是对结果字段过滤，则再包裹一层sql - String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) - .map(y -> { - String[] s = y.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - if (y.getDeType() == 1 && y.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()) - .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("%").append(f.getValue()).append("%"); - } else { - filter.append("'" + f.getValue() + "'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - if (resultFilter.length == 0) { - return sql; - } else { - String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}", - "(" + sql + ") AS tmp", - StringUtils.join(resultFilter, " ")); - return filterSql; - } - } - - public String transMysqlSQL(String table, List xAxis, List yAxis, List extFilterRequestList) { - // 字段汇总 排序等 - String[] field = yAxis.stream().map(y -> { - StringBuilder f = new StringBuilder(); - if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) { - f.append(y.getSummary()).append("(").append(y.getDataeaseName()).append(")"); - } else { - if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { - f.append("CAST(") - .append(y.getSummary()).append("(") - .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(") AS DECIMAL(20,2)").append(")"); - } else { - f.append(y.getSummary()).append("(") - .append("CAST(").append(y.getDataeaseName()).append(" AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(")"); - } - } - f.append(" AS _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); - return f.toString(); - }).toArray(String[]::new); - String[] groupField = xAxis.stream().map(x -> { - StringBuilder stringBuilder = new StringBuilder(); - // 如果原始类型为时间 - if (x.getDeExtractType() == 1) { - if (x.getDeType() == 2 || x.getDeType() == 3) { - stringBuilder.append("unix_timestamp(").append(x.getDataeaseName()).append(")*1000 as ").append(x.getDataeaseName()); - } else { - stringBuilder.append(x.getDataeaseName()); - } - } else { - if (x.getDeType() == 1) { - stringBuilder.append("FROM_UNIXTIME(cast(").append(x.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(x.getDataeaseName()); - } else { - stringBuilder.append(x.getDataeaseName()); - } - } - return stringBuilder.toString(); - }).toArray(String[]::new); - String[] group = xAxis.stream().map(ChartViewFieldDTO::getDataeaseName).toArray(String[]::new); - String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> f.getDataeaseName() + " " + f.getSort()).toArray(String[]::new); - String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> "_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getDataeaseName(), "*") ? "" : f.getDataeaseName()) + " " + f.getSort()).toArray(String[]::new); - String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); - System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); - - String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) - .map(x -> { - String[] s = x.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - if (x.getDeType() == 1 && x.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(") - .append(x.getDataeaseName()) - .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND ").append(x.getDataeaseName()); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("%").append(f.getValue()).append("%"); - } else { - filter.append("'").append(f.getValue()).append("'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - - String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", - StringUtils.join(groupField, ","), - StringUtils.join(field, ","), - table, - xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "" + transMysqlExtFilter(extFilterRequestList),// origin field filter and panel field filter - StringUtils.join(group, ","), - StringUtils.join(order, ",")); - if (sql.endsWith(",")) { - sql = sql.substring(0, sql.length() - 1); - } - // 如果是对结果字段过滤，则再包裹一层sql - String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) - .map(y -> { - String[] s = y.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - // 原始类型不是时间，在de中被转成时间的字段做处理 - if (y.getDeType() == 1 && y.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()) - .append(" AS decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND _").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*") ? "" : y.getDataeaseName()); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("%").append(f.getValue()).append("%"); - } else { - filter.append("'").append(f.getValue()).append("'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - if (resultFilter.length == 0) { - return sql; - } else { - String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1}", - "(" + sql + ") AS tmp", - StringUtils.join(resultFilter, " ")); - return filterSql; - } - } - - public String transMysqlFilterTerm(String term) { - switch (term) { - case "eq": - return " = "; - case "not_eq": - return " <> "; - case "lt": - return " < "; - case "le": - return " <= "; - case "gt": - return " > "; - case "ge": - return " >= "; - case "in": - return " IN "; - case "not in": - return " NOT IN "; - case "like": - return " LIKE "; - case "not like": - return " NOT LIKE "; - case "null": - return " IS NULL "; - case "not_null": - return " IS NOT NULL "; - default: - return ""; - } - } - private void checkName(ChartViewWithBLOBs chartView) { if (StringUtils.isEmpty(chartView.getId())) { return; diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 02cbd816bc..ecaef16c91 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -8,7 +8,6 @@ import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasourceMapper; import io.dataease.commons.utils.*; import io.dataease.controller.request.dataset.DataSetTableRequest; -import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.JdbcProvider; @@ -18,6 +17,8 @@ import io.dataease.dto.dataset.DataSetPreviewPage; import io.dataease.dto.dataset.DataSetTableUnionDTO; import io.dataease.dto.dataset.DataTableInfoCustomUnion; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.provider.DDLProvider; +import io.dataease.provider.QueryProvider; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -126,9 +127,10 @@ public class DataSetTableService { JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); - datasourceRequest.setQuery("drop table if exists " + dorisTableName); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType()); + datasourceRequest.setQuery(ddlProvider.dropTableOrView(dorisTableName)); jdbcProvider.exec(datasourceRequest); - datasourceRequest.setQuery("drop table if exists " + DorisTableUtils.dorisTmpName(dorisTableName)); + datasourceRequest.setQuery(ddlProvider.dropTableOrView(DorisTableUtils.dorisTmpName(dorisTableName))); jdbcProvider.exec(datasourceRequest); } @@ -224,14 +226,15 @@ public class DataSetTableService { datasourceRequest.setDatasource(ds); String table = dataTableInfoDTO.getTable(); - datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); try { data.addAll(datasourceProvider.getData(datasourceRequest)); } catch (Exception e) { e.printStackTrace(); } try { - datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table)); + datasourceRequest.setQuery(qp.createQueryCountSQL(table)); dataSetPreviewPage.setTotal(Integer.valueOf(datasourceProvider.getData(datasourceRequest).get(0)[0])); } catch (Exception e) { e.printStackTrace(); @@ -243,14 +246,15 @@ public class DataSetTableService { datasourceRequest.setDatasource(ds); String sql = dataTableInfoDTO.getSql(); - datasourceRequest.setQuery(createQuerySQL(ds.getType(), " (" + sql + ") AS tmp ", fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLAsTmpWithPage(sql, fields, page, pageSize, realSize)); try { data.addAll(datasourceProvider.getData(datasourceRequest)); } catch (Exception e) { e.printStackTrace(); } try { - datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), " (" + sql + ") AS tmp ")); + datasourceRequest.setQuery(qp.createQueryCountSQLAsTmp(sql)); dataSetPreviewPage.setTotal(Integer.valueOf(datasourceProvider.getData(datasourceRequest).get(0)[0])); } catch (Exception e) { e.printStackTrace(); @@ -261,7 +265,8 @@ public class DataSetTableService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); - datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); try { data.addAll(jdbcProvider.getData(datasourceRequest)); } catch (Exception e) { @@ -269,7 +274,7 @@ public class DataSetTableService { } try { - datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table)); + datasourceRequest.setQuery(qp.createQueryCountSQL(table)); dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); } catch (Exception e) { e.printStackTrace(); @@ -280,7 +285,8 @@ public class DataSetTableService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); - datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fields) + " LIMIT " + (page - 1) * pageSize + "," + realSize); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); try { data.addAll(jdbcProvider.getData(datasourceRequest)); } catch (Exception e) { @@ -288,7 +294,7 @@ public class DataSetTableService { } try { - datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table)); + datasourceRequest.setQuery(qp.createQueryCountSQL(table)); dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); } catch (Exception e) { e.printStackTrace(); @@ -324,7 +330,8 @@ public class DataSetTableService { datasourceRequest.setQuery(sql); List previewFields = datasourceProvider.fetchResultField(datasourceRequest); // 正式执行 - datasourceRequest.setQuery("SELECT * FROM (" + sql + ") AS tmp ORDER BY " + previewFields.get(0).getFieldName() + " LIMIT 0,1000"); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName())); Map result = datasourceProvider.fetchResultAndField(datasourceRequest); List data = result.get("dataList"); List fields = result.get("fieldList"); @@ -362,7 +369,8 @@ public class DataSetTableService { datasourceRequest.setQuery(sql); List previewFields = jdbcProvider.fetchResultField(datasourceRequest); - datasourceRequest.setQuery("SELECT * FROM (" + sql + ") AS tmp ORDER BY " + previewFields.get(0).getFieldName() + " LIMIT 0,1000"); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName())); Map result = jdbcProvider.fetchResultAndField(datasourceRequest); List data = result.get("dataList"); List fields = result.get("fieldList"); @@ -400,6 +408,7 @@ public class DataSetTableService { return map; } + // 自助数据集从doris里预览数据 private String getCustomSQL(DataTableInfoDTO dataTableInfoDTO, List list) { Map customInfo = new TreeMap<>(); dataTableInfoDTO.getList().forEach(ele -> { @@ -499,6 +508,7 @@ public class DataSetTableService { createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQL(dataTableInfoDTO, dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()))); return; } + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { TableFiled filed = fields.get(i); @@ -516,7 +526,7 @@ public class DataSetTableService { datasetTableField.setDeType(transFieldType(filed.getFieldType())); datasetTableField.setDeExtractType(transFieldType(filed.getFieldType())); } else { - Integer fieldType = transFieldType(ds.getType(), filed.getFieldType()); + Integer fieldType = qp.transFieldType(filed.getFieldType()); datasetTableField.setDeType(fieldType == 4 ? 2 : fieldType); datasetTableField.setDeExtractType(fieldType); } @@ -534,56 +544,14 @@ public class DataSetTableService { JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); + DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType()); // 先删除表 - datasourceRequest.setQuery("DROP VIEW IF EXISTS " + dorisTableName); + datasourceRequest.setQuery(ddlProvider.dropTableOrView(dorisTableName)); jdbcProvider.exec(datasourceRequest); - datasourceRequest.setQuery("CREATE VIEW IF NOT EXISTS " + dorisTableName + " AS (" + customSql + ")"); + datasourceRequest.setQuery(ddlProvider.createView(dorisTableName, customSql)); jdbcProvider.exec(datasourceRequest); } - public String createQueryCountSQL(String type, String table) { - DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); - switch (datasourceType) { - case mysql: - return MessageFormat.format("SELECT count(*) FROM {0}", table); - case sqlServer: - return MessageFormat.format("SELECT count(*) FROM {0}", table); - default: - return MessageFormat.format("SELECT count(*) FROM {0}", table); - } - } - - public String createQuerySQL(String type, String table, List fields) { - String[] array = fields.stream().map(f -> { - StringBuilder stringBuilder = new StringBuilder(); - // 如果原始类型为时间 - if (f.getDeExtractType() == 1) { - if (f.getDeType() == 2 || f.getDeType() == 3) { - stringBuilder.append("unix_timestamp(").append(f.getDataeaseName()).append(")*1000 as ").append(f.getDataeaseName()); - } else { - stringBuilder.append(f.getDataeaseName()); - } - } else { - if (f.getDeType() == 1) { - stringBuilder.append("FROM_UNIXTIME(cast(").append(f.getDataeaseName()).append(" as decimal(20,0))/1000,'%Y-%m-%d %H:%i:%S') as ").append(f.getDataeaseName()); - } else { - stringBuilder.append(f.getDataeaseName()); - } - } - return stringBuilder.toString(); - }).toArray(String[]::new); - - DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); - switch (datasourceType) { - case mysql: - return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); - case sqlServer: - return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); - default: - return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getDataeaseName() : "null"), StringUtils.join(array, ","), table); - } - } - public Integer transFieldType(String field) { switch (field) { case "TEXT": @@ -599,51 +567,6 @@ public class DataSetTableService { } } - public Integer transFieldType(String type, String field) { - DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); - switch (datasourceType) { - case mysql: - return transMysqlField(field); - case sqlServer: - default: - return 0; - } - } - - public Integer transMysqlField(String field) { - switch (field) { - case "CHAR": - case "VARCHAR": - case "TEXT": - case "TINYTEXT": - case "MEDIUMTEXT": - case "LONGTEXT": - case "ENUM": - return 0;// 文本 - case "DATE": - case "TIME": - case "YEAR": - case "DATETIME": - case "TIMESTAMP": - return 1;// 时间 - case "INT": - case "SMALLINT": - case "MEDIUMINT": - case "INTEGER": - case "BIGINT": - return 2;// 整型 - case "FLOAT": - case "DOUBLE": - case "DECIMAL": - return 3;// 浮点 - case "BIT": - case "TINYINT": - return 4;// 布尔 - default: - return 0; - } - } - public DatasetTableIncrementalConfig incrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) { if (StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { return new DatasetTableIncrementalConfig(); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index f60dab68ae..c9b6f325aa 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -19,6 +19,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.dataset.DataSetTaskLogDTO; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.provider.QueryProvider; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -109,15 +110,15 @@ public class ExtractDataService { private static String shellScript = "curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load\n" + "rm -rf %s\n"; - private String createDorisTablColumnSql( List datasetTableFields){ + private String createDorisTablColumnSql(List datasetTableFields) { String Column_Fields = "dataease_uuid varchar(50), `"; for (DatasetTableField datasetTableField : datasetTableFields) { Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` "; - switch (datasetTableField.getDeExtractType()){ + switch (datasetTableField.getDeExtractType()) { case 0: - if(datasetTableField.getSize() > 65533){ + if (datasetTableField.getSize() > 65533) { Column_Fields = Column_Fields + "String" + ",`"; - }else { + } else { Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`"; } break; @@ -138,23 +139,25 @@ public class ExtractDataService { break; } } - Column_Fields = Column_Fields.substring(0, Column_Fields.length() -2); - Column_Fields = "(" + Column_Fields + ")\n"; + Column_Fields = Column_Fields.substring(0, Column_Fields.length() - 2); + Column_Fields = "(" + Column_Fields + ")\n"; return Column_Fields; } - private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception{ - Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource"); - JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);; + private void createDorisTable(String dorisTableName, String dorisTablColumnSql) throws Exception { + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + ; DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql)); jdbcProvider.exec(datasourceRequest); } - private void replaceTable (String dorisTableName) throws Exception{ - Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource"); - JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);; + private void replaceTable(String dorisTableName) throws Exception { + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + ; DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dorisDatasource); datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName))); @@ -168,9 +171,9 @@ public class ExtractDataService { Datasource datasource = new Datasource(); try { datasetTable = dataSetTableService.get(datasetTableId); - if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){ + if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); - }else { + } else { datasource.setType(datasetTable.getType()); } @@ -288,10 +291,10 @@ public class ExtractDataService { String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null); SlaveServerJobStatus jobStatus = null; boolean executing = true; - while(executing) { + while (executing) { jobStatus = remoteSlaveServer.getJobStatus(jobMeta.getName(), lastCarteObjectId, 0); executing = jobStatus.isRunning() || jobStatus.isWaiting(); - if(!executing) + if (!executing) break; Thread.sleep(1000); } @@ -320,7 +323,7 @@ public class ExtractDataService { String dorisOutputTable = null; String jobName = null; String script = null; - Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource"); + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); DorisConfigration dorisConfigration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfigration.class); String columns = columnFeilds + ",dataease_uuid"; String transName = null; @@ -329,18 +332,18 @@ public class ExtractDataService { transName = "trans_" + datasetTable.getId(); dorisOutputTable = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())); jobName = "job_" + datasetTable.getId(); - script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND",root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention); + script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention); break; case "incremental_add": transName = "trans_add_" + datasetTable.getId(); dorisOutputTable = DorisTableUtils.dorisName(datasetTable.getId()); jobName = "job_add_" + datasetTable.getId(); - script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention); + script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "APPEND", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), dorisOutputTable, root_path + dorisOutputTable + "." + extention); break; case "incremental_delete": transName = "trans_delete_" + datasetTable.getId(); dorisOutputTable = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId())); - script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "DELETE", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(),dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + dorisOutputTable + "." + extention); + script = String.format(shellScript, dorisConfigration.getUsername(), dorisConfigration.getPassword(), String.valueOf(System.currentTimeMillis()), separator, columns, "DELETE", root_path + dorisOutputTable + "." + extention, dorisConfigration.getHost(), dorisConfigration.getHttpPort(), dorisConfigration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + dorisOutputTable + "." + extention); jobName = "job_delete_" + datasetTable.getId(); break; default: @@ -402,14 +405,14 @@ public class ExtractDataService { FileUtils.writeStringToFile(file, jobXml, "UTF-8"); } - private String fetchSqlField(String sql, Datasource ds)throws Exception{ + private String fetchSqlField(String sql, Datasource ds) throws Exception { String tmpSql = sql; DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - if(tmpSql.trim().endsWith(";")){ - tmpSql = tmpSql.substring(0, tmpSql.length() -1 ) + " limit 0"; - }else { + if (tmpSql.trim().endsWith(";")) { + tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0"; + } else { tmpSql = tmpSql + " limit 0"; } datasourceRequest.setQuery(tmpSql); @@ -433,9 +436,10 @@ public class ExtractDataService { MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class); dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword()); transMeta.addDatabase(dataMeta); - if(extractType.equalsIgnoreCase("all_scope")){ + if (extractType.equalsIgnoreCase("all_scope")) { String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); - selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), tableName, datasetTableFields); + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createQuerySQL(tableName, datasetTableFields); } inputStep = inputStep(transMeta, selectSQL); break; @@ -481,7 +485,7 @@ public class ExtractDataService { FileUtils.writeStringToFile(file, transXml, "UTF-8"); } - private StepMeta inputStep(TransMeta transMeta, String selectSQL){ + private StepMeta inputStep(TransMeta transMeta, String selectSQL) { TableInputMeta tableInput = new TableInputMeta(); DatabaseMeta database = transMeta.findDatabase("db"); tableInput.setDatabaseMeta(database); @@ -492,7 +496,7 @@ public class ExtractDataService { return fromStep; } - private StepMeta excelInputStep(String filePath, List datasetTableFields){ + private StepMeta excelInputStep(String filePath, List datasetTableFields) { ExcelInputMeta excelInputMeta = new ExcelInputMeta(); excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI); excelInputMeta.setPassword("Encrypted"); @@ -500,7 +504,7 @@ public class ExtractDataService { excelInputMeta.setStartsWithHeader(true); excelInputMeta.setIgnoreEmptyRows(true); ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()]; - for (int i=0; i datasetTableFields){ + private StepMeta udjc(List datasetTableFields) { String needToChangeolumnType = ""; for (DatasetTableField datasetTableField : datasetTableFields) { - if(datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4){ + if (datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4) { needToChangeolumnType = needToChangeolumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getOriginName()); } } @@ -552,28 +556,28 @@ public class ExtractDataService { StepMeta userDefinedJavaClassStep = new StepMeta("UserDefinedJavaClass", "UserDefinedJavaClass", userDefinedJavaClassMeta); userDefinedJavaClassStep.setLocation(300, 100); userDefinedJavaClassStep.setDraw(true); - return userDefinedJavaClassStep; + return userDefinedJavaClassStep; } - public boolean isKettleRunning(){ + public boolean isKettleRunning() { try { if (!InetAddress.getByName(carte).isReachable(1000)) { return false; } HttpClient httpClient; - HttpGet getMethod = new HttpGet( "http://" + carte + ":" + port); + HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); clientBuilder.setConnectionTimeout(1); clientBuilder.setCredentials(user, passwd); httpClient = clientBuilder.build(); - HttpResponse httpResponse = httpClient.execute( getMethod ); + HttpResponse httpResponse = httpClient.execute(getMethod); int statusCode = httpResponse.getStatusLine().getStatusCode(); - if ( statusCode != -1 && statusCode < 400) { + if (statusCode != -1 && statusCode < 400) { return true; - }else { + } else { return false; } - }catch (Exception e){ + } catch (Exception e) { return false; } } diff --git a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java index b696ecf286..306051c0cd 100644 --- a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java +++ b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java @@ -7,6 +7,7 @@ import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.service.DatasourceService; +import io.dataease.provider.QueryProvider; import io.dataease.service.dataset.DataSetFieldService; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; @@ -58,7 +59,8 @@ public class DirectFieldService implements DataSetFieldService { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String querySQL = dataSetTableService.createQuerySQL(ds.getType(), tableName, Collections.singletonList(field)); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + String querySQL = qp.createQuerySQL(tableName, Collections.singletonList(field)); datasourceRequest.setQuery(querySQL); try { List rows = datasourceProvider.getData(datasourceRequest); diff --git a/frontend/src/components/LangSelect/index.vue b/frontend/src/components/LangSelect/index.vue index 355e29fcac..f606735f2c 100644 --- a/frontend/src/components/LangSelect/index.vue +++ b/frontend/src/components/LangSelect/index.vue @@ -1,5 +1,5 @@