diff --git a/backend/pom.xml b/backend/pom.xml index de5cb967ad..24ac942aac 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -222,11 +222,7 @@ dataease-plugin-view 1.9.0 - - cn.hutool - hutool-all - 5.7.4 - + pentaho-kettle diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtPanelGroupMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtPanelGroupMapper.xml index cf0db400c8..989b52f0e2 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtPanelGroupMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtPanelGroupMapper.xml @@ -33,6 +33,7 @@ panel_group.`name` AS label, panel_group.`source`, panel_group.`panel_type`, + sourcePanelGroup.`mobile_layout`, sourcePanelGroup.`name` as source_panel_name, authInfo.privileges as `privileges` from (select GET_V_AUTH_MODEL_ID_P_USE (#{userId}, 'panel') cids) t,panel_group @@ -105,6 +106,7 @@ panel_group.panel_type, panel_group.`name` AS label, panel_group.`node_type`, + panel_group.`mobile_layout`, (case when ISNULL(defaultPanelGroup.id) then false else true end) is_default, defaultPanelGroup.id as default_panel_id, defaultPanelGroup.`name` as default_panel_name, diff --git a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java index a28cbba4a4..6e5dd1fb08 100644 --- a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java @@ -4,6 +4,7 @@ public enum DatasourceTypes { excel("excel", "excel", "", "", "", "", ""), mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), hive("hive", "hive", "org.apache.hive.jdbc.HiveDriver", "`", "`", "'", "'"), + impala("impala", "impala", "org.apache.hive.jdbc.HiveDriver", "`", "`", "'", "'"), mariadb("mariadb", "mariadb", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""), diff --git a/backend/src/main/java/io/dataease/dto/datasource/ImpalaConfiguration.java b/backend/src/main/java/io/dataease/dto/datasource/ImpalaConfiguration.java new file mode 100644 index 0000000000..38added74e --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/datasource/ImpalaConfiguration.java @@ -0,0 +1,28 @@ +package io.dataease.dto.datasource; + +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +@Getter +@Setter +public class ImpalaConfiguration extends JdbcConfiguration { + + private String driver = "com.cloudera.impala.jdbc.Driver"; + private String extraParams = ""; + + public String getJdbc() { + if(StringUtils.isEmpty(extraParams.trim())){ + return "jdbc:impala://HOSTNAME:PORT/DATABASE" + .replace("HOSTNAME", getHost().trim()) + .replace("PORT", getPort().toString().trim()) + .replace("DATABASE", getDataBase().trim()); + }else { + return "jdbc:impala://HOSTNAME:PORT/DATABASE;EXTRA_PARAMS" + .replace("HOSTNAME", getHost().trim()) + .replace("PORT", getPort().toString().trim()) + .replace("DATABASE", getDataBase().trim()) + .replace("EXTRA_PARAMS", getExtraParams().trim()); + } + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/provider/ProviderFactory.java index 3dc13a582b..b874513dc2 100644 --- a/backend/src/main/java/io/dataease/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/provider/ProviderFactory.java @@ -54,6 +54,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("redshiftQuery", QueryProvider.class); case hive: return context.getBean("hiveQuery", QueryProvider.class); + case impala: + return context.getBean("impalaQuery", QueryProvider.class); case db2: return context.getBean("db2Query", QueryProvider.class); case api: diff --git a/backend/src/main/java/io/dataease/provider/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java index c7eca21213..b7ec46a861 100644 --- a/backend/src/main/java/io/dataease/provider/QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java @@ -9,7 +9,7 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; -import io.dataease.provider.query.pg.PgConstants; +import io.dataease.plugins.common.constants.PgConstants; import java.util.List; diff --git a/backend/src/main/java/io/dataease/provider/SQLConstants.java b/backend/src/main/java/io/dataease/provider/SQLConstants.java deleted file mode 100644 index 3fbe98f655..0000000000 --- a/backend/src/main/java/io/dataease/provider/SQLConstants.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.dataease.provider; - -import java.util.ArrayList; -import java.util.List; - -/** - * @Author gin - * @Date 2021/7/8 3:12 下午 - */ -public class SQLConstants { - /** - * 维度类型list - */ - public static final List DIMENSION_TYPE = new ArrayList() {{ - add(0);// 文本 - add(1);// 时间 - add(5);// 地理位置 - }}; - - /** - * 指标类型list - */ - public static final List QUOTA_TYPE = new ArrayList() {{ - add(2);// 整型 - add(3);// 浮点 - add(4);// 布尔 - }}; - - /** - * sql ST模板 - */ - public static final String SQL_TEMPLATE = "sql/sqlTemplate.stg"; - - public static final String TABLE_ALIAS_PREFIX = "t_a_%s"; - public static final String FIELD_ALIAS_X_PREFIX = "f_ax_%s"; - public static final String FIELD_ALIAS_Y_PREFIX = "f_ay_%s"; - public static final String GROUP_ALIAS_PREFIX = "g_a_%s"; - public static final String ORDER_ALIAS_X_PREFIX = "o_ax_%s"; - public static final String ORDER_ALIAS_Y_PREFIX = "o_ay_%s"; - public static final String WHERE_ALIAS_PREFIX = "w_a_%s"; -} diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 2a98a6c683..eac13d4f2d 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -154,7 +154,7 @@ public class JdbcProvider extends DatasourceProvider { while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); String database; - if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())) { + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name()) || datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.impala.name())) { database = resultSet.getString("TABLE_SCHEM"); } else { database = resultSet.getString("TABLE_CAT"); @@ -485,6 +485,14 @@ public class JdbcProvider extends DatasourceProvider { driver = hiveConfiguration.getDriver(); jdbcurl = hiveConfiguration.getJdbc(); break; + case impala: + ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class); + System.out.println(new Gson().toJson(impalaConfiguration)); + username = impalaConfiguration.getUsername(); + password = impalaConfiguration.getPassword(); + driver = impalaConfiguration.getDriver(); + jdbcurl = impalaConfiguration.getJdbc(); + break; case db2: Db2Configuration db2Configuration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), Db2Configuration.class); username = db2Configuration.getUsername(); @@ -586,6 +594,13 @@ public class JdbcProvider extends DatasourceProvider { dataSource.setUrl(hiveConfiguration.getJdbc()); jdbcConfiguration = hiveConfiguration; break; + case impala: + ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class); + dataSource.setPassword(impalaConfiguration.getPassword()); + dataSource.setDriverClassName(impalaConfiguration.getDriver()); + dataSource.setUrl(impalaConfiguration.getJdbc()); + jdbcConfiguration = impalaConfiguration; + break; case db2: Db2Configuration db2Configuration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), Db2Configuration.class); dataSource.setPassword(db2Configuration.getPassword()); @@ -614,6 +629,7 @@ public class JdbcProvider extends DatasourceProvider { case engine_doris: case ds_doris: case hive: + case impala: return "show tables"; case sqlServer: SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfiguration.class); diff --git a/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java deleted file mode 100644 index 2305748abd..0000000000 --- a/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java +++ /dev/null @@ -1,49 +0,0 @@ -package io.dataease.provider.engine.doris; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.engine_doris; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class DorisConstants extends SQLConstants { - public static final String KEYWORD_TABLE = engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; - - public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')"; - - public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; - - public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; - - public static final String DEFAULT_INT_FORMAT = "BIGINT"; - - public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String WHERE_NUMBER_VALUE = "%s"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; - - public static final String ROUND = "ROUND(%s,%s)"; - - public static final String VARCHAR = "VARCHAR"; -} diff --git a/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java index 22b807936b..2e071eb093 100644 --- a/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java @@ -10,8 +10,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.DorisConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -28,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java index e75a357da7..badf689d39 100644 --- a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java @@ -1,6 +1,7 @@ package io.dataease.provider.engine.mysql; -import io.dataease.provider.SQLConstants; + +import io.dataease.plugins.common.constants.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.engine_mysql; diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java index 9909543e8e..3d4d3b85f0 100644 --- a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java @@ -11,7 +11,7 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -28,7 +28,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java b/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java deleted file mode 100644 index 7f9bd5d9ec..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.dataease.provider.query.ck; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.ck; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class CKConstants extends SQLConstants { - public static final String KEYWORD_TABLE = ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix(); - - public static final String toInt32 = "toInt32(%s)"; - - public static final String toDateTime = "toDateTime(%s)"; - - public static final String toInt64 = "toInt64(%s)"; - - public static final String toFloat64 = "toFloat64(%s)"; - - public static final String formatDateTime = "formatDateTime(%s,'%s')"; - - public static final String toDecimal = "toDecimal64(%s,2)"; - - public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java index 8f5d32a266..bfd9e3726e 100644 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java @@ -11,8 +11,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.CKConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +30,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java deleted file mode 100644 index e914d91b9c..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.dataease.provider.query.db2; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.db2; - -public class Db2Constants extends SQLConstants { - public static final String KEYWORD_TABLE = db2.getKeywordPrefix() + "%s" + db2.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + db2.getKeywordPrefix() + "%s" + db2.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "BIGINT(TIMESTAMPDIFF(2,CHAR(%s -TIMESTAMP('1970-01-01 08:00:00'))))"; - - public static final String DATE_FORMAT = "TO_CHAR(TIMESTAMP(%s),'%s')"; - - public static final String FROM_UNIXTIME = "TO_CHAR(TIMESTAMP('1970-01-01 08:00:00') +(%s)SECONDS, '%s')"; - - public static final String STR_TO_DATE = "timestamp(trim(char(%s)))"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-DD HH24:MI:SS"; - - public static final String DEFAULT_INT_FORMAT = "BIGINT"; - - public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java index d9fd103ce0..7daae940a5 100644 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java @@ -13,8 +13,9 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.Db2Configuration; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.Db2Constants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; @Service("db2Query") public class Db2QueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java index 95fb323686..d5d5138e77 100644 --- a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java @@ -11,8 +11,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.EsSqlLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +30,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; @Service("esQuery") public class EsQueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java b/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java deleted file mode 100644 index a9186a68e7..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java +++ /dev/null @@ -1,37 +0,0 @@ -package io.dataease.provider.query.es; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.es; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class EsSqlLConstants extends SQLConstants { - public static final String KEYWORD_TABLE = es.getKeywordPrefix() + "%s" + es.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + es.getKeywordPrefix() + "%s" + es.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; - - public static final String DATETIME_FORMAT = "DATETIME_FORMAT(%s,'%s')"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String ROUND = "ROUND(%s, %s)"; - - public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java index 9e054efe3f..8252f6d444 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java @@ -11,8 +11,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.HiveConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +30,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaConstants.java similarity index 79% rename from backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java rename to backend/src/main/java/io/dataease/provider/query/impala/ImpalaConstants.java index ec1a82007f..a7ffdc6824 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java +++ b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaConstants.java @@ -1,21 +1,17 @@ -package io.dataease.provider.query.hive; +package io.dataease.provider.query.impala; import io.dataease.provider.SQLConstants; import static io.dataease.commons.constants.DatasourceTypes.mysql; -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class HiveConstants extends SQLConstants { +public class ImpalaConstants extends SQLConstants { public static final String KEYWORD_TABLE = mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); public static final String KEYWORD_FIX = "%s." + mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); public static final String UNIX_TIMESTAMP = "unix_timestamp(%s)"; - public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')"; + public static final String DATE_FORMAT = "from_unixtime(UNIX_TIMESTAMP(%s), '%s')"; public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; @@ -25,7 +21,7 @@ public class HiveConstants extends SQLConstants { public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; + public static final String DEFAULT_INT_FORMAT = "BIGINT"; public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; diff --git a/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java new file mode 100644 index 0000000000..9fc7c88c5a --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java @@ -0,0 +1,1084 @@ +package io.dataease.provider.query.impala; + +import io.dataease.base.domain.ChartViewWithBLOBs; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.commons.constants.DeTypeConstants; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartCustomFilterItemDTO; +import io.dataease.dto.chart.ChartFieldCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; + +import javax.annotation.Resource; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + + +@Service("impalaQuery") +public class ImpalaQueryProvider extends QueryProvider { + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Override + public Integer transFieldType(String field) { + field = field.toLowerCase(); + switch (field) { + case "varchar": + case "string": + case "char": + return 0;// 文本 + case "timestamp": + case "date": + return 1;// 时间 + case "tinyint": + case "smallint": + case "int": + case "bigint": + return 2;// 整型 + case "decimal": + case "float": + case "double": + return 3;// 浮点 + case "boolean": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 1000"; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField; + if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(f.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + // 处理横轴字段 + if (f.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + fieldName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else { + fieldName = originField; + } + } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) { + if (f.getDeType() == DeTypeConstants.DE_INT) { + fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT); + } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_FLOAT_FORMAT); + } else if (f.getDeType() == DeTypeConstants.DE_TIME) { + fieldName = String.format(ImpalaConstants.STR_TO_DATE, originField, ImpalaConstants.DEFAULT_DATE_FORMAT); + } else { + fieldName = originField; + } + } else { + if (f.getDeType() == DeTypeConstants.DE_TIME) { + String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000"; + fieldName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT); + } else if (f.getDeType() == 2) { + fieldName = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT); + } else { + fieldName = originField; + } + } + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); + } + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", isGroup); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + return st_sql.render(); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup, List fieldCustomFilter) { + return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null, fieldCustomFilter); + } + + @Override + public String createQueryTableWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + realSize; + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + limit; + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup, List fieldCustomFilter) { + return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + limit; + } + + @Override + public String createQuerySQLWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List fieldCustomFilter) { + return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(ImpalaConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLTableInfo(String table, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", false); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("previewSql"); + st.add("isGroup", false); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(ImpalaConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpTableInfo(String sql, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + return getSQLTableInfo("(" + sqlFix(sql) + ")", xAxis, fieldCustomFilter, extFilterRequestList, null, view); + } + + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, null, view); + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + List xList = new ArrayList<>(); + xList.addAll(xAxis); + xList.addAll(extStack); + if (CollectionUtils.isNotEmpty(xList)) { + for (int i = 0; i < xList.size(); i++) { + ChartViewFieldDTO x = xList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(ImpalaConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, ChartViewWithBLOBs view) { + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extStack, null, view); + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, Datasource ds, ChartViewWithBLOBs view) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + List yList = new ArrayList<>(); + yList.addAll(yAxis); + yList.addAll(extBubble); + if (CollectionUtils.isNotEmpty(yList)) { + for (int i = 0; i < yList.size(); i++) { + ChartViewFieldDTO y = yList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(ImpalaConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, ChartViewWithBLOBs view) { + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extBubble, null, view); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + @Override + public String getSQLSummary(String table, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + // 字段汇总 排序等 + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(ImpalaConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.add(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + // 处理仪表板字段过滤 + String extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + if (extWheres != null) wheres.add(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList())); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(ImpalaConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return sqlLimit(st.render(), view); + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, fieldCustomFilter, extFilterRequestList, view); + } + + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0"; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + if (f.getDeExtractType() == 4) { // 处理 tinyint + stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName()); + } else { + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + } + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + } + + @Override + public String convertTableToSql(String tableName, Datasource ds) { + return createSQLPreview("SELECT * FROM " + String.format(ImpalaConstants.KEYWORD_TABLE, tableName), null); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + case "empty": + return " = "; + case "not_empty": + return " <> "; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public String transCustomFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + + List res = new ArrayList<>(); + for (ChartFieldCustomFilterDTO request : requestList) { + List list = new ArrayList<>(); + DatasetTableField field = request.getField(); + + if (ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + if (field.getDeType() == DeTypeConstants.DE_TIME) { + if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { + whereName = String.format(ImpalaConstants.STR_TO_DATE, originName, ImpalaConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == DeTypeConstants.DE_TIME) { + whereName = originName; + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_FLOAT_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else { + whereName = originName; + } + + if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) { + if (CollectionUtils.isNotEmpty(request.getEnumCheckField())) { + res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))"); + } + } else { + List filter = request.getFilter(); + for (ChartCustomFilterItemDTO filterItemDTO : filter) { + String value = filterItemDTO.getValue(); + String whereTerm = transMysqlFilterTerm(filterItemDTO.getTerm()); + String whereValue = ""; + + if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, value); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + if (CollectionUtils.isNotEmpty(list)) { + res.add("(" + String.join(" " + getLogic(request.getLogic()) + " ", strList) + ")"); + } + } + } + return CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null; + } + + public String transExtFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + + if (field.getDeType() == DeTypeConstants.DE_TIME) { + if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { + whereName = String.format(ImpalaConstants.STR_TO_DATE, originName, ImpalaConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == DeTypeConstants.DE_TIME) { + whereName = originName; + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(ImpalaConstants.CAST, originName, ImpalaConstants.DEFAULT_FLOAT_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else { + whereName = originName; + } + + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == DeTypeConstants.DE_TIME) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + whereValue = String.format(ImpalaConstants.WHERE_BETWEEN, startTime, endTime); + } else { + whereValue = String.format(ImpalaConstants.WHERE_BETWEEN, value.get(0), value.get(1)); + } + } else { + whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, value.get(0)); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null; + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } else { + split = "-"; + } + + if (StringUtils.isEmpty(dateStyle)) { + return "yyyy-MM-dd HH:mm:ss"; + } + + switch (dateStyle) { + case "y": + return "yyyy"; + case "y_M": + return "yyyy" + split + "MM"; + case "y_M_d": + return "yyyy" + split + "MM" + split + "dd"; + case "H_m_s": + return "HH:mm:ss"; + case "y_M_d_H_m": + return "yyyy" + split + "MM" + split + "dd" + " HH:mm"; + case "y_M_d_H_m_s": + return "yyyy" + split + "MM" + split + "dd" + " HH:mm:ss"; + default: + return "yyyy-MM-dd HH:mm:ss"; + } + } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + fieldName = String.format(ImpalaConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + fieldName = String.format(ImpalaConstants.DATE_FORMAT, originField, format); + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == DeTypeConstants.DE_STRING) { + fieldName = String.format(ImpalaConstants.DATE_FORMAT, originField, format); + } else { + String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000"; + String from_unixtime = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT); + fieldName = String.format(ImpalaConstants.DATE_FORMAT, from_unixtime, format); + } + } else { + fieldName = originField; + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getXWheres(ChartViewFieldDTO x, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) { + x.getFilter().forEach(f -> { + String whereName = ""; + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + if (x.getDeType() == DeTypeConstants.DE_TIME && x.getDeExtractType() != 1) { + String cast = String.format(ImpalaConstants.CAST, originField, ImpalaConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(ImpalaConstants.FROM_UNIXTIME, cast, ImpalaConstants.DEFAULT_DATE_FORMAT); + } else { + whereName = originField; + } + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + fieldName = ImpalaConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), originField); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format(ImpalaConstants.CAST, originField, y.getDeType() == 2 ? ImpalaConstants.DEFAULT_INT_FORMAT : ImpalaConstants.DEFAULT_FLOAT_FORMAT); + String agg = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), cast); + fieldName = String.format(ImpalaConstants.CAST, agg, ImpalaConstants.DEFAULT_FLOAT_FORMAT); + } else { + String cast = String.format(ImpalaConstants.CAST, originField, y.getDeType() == 2 ? ImpalaConstants.DEFAULT_INT_FORMAT : ImpalaConstants.DEFAULT_FLOAT_FORMAT); + fieldName = String.format(ImpalaConstants.AGG_FIELD, y.getSummary(), cast); + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private String getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(ImpalaConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + getLogic(y.getLogic()) + " ", strList) + ")" : null; + } + + private String calcFieldRegex(String originField, SQLObj tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } + return originField; + } + + private String sqlLimit(String sql, ChartViewWithBLOBs view) { + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + return sql + " LIMIT " + view.getResultCount(); + } else { + return sql; + } + } +} diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java deleted file mode 100644 index 14a7107f96..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.dataease.provider.query.mongodb; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.mongo; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class MongoConstants extends SQLConstants { - public static final String KEYWORD_TABLE = "%s"; - - public static final String KEYWORD_FIX = "%s." + mongo.getKeywordPrefix() + "%s" + mongo.getKeywordSuffix(); - - public static final String ALIAS_FIX = mongo.getAliasPrefix() + "%s" + mongo.getAliasSuffix(); - - public static final String toInt32 = "toInt32(%s)"; - - public static final String toDateTime = "toDateTime(%s)"; - - public static final String toInt64 = "toInt64(%s)"; - - public static final String toFloat64 = "toFloat64(%s)"; - - public static final String formatDateTime = "formatDateTime(%s,'%s')"; - - public static final String toDecimal = "toDecimal64(%s,2)"; - - public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java index 97f7bcf7f1..eec743a7eb 100644 --- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java @@ -11,8 +11,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.MongoConstants; +import io.dataease.plugins.common.constants.SQLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -29,7 +30,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java b/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java deleted file mode 100644 index 33b1e3d8ea..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.dataease.provider.query.mysql; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.mysql; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class MySQLConstants extends SQLConstants { - public static final String KEYWORD_TABLE = mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; - - public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')"; - - public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; - - public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; - - public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; - - public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java index 95f653fb12..96b7208690 100644 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java @@ -10,8 +10,9 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.MySQLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -28,7 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java deleted file mode 100644 index 013e51defc..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java +++ /dev/null @@ -1,55 +0,0 @@ -package io.dataease.provider.query.oracle; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.oracle; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class OracleConstants extends SQLConstants { - public static final String KEYWORD_TABLE = oracle.getKeywordPrefix() + "%s" + oracle.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + oracle.getKeywordPrefix() + "%s" + oracle.getKeywordSuffix(); - - public static final String ALIAS_FIX = oracle.getAliasPrefix() + "%s" + oracle.getAliasSuffix(); - - public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; - - public static final String DATE_FORMAT = "to_timestamp(%s,'%s')"; - - public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-DD HH24:MI:SS"; - - public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; - - public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; - - public static final String TO_NUMBER = "TO_NUMBER(%s)"; - - public static final String TO_DATE = "TO_DATE(%s,'%s')"; - - public static final String TO_CHAR = "TO_CHAR(%s,'%s')"; - - public static final String DEFAULT_START_DATE = "'1970-01-01 8:0:0'"; - - public static final String TO_MS = " * 24 * 60 * 60 * 100"; - - public static final String CALC_SUB = "%s - %s"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java index 48d04457c5..941c45aa1f 100644 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java @@ -13,8 +13,9 @@ import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.datasource.OracleConfiguration; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.OracleConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** * @Author gin diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java b/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java deleted file mode 100644 index d0e9f7cdeb..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.dataease.provider.query.pg; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.pg; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class PgConstants extends SQLConstants { - public static final String KEYWORD_TABLE = pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "floor(extract(epoch from(( %s - timestamp '1970-01-01 00:00:00')*1000))) "; - - public static final String DATE_FORMAT = "to_char(%s, '%s')"; - - public static final String FROM_UNIXTIME = "to_timestamp(%s)"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-DD HH24:MI:SS"; - - public static final String DEFAULT_INT_FORMAT = "numeric(18,0)"; - - public static final String DEFAULT_FLOAT_FORMAT = "numeric(18,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; - - -} diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java index b231d74536..6b970abb72 100644 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java @@ -13,9 +13,10 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.PgConstants; +import io.dataease.plugins.common.constants.SqlServerSQLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; -import io.dataease.provider.query.sqlserver.SqlServerSQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -32,7 +33,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; @Service("pgQuery") diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java deleted file mode 100644 index f1ba7179aa..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java +++ /dev/null @@ -1,49 +0,0 @@ -package io.dataease.provider.query.redshift; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.pg; - -/** - * Redshift 静态变量 - * - * @className: RedshiftConstants - * @description: Redshift 静态变量 - * @author: Jiantao Yan - * @date: 2021/10/11 17:12 - **/ -public class RedshiftConstants extends SQLConstants { - public static final String KEYWORD_TABLE = pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "floor(extract(epoch from(( %s - timestamp '1970-01-01 00:00:00')*1000))) "; - - public static final String DATE_FORMAT = "to_char(%s, %s)"; - - public static final String FROM_UNIXTIME = "to_timestamp(%s)"; - - public static final String TO_DATE = "to_date(%s,'%s')"; - - public static final String CAST = "CAST(%s AS %s)"; - - public static final String DEFAULT_DATE_FORMAT = "'YYYY-MM-DD HH24:MI:SS'"; - - public static final String DEFAULT_INT_FORMAT = "numeric(18,0)"; - - public static final String DEFAULT_FLOAT_FORMAT = "numeric(18,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; - - -} diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java index 3067caaf8e..75e142e5ed 100644 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java @@ -13,10 +13,11 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.RedshiftConstants; +import io.dataease.plugins.common.constants.SqlServerSQLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; -import io.dataease.provider.query.pg.PgConstants; -import io.dataease.provider.query.sqlserver.SqlServerSQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; +import io.dataease.plugins.common.constants.PgConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -33,7 +34,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; /** diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java deleted file mode 100644 index ac5862abfa..0000000000 --- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.dataease.provider.query.sqlserver; - -import io.dataease.provider.SQLConstants; - -import static io.dataease.commons.constants.DatasourceTypes.sqlServer; - -/** - * @Author gin - * @Date 2021/7/8 7:22 下午 - */ -public class SqlServerSQLConstants extends SQLConstants { - public static final String KEYWORD_TABLE = sqlServer.getKeywordPrefix() + "%s" + sqlServer.getKeywordSuffix(); - - public static final String KEYWORD_FIX = "%s." + sqlServer.getKeywordPrefix() + "%s" + sqlServer.getKeywordSuffix(); - - public static final String UNIX_TIMESTAMP = "CAST(DATEDIFF(ss,'1970-01-01 08:00:00', %s) as bigint ) * 1000 "; - - public static final String DATE_FORMAT = "CONVERT(varchar(100), %s, %s)"; - - public static final String FROM_UNIXTIME = "convert(varchar, %s ,120)"; - - public static final String CONVERT = "CONVERT(%s, %s)"; - - public static final String LONG_TO_DATE = "DATEADD(second,%s,'1970-01-01 08:00:00')"; - - public static final String STRING_TO_DATE = "CONVERT(datetime, %s ,120)"; - - public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; - - public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; - - public static final String WHERE_VALUE_NULL = "(NULL,'')"; - - public static final String WHERE_VALUE_VALUE = "'%s'"; - - public static final String AGG_COUNT = "COUNT(*)"; - - public static final String AGG_FIELD = "%s(%s)"; - - public static final String WHERE_BETWEEN = "'%s' AND '%s'"; - - public static final String BRACKETS = "(%s)"; -} diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java index 2718468fdf..c6cc4bcbec 100644 --- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java @@ -13,8 +13,9 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.datasource.JdbcConfiguration; import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.plugins.common.constants.SqlServerSQLConstants; import io.dataease.provider.QueryProvider; -import io.dataease.provider.SQLConstants; +import io.dataease.plugins.common.constants.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -31,7 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; @Service("sqlserverQuery") public class SqlserverQueryProvider extends QueryProvider { diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index b766f250b7..b7e8457839 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -13,8 +13,10 @@ import io.dataease.base.mapper.ext.ExtChartViewMapper; import io.dataease.commons.constants.ColumnPermissionConstants; import io.dataease.commons.constants.CommonConstants; import io.dataease.commons.constants.JdbcConstants; +import io.dataease.commons.exception.DEException; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; +import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.LogUtil; import io.dataease.controller.request.chart.*; import io.dataease.controller.request.datasource.DatasourceRequest; @@ -27,6 +29,9 @@ import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; import io.dataease.listener.util.CacheUtils; +import io.dataease.plugins.config.SpringContextUtil; +import io.dataease.plugins.view.entity.*; +import io.dataease.plugins.view.service.ViewPluginService; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.QueryProvider; @@ -59,6 +64,9 @@ import java.util.stream.Collectors; */ @Service public class ChartViewService { + + private static Gson gson = new Gson(); + @Resource private ChartViewMapper chartViewMapper; @Resource @@ -292,25 +300,6 @@ public class ChartViewService { } -// private void checkPermissions(List chartViewFieldDTOS, List fields, List desensitizationList, Boolean alowDesensitization) throws Exception { -// String filedName = ""; -// for (ChartViewFieldBaseDTO chartViewFieldDTO : chartViewFieldDTOS) { -// if (alowDesensitization) { -// if (!fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) { -// filedName = filedName + chartViewFieldDTO.getName() + " ,"; -// } -// } else { -// if (desensitizationList.contains(chartViewFieldDTO.getDataeaseName()) || !fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) { -// filedName = filedName + chartViewFieldDTO.getName() + " ,"; -// } -// } -// } -// filedName = filedName.endsWith(",") ? filedName.substring(0, filedName.length() - 1) : filedName; -// if (StringUtils.isNotEmpty(filedName)) { -// throw new Exception("以下字段没有权限: " + filedName); -// } -// } - public ChartViewDTO calcData(ChartViewDTO view, ChartExtRequest requestList, boolean cache) throws Exception { if (ObjectUtils.isEmpty(view)) { throw new RuntimeException(Translator.get("i18n_chart_delete")); @@ -499,16 +488,41 @@ public class ChartViewService { // 判断连接方式，直连或者定时抽取 table.mode DatasourceRequest datasourceRequest = new DatasourceRequest(); + Datasource ds = table.getMode() == 0 ? datasourceService.get(table.getDataSourceId()) : engineService.getDeEngine(); + datasourceRequest.setDatasource(ds); + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); List data = new ArrayList<>(); + + + // 如果是插件视图 走插件内部的逻辑 + if (view.getIsPlugin()) { + Map> fieldMap = new HashMap<>(); + fieldMap.put("xAxis",xAxis); + fieldMap.put("yAxis",yAxis); + fieldMap.put("extStack",extStack); + fieldMap.put("extBubble",extBubble); + PluginViewParam pluginViewParam = buildPluginParam(fieldMap, fieldCustomFilter, extFilterList, ds, table, view); + String sql = pluginViewSql(pluginViewParam, view); + datasourceRequest.setQuery(sql); + data = datasourceProvider.getData(datasourceRequest); + + Map mapChart = pluginViewResult(pluginViewParam, view, data, isDrill); + Map mapTableNormal = ChartDataBuild.transTableNormal(xAxis, yAxis, view, data, extStack, desensitizationList); + + return uniteViewResult(datasourceRequest.getQuery(), mapChart, mapTableNormal,view, isDrill, drillFilters); + // 如果是插件到此结束 + } + + //如果不是插件视图 走原生逻辑 if (table.getMode() == 0) {// 直连 - Datasource ds = datasourceService.get(table.getDataSourceId()); + // Datasource ds = datasourceService.get(table.getDataSourceId()); if (ObjectUtils.isEmpty(ds)) { throw new RuntimeException(Translator.get("i18n_datasource_delete")); } if (StringUtils.isNotEmpty(ds.getStatus()) && ds.getStatus().equalsIgnoreCase("Error")) { throw new Exception(Translator.get("i18n_invalid_ds")); } - DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + // DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); datasourceRequest.setDatasource(ds); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); @@ -572,8 +586,8 @@ public class ChartViewService { data = datasourceProvider.getData(datasourceRequest); } else if (table.getMode() == 1) {// 抽取 // 连接doris，构建doris数据源查询 - Datasource ds = engineService.getDeEngine(); - DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + // Datasource ds = engineService.getDeEngine(); + // DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); datasourceRequest.setDatasource(ds); String tableName = "ds_" + table.getId().replaceAll("-", "_"); datasourceRequest.setTable(tableName); @@ -732,12 +746,16 @@ public class ChartViewService { mapChart = ChartDataBuild.transChartDataAntV(xAxis, yAxis, view, data, isDrill); } } - // table组件，明细表，也用于导出数据 Map mapTableNormal = ChartDataBuild.transTableNormal(xAxis, yAxis, view, data, extStack, desensitizationList); + return uniteViewResult(datasourceRequest.getQuery(), mapChart, mapTableNormal,view, isDrill, drillFilters); + } - map.putAll(mapChart); - map.putAll(mapTableNormal); + public ChartViewDTO uniteViewResult(String sql, Map chartData, Map tabelData, ChartViewDTO view, Boolean isDrill, List drillFilters) { + + Map map = new HashMap<>(); + map.putAll(chartData); + map.putAll(tabelData); List sourceFields = dataSetTableFieldsService.getFieldsByTableId(view.getTableId()); map.put("sourceFields", sourceFields); @@ -745,13 +763,65 @@ public class ChartViewService { ChartViewDTO dto = new ChartViewDTO(); BeanUtils.copyBean(dto, view); dto.setData(map); - dto.setSql(datasourceRequest.getQuery()); - + dto.setSql(sql); dto.setDrill(isDrill); dto.setDrillFilters(drillFilters); return dto; } + private PluginViewParam buildPluginParam(Map> fieldMap, List customFilters, List extFilters, Datasource ds, DatasetTable table, ChartViewDTO view) { + PluginViewParam pluginViewParam = new PluginViewParam(); + PluginViewSet pluginViewSet = BeanUtils.copyBean(new PluginViewSet(), table); + pluginViewSet.setDsType(ds.getType()); + PluginViewLimit pluginViewLimit = BeanUtils.copyBean(new PluginViewLimit(), view); + + List fieldFilters = customFilters.stream().map(filter -> BeanUtils.copyBean(new PluginChartFieldCustomFilter(), filter)).collect(Collectors.toList()); + List panelFilters = extFilters.stream().map(filter -> BeanUtils.copyBean(new PluginChartExtFilter(), filter)).collect(Collectors.toList()); + + List pluginViewFields = fieldMap.entrySet().stream().flatMap(entry -> entry.getValue().stream().map(field -> { + PluginViewField pluginViewField = BeanUtils.copyBean(new PluginViewField(), field); + pluginViewField.setTypeField(entry.getKey()); + return pluginViewField; + })).collect(Collectors.toList()); + pluginViewParam.setPluginViewSet(pluginViewSet); + pluginViewParam.setPluginViewFields(pluginViewFields); + pluginViewParam.setPluginChartFieldCustomFilters(fieldFilters); + pluginViewParam.setPluginChartExtFilters(panelFilters); + pluginViewParam.setPluginViewLimit(pluginViewLimit); + pluginViewParam.setUserId(AuthUtils.getUser().getUserId()); + return pluginViewParam; + } + + private ViewPluginService getPluginService(String viewType) { + Map beanMap = SpringContextUtil.getApplicationContext().getBeansOfType(ViewPluginService.class); + + if (beanMap.keySet().size() == 0) { + DEException.throwException("没有此插件"); + + } + ViewPluginService viewPluginService = null; + for (Map.Entry entry : beanMap.entrySet()) { + if (StringUtils.equals(entry.getValue().viewType().getValue(), viewType)) { + viewPluginService = entry.getValue(); + return viewPluginService; + } + } + if (null == viewPluginService) DEException.throwException("没有此插件"); + return viewPluginService; + } + + private String pluginViewSql(PluginViewParam param, ChartViewDTO view) { + ViewPluginService viewPluginService = getPluginService(view.getType()); + String sql = viewPluginService.generateSQL(param); + return sql; + } + + private Map pluginViewResult(PluginViewParam param, ChartViewDTO view, List args, Boolean isDrill) { + ViewPluginService viewPluginService = getPluginService(view.getType()); + Map result = viewPluginService.formatResult(param, args, isDrill); + return result; + } + private ChartViewDTO emptyChartViewDTO(ChartViewDTO view) { ChartViewDTO dto = new ChartViewDTO(); BeanUtils.copyBean(dto, view); diff --git a/backend/src/main/java/io/dataease/service/chart/ViewPluginBaseServiceImpl.java b/backend/src/main/java/io/dataease/service/chart/ViewPluginBaseServiceImpl.java new file mode 100644 index 0000000000..651739ded1 --- /dev/null +++ b/backend/src/main/java/io/dataease/service/chart/ViewPluginBaseServiceImpl.java @@ -0,0 +1,506 @@ +package io.dataease.service.chart; + +import com.google.gson.Gson; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.dto.dataset.DataSetTableUnionDTO; +import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.plugins.common.constants.SQLConstants; +import io.dataease.plugins.common.util.BeanUtils; +import io.dataease.plugins.common.util.ConstantsUtil; +import io.dataease.plugins.view.entity.*; +import io.dataease.plugins.view.service.ViewPluginBaseService; +import io.dataease.service.dataset.DataSetTableService; +import io.dataease.service.dataset.DataSetTableUnionService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static io.dataease.plugins.common.constants.SQLConstants.TABLE_ALIAS_PREFIX; + +@Service +public class ViewPluginBaseServiceImpl implements ViewPluginBaseService { + + + private static final Integer STRING = 0; + private static final Integer TIME = 1; + private static final Integer INT = 2; + private static final Integer FLOAT = 3; + /*private static final Integer BOOLEAN = 4;*/ + + @Resource + private DataSetTableUnionService dataSetTableUnionService; + + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Resource + private DataSetTableService dataSetTableService; + + + @Override + public PluginSingleField buildField(String dsType, PluginViewField pluginViewField, PluginViewSQL tableObj, int index) { + PluginSingleField result = new PluginSingleField(); + String FIELD_ALIAS_PREFIX = StringUtils.equals(pluginViewField.getTypeField(), "xAxis") ? SQLConstants.FIELD_ALIAS_X_PREFIX : SQLConstants.FIELD_ALIAS_Y_PREFIX; + + String originField = getOriginName(dsType, pluginViewField, tableObj); + + PluginViewSQL field = null; + String where = null; + String alias_fix = ConstantsUtil.constantsValue(dsType, "ALIAS_FIX"); + String fieldAlias = String.format(alias_fix, String.format(FIELD_ALIAS_PREFIX, index)); + if (StringUtils.equals(pluginViewField.getTypeField(), "xAxis") || StringUtils.equals(pluginViewField.getTypeField(), "extStack")) { + field = getXFields(dsType, pluginViewField, originField, fieldAlias); + + }else { + field = getYFields(dsType, pluginViewField, originField, fieldAlias); + where = getYWheres(dsType, pluginViewField, fieldAlias); + } + PluginViewSQL sort = addSort(pluginViewField.getSort(), originField, fieldAlias); + + Optional.ofNullable(field).ifPresent(f -> result.setField(f)); + Optional.ofNullable(sort).ifPresent(s -> result.setSort(s)); + Optional.ofNullable(where).ifPresent(w -> result.setWhere(w)); + return result; + } + + @Override + public String customWhere(String dsType, List list, PluginViewSQL pluginViewSQL) { + return transCustomFilterList(dsType, pluginViewSQL, list); + } + + @Override + public String panelWhere(String dsType, List list, PluginViewSQL pluginViewSQL) { + return transExtFilterList(dsType, pluginViewSQL, list); + } + + @Override + public PluginViewSQL getTableObj(PluginViewSet pluginViewSet) { + String tableName = null; + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(pluginViewSet.getInfo(), DataTableInfoDTO.class); + switch (pluginViewSet.getType()) { + case "db": + tableName = dataTableInfoDTO.getTable(); + break; + case "sql": + tableName = dataTableInfoDTO.getSql(); + break; + case "custom": + List list = dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()); + Datasource ds = new Datasource(); + ds.setType(pluginViewSet.getDsType()); + tableName = dataSetTableService.getCustomSQLDatasource(dataTableInfoDTO, list, ds); + break; + case "union": + Datasource datasource = new Datasource(); + datasource.setType(pluginViewSet.getDsType()); + Map sqlMap = dataSetTableService.getUnionSQLDatasource(dataTableInfoDTO, datasource); + tableName = (String) sqlMap.get("sql"); + break; + default: + break; + } + + String keyword = ConstantsUtil.constantsValue(pluginViewSet.getDsType(), "KEYWORD_TABLE"); + String tabelName = (tableName.startsWith("(") && tableName.endsWith(")")) ? tableName : String.format(keyword, tableName); + String tabelAlias = String.format(TABLE_ALIAS_PREFIX, 0); + PluginViewSQL tableObj = PluginViewSQL.builder().tableName(tabelName).tableAlias(tabelAlias).build(); + return tableObj; + } + + private String getOriginName(String dsType, PluginViewField pluginViewField, PluginViewSQL tableObj) { + String keyword_fix = ConstantsUtil.constantsValue(dsType, "KEYWORD_FIX"); + String originField; + if (ObjectUtils.isNotEmpty(pluginViewField.getExtField()) && pluginViewField.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(dsType,pluginViewField.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(pluginViewField.getExtField()) && pluginViewField.getExtField() == 1) { + originField = String.format(keyword_fix, tableObj.getTableAlias(), pluginViewField.getOriginName()); + } else { + originField = String.format(keyword_fix, tableObj.getTableAlias(), pluginViewField.getOriginName()); + } + return originField; + } + + private String calcFieldRegex(String dsType, String originField, PluginViewSQL tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + String keyword_fix = ConstantsUtil.constantsValue(dsType, "KEYWORD_FIX"); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", String.format(keyword_fix, tableObj.getTableAlias(), ele.getOriginName())); + } + return originField; + } + + + + + private PluginViewSQL getXFields(String dsType, PluginViewField x, String originField, String fieldAlias) { + + String fieldName = ""; + String calc_sub = ConstantsUtil.constantsValue(dsType, "CALC_SUB"); + String to_date = ConstantsUtil.constantsValue(dsType, "TO_DATE"); + String default_start_date = ConstantsUtil.constantsValue(dsType, "DEFAULT_START_DATE"); + String default_date_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_DATE_FORMAT"); + String to_number = ConstantsUtil.constantsValue(dsType, "TO_NUMBER"); + String to_ms = ConstantsUtil.constantsValue(dsType, "TO_MS"); + String to_char = ConstantsUtil.constantsValue(dsType, "TO_CHAR"); + String default_start_date_c = ConstantsUtil.constantsValue(dsType, "DEFAULT_START_DATE"); + if (x.getDeExtractType() == TIME) { + if (x.getDeType() == INT || x.getDeType() == FLOAT) { //时间转数值 + if (x.getType().equalsIgnoreCase("DATE")) { + String date = String.format(calc_sub, originField, String.format(to_date, default_start_date, default_date_format)); + fieldName = String.format(to_number, date) + to_ms; + } else { + String toChar = String.format(to_char, originField, default_date_format); + String toDate = String.format(to_date, toChar, default_date_format); + String toDate1 = String.format(to_date, default_start_date_c, default_date_format); + fieldName = String.format(to_number, String.format(calc_sub, toDate, toDate1)) + to_ms; + } + } else if (x.getDeType() == TIME) { //格式化显示时间 + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getType().equalsIgnoreCase("DATE")) { + fieldName = String.format(to_char, originField, format); + } else { + String toChar = String.format(to_char, originField, default_date_format); + String toDate = String.format(to_date, toChar, default_date_format); + fieldName = String.format(to_char, toDate, format); + } + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == STRING) { //字符串转时间 + String toDate = String.format(to_date, originField, default_date_format); + fieldName = String.format(to_char, toDate, format); + } else { //数值转时间 + String date = originField + "/(1000 * 60 * 60 * 24)+" + String.format(to_date, default_start_date_c, default_date_format); + fieldName = String.format(to_char, date, format); + } + } else { + fieldName = originField; + } + } + return PluginViewSQL.builder().fieldName(fieldName).fieldAlias(fieldAlias).build(); + } + + private PluginViewSQL getYFields(String dsType, PluginViewField y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + + fieldName = ConstantsUtil.constantsValue(dsType, "AGG_COUNT"); + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(ConstantsUtil.constantsValue(dsType, "AGG_FIELD"), y.getSummary(), originField); + } else { + String cast_constants = ConstantsUtil.constantsValue(dsType, "CAST"); + String agg_field_constants = ConstantsUtil.constantsValue(dsType, "AGG_FIELD"); + String default_int_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_INT_FORMAT"); + String default_float_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_FLOAT_FORMAT"); + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format(cast_constants, originField, y.getDeType() == 2 ? default_int_format : default_float_format); + String agg = String.format(agg_field_constants, y.getSummary(), cast); + fieldName = String.format(cast_constants, agg, ConstantsUtil.constantsValue(dsType, "DEFAULT_FLOAT_FORMAT")); + } else { + String cast = String.format(cast_constants, originField, y.getDeType() == 2 ? default_int_format : default_float_format); + fieldName = String.format(agg_field_constants, y.getSummary(), cast); + } + } + return PluginViewSQL.builder().fieldName(fieldName).fieldAlias(fieldAlias).build(); + } + + private PluginViewSQL addSort(String sort, String originField, String fieldAlias) { + if (StringUtils.isNotEmpty(sort) && !StringUtils.equalsIgnoreCase(sort, "none")) { + return PluginViewSQL.builder().orderField(originField).orderAlias(fieldAlias).orderDirection(sort).build(); + } + return null; + } + + private String getYWheres(String dsType, PluginViewField y,String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + String where_value_value = ConstantsUtil.constantsValue(dsType, "WHERE_VALUE_VALUE"); + whereValue = String.format(where_value_value, f.getValue()); + } + list.add(PluginViewSQL.builder().whereField(fieldAlias).whereAlias(fieldAlias).whereTermAndValue(whereTerm + whereValue).build()); + }); + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + getLogic(y.getLogic()) + " ", strList) + ")" : null; + } + + public String getLogic(String logic) { + if (logic != null) { + switch (logic) { + case "and": + return "AND"; + case "or": + return "OR"; + } + } + return "AND"; + } + + private String transDateFormat(String dateStyle, String datePattern) { + + return null; + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + case "empty": + return " = "; + case "not_empty": + return " <> "; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public String transCustomFilterList(String dsType, PluginViewSQL tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List res = new ArrayList<>(); + String to_date = ConstantsUtil.constantsValue(dsType, "TO_DATE"); + String default_date_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_DATE_FORMAT"); + String cast_c = ConstantsUtil.constantsValue(dsType, "CAST"); + String default_int_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_INT_FORMAT"); + String from_unixtime = ConstantsUtil.constantsValue(dsType, "FROM_UNIXTIME"); + String default_float_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_FLOAT_FORMAT"); + String unix_timestamp = ConstantsUtil.constantsValue(dsType, "UNIX_TIMESTAMP"); + String where_value_value = ConstantsUtil.constantsValue(dsType, "WHERE_VALUE_VALUE"); + for (PluginChartFieldCustomFilter request : requestList) { + List list = new ArrayList<>(); + PluginDatasetTableField field = request.getField(); + + if (ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + + PluginViewField pluginViewField = BeanUtils.copyBean(new PluginViewField(), field); + String originName = getOriginName(dsType, pluginViewField, tableObj); + + + + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(to_date, originName, default_date_format); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(cast_c, originName, default_int_format) + "/1000"; + whereName = String.format(from_unixtime, cast, default_float_format); + } + if (field.getDeExtractType() == 1) { + whereName = originName; + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(cast_c, originName,default_float_format); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(unix_timestamp, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else { + whereName = originName; + } + + if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) { + if (CollectionUtils.isNotEmpty(request.getEnumCheckField())) { + res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))"); + } + } else { + List filter = request.getFilter(); + for (PluginChartCustomFilterItem filterItemDTO : filter) { + String value = filterItemDTO.getValue(); + String whereTerm = transMysqlFilterTerm(filterItemDTO.getTerm()); + String whereValue = ""; + + if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) { + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + if (field.getDeType() == 1) { + whereValue = String.format(to_date, "'" + value + "'", default_date_format); + } else { + whereValue = String.format(where_value_value, value); + } + } + list.add(PluginViewSQL.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + if (CollectionUtils.isNotEmpty(list)) { + res.add("(" + String.join(" " + getLogic(request.getLogic()) + " ", strList) + ")"); + } + } + } + return CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null; + } + + public String transExtFilterList(String dsType, PluginViewSQL tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + String where_between = ConstantsUtil.constantsValue(dsType, "WHERE_BETWEEN"); + String default_date_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_DATE_FORMAT"); + String to_date = ConstantsUtil.constantsValue(dsType, "TO_DATE"); + String cast_c = ConstantsUtil.constantsValue(dsType, "CAST"); + String from_unixtime = ConstantsUtil.constantsValue(dsType, "FROM_UNIXTIME"); + String default_int_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_INT_FORMAT"); + String default_float_format = ConstantsUtil.constantsValue(dsType, "DEFAULT_FLOAT_FORMAT"); + String unix_timestamp = ConstantsUtil.constantsValue(dsType, "UNIX_TIMESTAMP"); + String where_value_value = ConstantsUtil.constantsValue(dsType, "WHERE_VALUE_VALUE"); + + for (PluginChartExtFilter request : requestList) { + List value = request.getValue(); + PluginDatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName = getOriginName(dsType, BeanUtils.copyBean(new PluginViewField(), field), tableObj); + + + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(to_date, originName, default_date_format); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(cast_c, originName, default_int_format) + "/1000"; + whereName = String.format(from_unixtime, cast, default_date_format); + } + if (field.getDeExtractType() == 1) { + whereName = originName; + } + } else if (field.getDeType() == 2 || field.getDeType() == 3) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(cast_c, originName, default_float_format); + } + if (field.getDeExtractType() == 1) { + whereName = String.format(unix_timestamp, originName) + "*1000"; + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + whereName = originName; + } + } else { + whereName = originName; + } + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == 1) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + String st = String.format(to_date, "'" + startTime + "'", default_date_format); + String et = String.format(to_date, "'" + endTime + "'", default_date_format); + whereValue = st + " AND " + et; + } else { + whereValue = String.format(where_between, value.get(0), value.get(1)); + } + } else { + whereValue = String.format(where_value_value, value.get(0)); + } + list.add(PluginViewSQL.builder().whereField(whereName).whereTermAndValue(whereTerm + whereValue).build()); + + + + } + List strList = new ArrayList<>(); + list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue())); + return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null; + } +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 67b2d1be13..cb32136baa 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -101,6 +101,7 @@ public class DataSetTableService { @Resource private EngineService engineService; + private static boolean isUpdatingDatasetTableStatus = false; private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; @@ -2182,15 +2183,30 @@ public class DataSetTableService { private UtilMapper utilMapper; public void updateDatasetTableStatus() { + if(this.isUpdatingDatasetTableStatus){ + return; + }else { + this.isUpdatingDatasetTableStatus = true; + } + + try { + doUpdate(); + }catch (Exception e){} + finally { + this.isUpdatingDatasetTableStatus = false; + } + } + + private void doUpdate(){ List qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); List activeQrtzInstances = qrtzSchedulerStates.stream() .filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()) .map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); List jobStoppeddDatasetTables = new ArrayList<>(); + DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); - datasetTableMapper.selectByExample(example).forEach(datasetTable -> { if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains( datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) { @@ -2202,6 +2218,7 @@ public class DataSetTableService { return; } + //DatasetTable DatasetTable record = new DatasetTable(); record.setSyncStatus(JobStatus.Error.name()); example.clear(); @@ -2209,6 +2226,14 @@ public class DataSetTableService { .andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); datasetTableMapper.updateByExampleSelective(record, example); + //Task + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria(); + criteria.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())).andStatusEqualTo(JobStatus.Underway.name()); + List datasetTableTasks = dataSetTableTaskService.list(datasetTableTaskExample); + dataSetTableTaskService.updateTaskStatus(datasetTableTasks, JobStatus.Error); + + //TaskLog DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setInfo("Job stopped due to system error."); @@ -2216,19 +2241,14 @@ public class DataSetTableService { DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()) .andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); - List taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream() - .map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList()); datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); - dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error); - for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) { extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("incremental_delete", jobStoppeddDatasetTable.getId()); } } - /* * 判断数组中是否有重复的值 */ diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index 0092483cc4..afd8fcabd7 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -161,13 +161,13 @@ public class DataSetTableTaskService { return datasetTableTaskMapper.selectByPrimaryKey(id); } - public void updateTaskStatus(List taskIds, JobStatus lastExecStatus) { - if (CollectionUtils.isEmpty(taskIds)) { - return; - } - DatasetTableTaskExample example = new DatasetTableTaskExample(); - example.createCriteria().andIdIn(taskIds); - List datasetTableTasks = datasetTableTaskMapper.selectByExample(example); + + public List list(DatasetTableTaskExample example) { + return datasetTableTaskMapper.selectByExample(example); + } + + + public void updateTaskStatus(List datasetTableTasks, JobStatus lastExecStatus) { for (DatasetTableTask tableTask : datasetTableTasks) { updateTaskStatus(tableTask, lastExecStatus); } @@ -202,7 +202,7 @@ public class DataSetTableTaskService { if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) { datasetTableTask.setStatus(TaskStatus.Stopped.name()); } else { - datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId()); +// datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId()); datasetTableTask.setLastExecStatus(lastExecStatus.name()); if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) { BaseGridRequest request = new BaseGridRequest(); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index e91982aa8d..a5e847554a 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -140,7 +140,7 @@ public class ExtractDataService { datasetTableTaskLog.setTableId(datasetTable.getId()); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - return !CollectionUtils.isNotEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name()); + return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name()); } else { datasetTableTask.setLastExecTime(startTime); datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); @@ -159,7 +159,7 @@ public class ExtractDataService { return; } UpdateType updateType = UpdateType.valueOf(type); - DatasetTableTaskLog datasetTableTaskLog; + if (datasetTableFields == null) { datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build()); } @@ -174,10 +174,10 @@ public class ExtractDataService { return o1.getColumnIndex().compareTo(o2.getColumnIndex()); }); + DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); switch (updateType) { case all_scope: // 全量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields); createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields); Long execTime = System.currentTimeMillis(); @@ -222,7 +222,7 @@ public class ExtractDataService { toDelete.forEach(datasetTableField -> dataSetTableFieldsService.delete(datasetTableField.getId())); } } catch (Exception e) { - saveErrorLog(datasetTableId, null, e); + saveErrorLog(datasetTableTaskLog, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); } finally { @@ -233,7 +233,6 @@ public class ExtractDataService { case add_scope: // 增量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); Long execTime = System.currentTimeMillis(); if (!engineService.isSimpleMode()) { generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); @@ -245,7 +244,7 @@ public class ExtractDataService { saveSuccessLog(datasetTableTaskLog); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } catch (Exception e) { - saveErrorLog(datasetTableId, null, e); + saveErrorLog(datasetTableTaskLog, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); } finally { deleteFile("incremental_add", datasetTableId); @@ -312,7 +311,7 @@ public class ExtractDataService { msg = true; lastExecStatus = JobStatus.Completed; } catch (Exception e) { - saveErrorLog(datasetTableId, taskId, e); + saveErrorLog(datasetTableTaskLog, e); msg = false; lastExecStatus = JobStatus.Error; execTime = null; @@ -372,7 +371,7 @@ public class ExtractDataService { msg = true; lastExecStatus = JobStatus.Completed; } catch (Exception e) { - saveErrorLog(datasetTableId, taskId, e); + saveErrorLog(datasetTableTaskLog, e); msg = false; lastExecStatus = JobStatus.Error; execTime = null; @@ -592,26 +591,14 @@ public class ExtractDataService { dataSetTableTaskLogService.save(datasetTableTaskLog); } - private void saveErrorLog(String datasetTableId, String taskId, Exception e) { - LogUtil.error("Extract data error: " + datasetTableId, e); - DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); - datasetTableTaskLog.setTableId(datasetTableId); - datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - if (StringUtils.isNotEmpty(taskId)) { - datasetTableTaskLog.setTaskId(taskId); - } - List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { - datasetTableTaskLog = datasetTableTaskLogs.get(0); - datasetTableTaskLog.setStatus(JobStatus.Error.name()); - datasetTableTaskLog.setInfo(e.getMessage()); - datasetTableTaskLog.setEndTime(System.currentTimeMillis()); - dataSetTableTaskLogService.save(datasetTableTaskLog); - } - + private void saveErrorLog(DatasetTableTaskLog datasetTableTaskLog, Exception e) { + LogUtil.error("Extract data error: " + datasetTableTaskLog.getTaskId(), e); + datasetTableTaskLog.setStatus(JobStatus.Error.name()); + datasetTableTaskLog.setInfo(e.getMessage()); + datasetTableTaskLog.setEndTime(System.currentTimeMillis()); + dataSetTableTaskLogService.save(datasetTableTaskLog); } - private void createEngineTable(String tableName, List datasetTableFields) throws Exception { Datasource engine = engineService.getDeEngine(); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); @@ -671,14 +658,9 @@ public class ExtractDataService { datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setTriggerType(TriggerType.Cron.name()); - List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if (CollectionUtils.isEmpty(datasetTableTaskLogs)) { - datasetTableTaskLog.setStartTime(System.currentTimeMillis()); - dataSetTableTaskLogService.save(datasetTableTaskLog); - return datasetTableTaskLog; - } else { - return datasetTableTaskLogs.get(0); - } + datasetTableTaskLog.setStartTime(System.currentTimeMillis()); + dataSetTableTaskLogService.save(datasetTableTaskLog); + return datasetTableTaskLog; } private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId, Long startTime) { @@ -687,7 +669,7 @@ public class ExtractDataService { datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 5; i++) { List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { return datasetTableTaskLogs.get(0); diff --git a/drivers/ImpalaJDBC41.jar b/drivers/ImpalaJDBC41.jar new file mode 100644 index 0000000000..4c0c6b063f Binary files /dev/null and b/drivers/ImpalaJDBC41.jar differ diff --git a/frontend/src/api/dataset/dataset.js b/frontend/src/api/dataset/dataset.js index 4eca52a04d..920baa29fe 100644 --- a/frontend/src/api/dataset/dataset.js +++ b/frontend/src/api/dataset/dataset.js @@ -213,5 +213,5 @@ export function checkCustomDs() { loading: true }) } - +export const disabledSyncDs= ['es', 'ck', 'mongo', 'redshift', 'hive', 'impala'] export default { loadTable, getScene, addGroup, delGroup, addTable, delTable, groupTree, checkCustomDs } diff --git a/frontend/src/api/system/template.js b/frontend/src/api/system/template.js index 5b83b15bcb..3e17a9be40 100644 --- a/frontend/src/api/system/template.js +++ b/frontend/src/api/system/template.js @@ -4,7 +4,8 @@ export function save(data) { return request({ url: '/template/save', data: data, - method: 'post' + method: 'post', + loading: true }) } export function templateDelete(id) { diff --git a/frontend/src/icons/svg/panel-mobile.svg b/frontend/src/icons/svg/panel-mobile.svg new file mode 100644 index 0000000000..99ce7a434a --- /dev/null +++ b/frontend/src/icons/svg/panel-mobile.svg @@ -0,0 +1,18 @@ + + + + + + + + Layer 1 + + + + + + + + + + \ No newline at end of file diff --git a/frontend/src/views/chart/components/normal/LabelNormalText.vue b/frontend/src/views/chart/components/normal/LabelNormalText.vue index 9ee8d1b7dd..a702c9ff47 100644 --- a/frontend/src/views/chart/components/normal/LabelNormalText.vue +++ b/frontend/src/views/chart/components/normal/LabelNormalText.vue @@ -11,12 +11,11 @@ {{ chart.data.series[0].data[0] }}

- - - - - - + +

+ {{ chart.data.series[0].name }} +

+ diff --git a/frontend/src/views/chart/components/shape-attr/ColorSelector.vue b/frontend/src/views/chart/components/shape-attr/ColorSelector.vue index e90d82139e..d9e775764f 100644 --- a/frontend/src/views/chart/components/shape-attr/ColorSelector.vue +++ b/frontend/src/views/chart/components/shape-attr/ColorSelector.vue @@ -48,7 +48,7 @@ - + diff --git a/frontend/src/views/chart/components/shape-attr/SizeSelector.vue b/frontend/src/views/chart/components/shape-attr/SizeSelector.vue index 7b9dfafa57..3a9c20a990 100644 --- a/frontend/src/views/chart/components/shape-attr/SizeSelector.vue +++ b/frontend/src/views/chart/components/shape-attr/SizeSelector.vue @@ -133,15 +133,15 @@ - + {{ $t('chart.show') }} - + - + diff --git a/frontend/src/views/chart/components/shape-attr/SizeSelectorAntV.vue b/frontend/src/views/chart/components/shape-attr/SizeSelectorAntV.vue index 8138e1105b..a911e7ea4d 100644 --- a/frontend/src/views/chart/components/shape-attr/SizeSelectorAntV.vue +++ b/frontend/src/views/chart/components/shape-attr/SizeSelectorAntV.vue @@ -159,15 +159,15 @@ - + {{ $t('chart.show') }} - + - + diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 6d980231ac..0107943790 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -2882,11 +2882,11 @@ span { } ::v-deep .el-slider__input { - width: 80px !important; + width: 100px !important; } ::v-deep .el-input-number--mini { - width: 80px !important; + width: 100px !important; } .no-senior { diff --git a/frontend/src/views/dataset/add/AddDB.vue b/frontend/src/views/dataset/add/AddDB.vue index d466b35828..a3b89cfe0b 100644 --- a/frontend/src/views/dataset/add/AddDB.vue +++ b/frontend/src/views/dataset/add/AddDB.vue @@ -69,7 +69,7 @@