diff --git a/backend/pom.xml b/backend/pom.xml index 500999f194..b7bb00609b 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -365,6 +365,11 @@ 12.2.0.1 + + org.antlr + ST4 + 4.0.8 + diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index 2d77ec9549..fcaf060a42 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -1,5 +1,55 @@ package io.dataease.datasource.constants; public enum DatasourceTypes { - mysql, sqlServer, excel, doris, oracle + excel("excel", "excel", "", "", "", "", ""), + mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), + sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), + doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), + oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""); + + private String feature; + private String desc; + private String driver; + private String keywordPrefix; + private String keywordSuffix; + private String aliasPrefix; + private String aliasSuffix; + + DatasourceTypes(String feature, String desc, String driver, String keywordPrefix, String keywordSuffix, String aliasPrefix, String aliasSuffix) { + this.feature = feature; + this.desc = desc; + this.driver = driver; + this.keywordPrefix = keywordPrefix; + this.keywordSuffix = keywordSuffix; + this.aliasPrefix = aliasPrefix; + this.aliasSuffix = aliasSuffix; + } + + public String getFeature() { + return feature; + } + + public String getDesc() { + return desc; + } + + public String getDriver() { + return driver; + } + + public String getKeywordPrefix() { + return keywordPrefix; + } + + public String getKeywordSuffix() { + return keywordSuffix; + } + + public String getAliasPrefix() { + return aliasPrefix; + } + + public String getAliasSuffix() { + return aliasSuffix; + } } diff --git a/backend/src/main/java/io/dataease/dto/sqlObj/SQLObj.java b/backend/src/main/java/io/dataease/dto/sqlObj/SQLObj.java new file mode 100644 index 0000000000..4505f205a7 --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/sqlObj/SQLObj.java @@ -0,0 +1,29 @@ +package io.dataease.dto.sqlObj; + +import lombok.Builder; +import lombok.Data; + +/** + * @Author gin + * @Date 2021/7/9 12:12 下午 + */ +@Data +@Builder +public class SQLObj { + private String tableName; + private String tableAlias; + + private String fieldName; + private String fieldAlias; + + private String groupField; + private String groupAlias; + + private String orderField; + private String orderAlias; + private String orderDirection; + + private String whereField; + private String whereAlias; + private String whereTermAndValue; +} diff --git a/backend/src/main/java/io/dataease/provider/SQLConstants.java b/backend/src/main/java/io/dataease/provider/SQLConstants.java index a0dacbd57a..76335f84bb 100644 --- a/backend/src/main/java/io/dataease/provider/SQLConstants.java +++ b/backend/src/main/java/io/dataease/provider/SQLConstants.java @@ -8,9 +8,34 @@ import java.util.List; * @Date 2021/7/8 3:12 下午 */ public class SQLConstants { + /** + * 维度类型list + */ public static final List DIMENSION_TYPE = new ArrayList() {{ add(0);// 文本 add(1);// 时间 add(5);// 地理位置 }}; + + /** + * 指标类型list + */ + public static final List QUOTA_TYPE = new ArrayList() {{ + add(2);// 整型 + add(3);// 浮点 + add(4);// 布尔 + }}; + + /** + * sql ST模板 + */ + public static final String SQL_TEMPLATE = "sql/sqlTemplate.stg"; + + public static final String TABLE_ALIAS_PREFIX = "T_A_%s"; + public static final String FIELD_ALIAS_X_PREFIX = "F_AX_%s"; + public static final String FIELD_ALIAS_Y_PREFIX = "F_AY_%s"; + public static final String GROUP_ALIAS_PREFIX = "G_A_%s"; + public static final String ORDER_ALIAS_X_PREFIX = "O_AX_%s"; + public static final String ORDER_ALIAS_Y_PREFIX = "O_AY_%s"; + public static final String WHERE_ALIAS_PREFIX = "W_A_%s"; } diff --git a/backend/src/main/java/io/dataease/provider/mysql/MySQLConstants.java b/backend/src/main/java/io/dataease/provider/mysql/MySQLConstants.java new file mode 100644 index 0000000000..841febf127 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/mysql/MySQLConstants.java @@ -0,0 +1,41 @@ +package io.dataease.provider.mysql; + +import io.dataease.provider.SQLConstants; + +import static io.dataease.datasource.constants.DatasourceTypes.mysql; + +/** + * @Author gin + * @Date 2021/7/8 7:22 下午 + */ +public class MySQLConstants extends SQLConstants { + public static final String KEYWORD_TABLE = mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); + + public static final String KEYWORD_FIX = "%s." + mysql.getKeywordPrefix() + "%s" + mysql.getKeywordSuffix(); + + public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; + + public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')"; + + public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')"; + + public static final String CAST = "CAST(%s AS %s)"; + + public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S"; + + public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; + + public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; + + public static final String WHERE_VALUE_NULL = "(null,'')"; + + public static final String WHERE_VALUE_VALUE = "'%s'"; + + public static final String AGG_COUNT = "COUNT(*)"; + + public static final String AGG_FIELD = "%s(%s)"; + + public static final String WHERE_BETWEEN = "'%s' AND '%s'"; + + public static final String BRACKETS = "(%s)"; +} diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java index d7a6155664..fb6fe589ac 100644 --- a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java @@ -4,19 +4,25 @@ import io.dataease.base.domain.DatasetTableField; import io.dataease.controller.request.chart.ChartExtFilterRequest; import io.dataease.dto.chart.ChartCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; import io.dataease.provider.QueryProvider; import io.dataease.provider.SQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; import java.text.MessageFormat; import java.text.SimpleDateFormat; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + /** * @Author gin * @Date 2021/5/17 2:43 下午 @@ -75,40 +81,59 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String createQuerySQL(String table, List fields) { - String[] array = fields.stream().map(f -> { - StringBuilder stringBuilder = new StringBuilder(); - // 如果原始类型为时间 - if (f.getDeExtractType() == 1) { - if (f.getDeType() == 2 || f.getDeType() == 3) { - stringBuilder.append("UNIX_TIMESTAMP(`").append(f.getOriginName()).append("`)*1000 AS ").append(f.getDataeaseName()); + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + // 处理横轴字段 + if (f.getDeExtractType() == 1) { + if (f.getDeType() == 2 || f.getDeType() == 3) { + fieldName = String.format(MySQLConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else { + fieldName = originField; + } + } else if (f.getDeExtractType() == 0) { + if (f.getDeType() == 2) { + fieldName = String.format(MySQLConstants.CAST, originField, MySQLConstants.DEFAULT_INT_FORMAT); + } else if (f.getDeType() == 3) { + fieldName = String.format(MySQLConstants.CAST, originField, MySQLConstants.DEFAULT_FLOAT_FORMAT); + } else if (f.getDeType() == 1) { + fieldName = String.format(MySQLConstants.DATE_FORMAT, originField, MySQLConstants.DEFAULT_DATE_FORMAT); + } else { + fieldName = originField; + } } else { - stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); - } - } else if (f.getDeExtractType() == 0) { - if (f.getDeType() == 2) { - stringBuilder.append("CAST(`").append(f.getOriginName()).append("` AS DECIMAL(20,0)) AS ").append(f.getDataeaseName()); - } else if (f.getDeType() == 3) { - stringBuilder.append("CAST(`").append(f.getOriginName()).append("` AS DECIMAL(20,2)) AS ").append(f.getDataeaseName()); - } else if (f.getDeType() == 1) { - stringBuilder.append("DATE_FORMAT(`").append(f.getOriginName()).append("`,'%Y-%m-%d %H:%i:%S') AS _").append(f.getDataeaseName()); - } else { - stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); - } - } else { - if (f.getDeType() == 1) { - stringBuilder.append("FROM_UNIXTIME(CAST(`").append(f.getOriginName()).append("` AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') AS ").append(f.getDataeaseName()); - } else { - stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + if (f.getDeType() == 1) { + String cast = String.format(MySQLConstants.CAST, originField, MySQLConstants.DEFAULT_INT_FORMAT) + "/1000"; + fieldName = String.format(MySQLConstants.FROM_UNIXTIME, cast, MySQLConstants.DEFAULT_DATE_FORMAT); + } else { + fieldName = originField; + } } + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); } - return stringBuilder.toString(); - }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + return st_sql.render(); } @Override public String createQuerySQLAsTmp(String sql, List fields) { - return createQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + return createQuerySQL("(" + sqlFix(sql) + ")", fields); } @Override @@ -133,145 +158,98 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - // 字段汇总 排序等 - String[] field = yAxis.stream().map(y -> { - StringBuilder f = new StringBuilder(); - if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { - f.append(y.getSummary()).append("(").append(y.getOriginName()).append(")"); - } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { - f.append(y.getSummary()).append("(`").append(y.getOriginName()).append("`)"); - } else { - if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { - f.append("CAST(") - .append(y.getSummary()).append("(") - .append("CAST(`").append(y.getOriginName()).append("` AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(") AS DECIMAL(20,2)").append(")"); - } else { - f.append(y.getSummary()).append("(") - .append("CAST(`").append(y.getOriginName()).append("` AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(")"); + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 + xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); } } - f.append(" AS `_").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`"); - return f.toString(); - }).toArray(String[]::new); - String[] groupField = xAxis.stream().map(x -> { - StringBuilder stringBuilder = new StringBuilder(); - // 如果原始类型为时间 - if (x.getDeExtractType() == 1) { - if (x.getDeType() == 2 || x.getDeType() == 3) { - stringBuilder.append("UNIX_TIMESTAMP(`").append(x.getOriginName()).append("`)*1000 AS `_").append(x.getOriginName()).append("`"); - } else if (x.getDeType() == 1) { - String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); - stringBuilder.append("DATE_FORMAT(`").append(x.getOriginName()).append("`,'").append(format).append("') AS `_").append(x.getOriginName()).append("`"); - } else { - stringBuilder.append("`").append(x.getOriginName()).append("` AS `_").append(x.getOriginName()).append("`"); - } - } else { - if (x.getDeType() == 1) { - String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); - if (x.getDeExtractType() == 0) { - stringBuilder.append("DATE_FORMAT(`").append(x.getOriginName()).append("`,'").append(format).append("') AS `_").append(x.getOriginName()).append("`"); - } else { - stringBuilder.append("DATE_FORMAT(").append("FROM_UNIXTIME(CAST(`").append(x.getOriginName()).append("` AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S')").append(",'").append(format).append("') AS `_").append(x.getOriginName()).append("`"); - } - } else { - stringBuilder.append("`").append(x.getOriginName()).append("` AS `_").append(x.getOriginName()).append("`"); + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); } } - return stringBuilder.toString(); - }).toArray(String[]::new); - String[] group = xAxis.stream().map(x -> "`_" + x.getOriginName() + "`").toArray(String[]::new); - String[] xOrder = xAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> "`_" + f.getOriginName() + "` " + f.getSort()).toArray(String[]::new); - String[] yOrder = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> "`_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getOriginName(), "*") ? "" : f.getOriginName()) + "` " + f.getSort()).toArray(String[]::new); - String[] order = Arrays.copyOf(xOrder, xOrder.length + yOrder.length); - System.arraycopy(yOrder, 0, order, xOrder.length, yOrder.length); - - String[] xFilter = xAxis.stream().filter(x -> CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) - .map(x -> { - String[] s = x.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - if (x.getDeType() == 1 && x.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(cast(`") - .append(x.getOriginName()) - .append("` AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND `").append(x.getOriginName()).append("`"); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { - filter.append("(null,'')"); - } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { - filter.append(" AND `").append(x.getOriginName()).append("`").append(" <> ''"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("'%").append(f.getValue()).append("%'"); - } else { - filter.append("'").append(f.getValue()).append("'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - - String sql = MessageFormat.format("SELECT {0},{1} FROM {2} WHERE 1=1 {3} GROUP BY {4} ORDER BY null,{5}", - StringUtils.join(groupField, ","), - StringUtils.join(field, ","), - table, - (xFilter.length > 0 ? StringUtils.join(xFilter, " ") : "") + transCustomFilter(customFilter) + transExtFilter(extFilterRequestList),// origin field filter and panel field filter - StringUtils.join(group, ","), - StringUtils.join(order, ",")); - if (sql.endsWith(",")) { - sql = sql.substring(0, sql.length() - 1); - } - // 如果是对结果字段过滤，则再包裹一层sql - String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) - .map(y -> { - String[] s = y.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - // 原始类型不是时间，在de中被转成时间的字段做处理 - if (y.getDeType() == 1 && y.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(CAST(`_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`") - .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND `_").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`"); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { - filter.append("(null,'')"); - } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { - filter.append(" AND `_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()) - .append("`").append(" <> ''"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("'%").append(f.getValue()).append("%'"); - } else { - filter.append("'").append(f.getValue()).append("'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - if (resultFilter.length == 0) { - return sql; - } else { - String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", - "(" + sql + ") AS tmp", - StringUtils.join(resultFilter, " "), - ObjectUtils.isNotEmpty(yOrder) ? StringUtils.join(yOrder, ",") : "null"); - return filterSql; } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MySQLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); } + @Override public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - return getSQL(" (" + sqlFix(sql) + ") AS tmp ", xAxis, yAxis, customFilter, extFilterRequestList); + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList); } @Override @@ -282,84 +260,70 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String getSQLSummary(String table, List yAxis, List customFilter, List extFilterRequestList) { // 字段汇总 排序等 - String[] field = yAxis.stream().map(y -> { - StringBuilder f = new StringBuilder(); - if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { - f.append(y.getSummary()).append("(").append(y.getOriginName()).append(")"); - } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { - f.append(y.getSummary()).append("(`").append(y.getOriginName()).append("`)"); - } else { - if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { - f.append("CAST(") - .append(y.getSummary()).append("(") - .append("CAST(`").append(y.getOriginName()).append("` AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(") AS DECIMAL(20,2)").append(")"); - } else { - f.append(y.getSummary()).append("(") - .append("CAST(`").append(y.getOriginName()).append("` AS ").append(y.getDeType() == 2 ? "DECIMAL(20,0)" : "DECIMAL(20,2)").append(")") - .append(")"); + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); } } - f.append(" AS `_").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`"); - return f.toString(); - }).toArray(String[]::new); - - String[] order = yAxis.stream().filter(f -> StringUtils.isNotEmpty(f.getSort()) && !StringUtils.equalsIgnoreCase(f.getSort(), "none")) - .map(f -> "`_" + f.getSummary() + "_" + (StringUtils.equalsIgnoreCase(f.getOriginName(), "*") ? "" : f.getOriginName()) + "` " + f.getSort()).toArray(String[]::new); - - String sql = MessageFormat.format("SELECT {0} FROM {1} WHERE 1=1 {2} ORDER BY null,{3}", - StringUtils.join(field, ","), - table, - transCustomFilter(customFilter) + transExtFilter(extFilterRequestList),// origin field filter and panel field filter - StringUtils.join(order, ",")); - if (sql.endsWith(",")) { - sql = sql.substring(0, sql.length() - 1); - } - // 如果是对结果字段过滤，则再包裹一层sql - String[] resultFilter = yAxis.stream().filter(y -> CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) - .map(y -> { - String[] s = y.getFilter().stream().map(f -> { - StringBuilder filter = new StringBuilder(); - // 原始类型不是时间，在de中被转成时间的字段做处理 - if (y.getDeType() == 1 && y.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(CAST(`_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`") - .append(" AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); - } else { - filter.append(" AND `_").append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()).append("`"); - } - filter.append(transMysqlFilterTerm(f.getTerm())); - if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { - filter.append("(null,'')"); - } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { - filter.append(" AND `_") - .append(y.getSummary()).append("_").append(StringUtils.equalsIgnoreCase(y.getOriginName(), "*") ? "" : y.getOriginName()) - .append("`").append(" <> ''"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(f.getValue(), "','")).append("')"); - } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { - filter.append("'%").append(f.getValue()).append("%'"); - } else { - filter.append("'").append(f.getValue()).append("'"); - } - return filter.toString(); - }).toArray(String[]::new); - return StringUtils.join(s, " "); - }).toArray(String[]::new); - if (resultFilter.length == 0) { - return sql; - } else { - String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", - "(" + sql + ") AS tmp", - StringUtils.join(resultFilter, " "), - ObjectUtils.isNotEmpty(order) ? StringUtils.join(order, ",") : "null"); - return filterSql; } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MySQLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); } @Override public String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList) { - return getSQLSummary(" (" + sqlFix(sql) + ") AS tmp ", yAxis, customFilter, extFilterRequestList); + return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList); } @Override @@ -412,7 +376,7 @@ public class MysqlQueryProvider extends QueryProvider { case "null": return " IN "; case "not_null": - return " IS NOT NULL "; + return " NOT IN "; case "between": return " BETWEEN "; default: @@ -420,77 +384,86 @@ public class MysqlQueryProvider extends QueryProvider { } } - public String transCustomFilter(List requestList) { + public List transCustomFilterList(SQLObj tableObj, List requestList) { if (CollectionUtils.isEmpty(requestList)) { - return ""; + return null; } - StringBuilder filter = new StringBuilder(); + List list = new ArrayList<>(); for (ChartCustomFilterDTO request : requestList) { - String value = request.getValue(); DatasetTableField field = request.getField(); if (ObjectUtils.isEmpty(field)) { continue; } + String value = request.getValue(); + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getTerm()); + String whereValue = ""; + String originName = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); if (field.getDeType() == 1 && field.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(CAST(`") - .append(field.getOriginName()) - .append("` AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + String cast = String.format(MySQLConstants.CAST, originName, MySQLConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MySQLConstants.FROM_UNIXTIME, cast, MySQLConstants.DEFAULT_DATE_FORMAT); } else { - filter.append(" AND `").append(field.getOriginName()).append("`"); + whereName = originName; } - filter.append(" ") - .append(transMysqlFilterTerm(request.getTerm())) - .append(" "); - if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) { - filter.append("(null,'')"); - } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) { - filter.append(" AND `").append(field.getOriginName()).append("`").append(" <> ''"); + if (StringUtils.containsIgnoreCase(request.getTerm(), "null")) { + whereValue = MySQLConstants.WHERE_VALUE_NULL; } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { - filter.append("('").append(StringUtils.join(value, "','")).append("')"); + whereValue = "('" + StringUtils.join(value, "','") + "')"; } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { - filter.append("'%").append(value).append("%'"); + whereValue = "'%" + value + "%'"; } else { - filter.append("'").append(value).append("'"); + whereValue = String.format(MySQLConstants.WHERE_VALUE_VALUE, value); } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); } - return filter.toString(); + return list; } - public String transExtFilter(List requestList) { + public List transExtFilterList(SQLObj tableObj, List requestList) { if (CollectionUtils.isEmpty(requestList)) { - return ""; + return null; } - StringBuilder filter = new StringBuilder(); + List list = new ArrayList<>(); for (ChartExtFilterRequest request : requestList) { List value = request.getValue(); - if (CollectionUtils.isEmpty(value)) { + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { continue; } - DatasetTableField field = request.getDatasetTableField(); + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + String originName = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { - filter.append(" AND FROM_UNIXTIME(CAST(`") - .append(field.getOriginName()) - .append("` AS DECIMAL(20,0))/1000,'%Y-%m-%d %H:%i:%S') "); + String cast = String.format(MySQLConstants.CAST, originName, MySQLConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MySQLConstants.FROM_UNIXTIME, cast, MySQLConstants.DEFAULT_DATE_FORMAT); } else { - filter.append(" AND `").append(field.getOriginName()).append("`"); + whereName = originName; } - filter.append(" ") - .append(transMysqlFilterTerm(request.getOperator())) - .append(" "); + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { - filter.append("('").append(StringUtils.join(value, "','")).append("')"); + whereValue = "('" + StringUtils.join(value, "','") + "')"; } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { - filter.append("'%").append(value.get(0)).append("%'"); + whereValue = "'%" + value.get(0) + "%'"; } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); - filter.append("'").append(startTime).append("' AND '").append(endTime).append("'"); + whereValue = String.format(MySQLConstants.WHERE_BETWEEN, startTime, endTime); } else { - filter.append("'").append(value.get(0)).append("'"); + whereValue = String.format(MySQLConstants.WHERE_VALUE_VALUE, value.get(0)); } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); } - return filter.toString(); + return list; } private String sqlFix(String sql) { @@ -525,4 +498,115 @@ public class MysqlQueryProvider extends QueryProvider { return "%Y-%m-%d %H:%i:%S"; } } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == 1) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + fieldName = String.format(MySQLConstants.UNIX_TIMESTAMP, originField) + "*1000"; + } else if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + fieldName = String.format(MySQLConstants.DATE_FORMAT, originField, format); + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == 1) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == 0) { + fieldName = String.format(MySQLConstants.DATE_FORMAT, originField, format); + } else { + String cast = String.format(MySQLConstants.CAST, originField, MySQLConstants.DEFAULT_INT_FORMAT) + "/1000"; + String from_unixtime = String.format(MySQLConstants.FROM_UNIXTIME, cast, MySQLConstants.DEFAULT_DATE_FORMAT); + fieldName = String.format(MySQLConstants.DATE_FORMAT, from_unixtime, format); + } + } else { + fieldName = originField; + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getXWheres(ChartViewFieldDTO x, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) { + x.getFilter().forEach(f -> { + String whereName = ""; + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + if (x.getDeType() == 1 && x.getDeExtractType() != 1) { + String cast = String.format(MySQLConstants.CAST, originField, MySQLConstants.DEFAULT_INT_FORMAT) + "/1000"; + whereName = String.format(MySQLConstants.FROM_UNIXTIME, cast, MySQLConstants.DEFAULT_DATE_FORMAT); + } else { + whereName = originField; + } + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + whereValue = MySQLConstants.WHERE_VALUE_NULL; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(MySQLConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + fieldName = MySQLConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(MySQLConstants.AGG_FIELD, y.getSummary(), originField); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format(MySQLConstants.CAST, originField, y.getDeType() == 2 ? MySQLConstants.DEFAULT_INT_FORMAT : MySQLConstants.DEFAULT_FLOAT_FORMAT); + String agg = String.format(MySQLConstants.AGG_FIELD, y.getSummary(), cast); + fieldName = String.format(MySQLConstants.CAST, agg, MySQLConstants.DEFAULT_FLOAT_FORMAT); + } else { + String cast = String.format(MySQLConstants.CAST, originField, y.getDeType() == 2 ? MySQLConstants.DEFAULT_INT_FORMAT : MySQLConstants.DEFAULT_FLOAT_FORMAT); + fieldName = String.format(MySQLConstants.AGG_FIELD, y.getSummary(), cast); + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.containsIgnoreCase(f.getTerm(), "null")) { + whereValue = MySQLConstants.WHERE_VALUE_NULL; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(MySQLConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } } diff --git a/backend/src/main/resources/sql/sqlTemplate.stg b/backend/src/main/resources/sql/sqlTemplate.stg new file mode 100644 index 0000000000..71eb2c3b70 --- /dev/null +++ b/backend/src/main/resources/sql/sqlTemplate.stg @@ -0,0 +1,28 @@ +querySql(groups, aggregators, filters, orders, table) +::=<< +SELECT + + * + + + AS }; separator=",\n"> + + , + + AS }; separator=",\n"> + +FROM + AS + +WHERE + }; separator="\nAND "> + + +GROUP BY + }; separator=",\n"> + + +ORDER BY + }; separator=",\n"> + +>> \ No newline at end of file