diff --git a/README.md b/README.md index 5e854a1256..d5a0aa813c 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,6 @@ FOSSA Status

-|说明| -|------------------| -|此分支为 DataEase v1.18 版本的开发分支。[DataEase v2.0.0](https://github.com/dataease/dataease/releases/tag/v2.0.0) 也已经正式发布,v2 版本的开发分支为 [dev-v2](https://github.com/dataease/dataease/tree/dev-v2)。v2 版本正在快速迭代中,如是在生产环境部署 DataEase,建议继续使用 v1.18.* 的最新稳定版本。| -
## 什么是 DataEase? diff --git a/core/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java b/core/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java index 556aa7fab4..e242b1bc14 100644 --- a/core/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java +++ b/core/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java @@ -112,6 +112,7 @@ public interface ParamConstants { MSG_TIME_OUT("basic.msgTimeOut"), LOG_TIME_OUT("basic.logTimeOut"), + DS_SYNC_LOG_TIME_OUT("basic.dsSyncLogTimeOut"), DS_CHECK_INTERVAL("basic.dsCheckInterval"), DS_CHECK_INTERVAL_TYPE("basic.dsCheckIntervalType"), DEFAULT_LOGIN_TYPE("basic.loginType"), diff --git a/core/backend/src/main/java/io/dataease/controller/datasource/DriverMgmController.java b/core/backend/src/main/java/io/dataease/controller/datasource/DriverMgmController.java index 32edc4ae23..3c4b1d88bd 100644 --- a/core/backend/src/main/java/io/dataease/controller/datasource/DriverMgmController.java +++ b/core/backend/src/main/java/io/dataease/controller/datasource/DriverMgmController.java @@ -34,7 +34,6 @@ public class DriverMgmController { @ApiOperation("驱动列表") @PostMapping("/list") public List listDeDriver() throws Exception{ - checkPermission(); return driverService.list(); } @@ -57,7 +56,6 @@ public class DriverMgmController { @ApiOperation("驱动列表") @GetMapping("/list/{type}") public List listDeDriver(@PathVariable String type) throws Exception{ - checkPermission(); return listDeDriver().stream().filter(driverDTO -> driverDTO.getType().equalsIgnoreCase(type)).collect(Collectors.toList()); } diff --git a/core/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java b/core/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java index ba7f003500..c4340ab033 100644 --- a/core/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java +++ b/core/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java @@ -15,6 +15,8 @@ public class BasicInfo extends LoginLimitInfo implements Serializable { private String msgTimeOut; @ApiModelProperty("日志保留时间") private String logTimeOut; + @ApiModelProperty("数据同步日志保留时间") + private String dsSyncLogTimeOut; @ApiModelProperty("显示首页") private String openHomePage; @ApiModelProperty("默认登录方式") diff --git a/core/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java b/core/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java index 0b4adc7626..4f850deeea 100644 --- a/core/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java @@ -6,6 +6,8 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.parser.Feature; import com.alibaba.fastjson.serializer.SerializerFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.JsonObject; @@ -239,7 +241,12 @@ public class ApiProvider extends Provider { String rootPath; if (response.startsWith("[")) { rootPath = "$[*]"; - JSONArray jsonArray = JSONObject.parseArray(response); + JsonNode jsonArray = null; + try { + jsonArray = new ObjectMapper().readTree(response); + } catch (Exception e) { + e.printStackTrace(); + } for (Object o : jsonArray) { handleStr(apiDefinition, o.toString(), fields, rootPath); } diff --git a/core/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java b/core/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java index ca537c3b49..54f492c74c 100644 --- a/core/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java @@ -181,7 +181,7 @@ public class EsProvider extends Provider { } for (String[] row : esResponse.getRows()) { - if(!row[1].equalsIgnoreCase("STRUCT")){ + if(!row[1].equalsIgnoreCase("STRUCT") && !row[2].equalsIgnoreCase("unsupported")){ TableField field = new TableField(); field.setFieldName(row[0]); field.setRemarks(row[0]); diff --git a/core/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java index 7d7fd45691..726d8e16dc 100644 --- a/core/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.engine.doris; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -197,7 +199,7 @@ public class DorisQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1418,27 +1420,54 @@ public class DorisQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(DorisConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(DorisConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java index 2b79c4571b..82fb97afb6 100644 --- a/core/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.engine.mysql; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -167,7 +169,7 @@ public class MysqlQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1366,27 +1368,54 @@ public class MysqlQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java index 521d1c7a7a..5b582ef804 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java @@ -1,7 +1,9 @@ package io.dataease.provider.query.ck; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; import io.dataease.commons.utils.BeanUtils; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -196,7 +198,7 @@ public class CKQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1416,27 +1418,54 @@ public class CKQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java index 545806915d..3e401f1476 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java @@ -2,7 +2,9 @@ package io.dataease.provider.query.db2; import com.alibaba.fastjson.JSONArray; import com.google.gson.Gson; +import io.dataease.commons.exception.DEException; import io.dataease.dto.datasource.Db2Configuration; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -172,7 +174,7 @@ public class Db2QueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1404,27 +1406,54 @@ public class Db2QueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(Db2Constants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(Db2Constants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java index 89069ca26d..b72cd5348a 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.query.es; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -344,8 +346,10 @@ public class EsQueryProvider extends QueryProvider { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); } else { - if (y.getDeType() == 2 || y.getDeType() == 3) { + if (y.getDeType() == 2) { originField = String.format(EsSqlLConstants.CAST, String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()), "bigint"); + } else if (y.getDeType() == 3) { + originField = String.format(EsSqlLConstants.CAST, String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()), "float"); } else { originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); } @@ -1320,27 +1324,54 @@ public class EsQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java index 8b0cb477d3..2e1510f271 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.query.hive; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -153,7 +155,7 @@ public class HiveQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1334,27 +1336,54 @@ public class HiveQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(HiveConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(HiveConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java index 39b1a3097a..a008acb38b 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.query.impala; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -18,7 +20,6 @@ import io.dataease.plugins.common.request.chart.ChartExtFilterRequest; import io.dataease.plugins.common.request.permission.DataSetRowPermissionsTreeDTO; import io.dataease.plugins.common.request.permission.DatasetRowPermissionsTreeItem; import io.dataease.plugins.datasource.entity.Dateformat; -import io.dataease.plugins.datasource.entity.PageInfo; import io.dataease.plugins.datasource.query.QueryProvider; import io.dataease.plugins.datasource.query.Utils; import org.apache.commons.collections4.CollectionUtils; @@ -1306,27 +1307,54 @@ public class ImpalaQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(ImpalaConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java index 7743e4d1fa..fd4bb88916 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.query.mongodb; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -139,7 +141,7 @@ public class MongoQueryProvider extends QueryProvider { List wheres = new ArrayList<>(); if (customWheres != null) wheres.add(customWheres); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (whereTrees != null) wheres.add(whereTrees); @@ -1159,27 +1161,54 @@ public class MongoQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java index 6d19c6b574..3b5c414b6d 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java @@ -1,6 +1,8 @@ package io.dataease.provider.query.mysql; import com.alibaba.fastjson.JSONArray; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -251,7 +253,7 @@ public class MysqlQueryProvider extends QueryProvider { if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1426,27 +1428,54 @@ public class MysqlQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java index 3fd78c57f7..419cb087fb 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java @@ -2,7 +2,9 @@ package io.dataease.provider.query.oracle; import com.alibaba.fastjson.JSONArray; import com.google.gson.Gson; +import io.dataease.commons.exception.DEException; import io.dataease.dto.datasource.OracleConfiguration; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -127,7 +129,7 @@ public class OracleQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -1516,27 +1518,54 @@ public class OracleQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(OracleConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(OracleConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java index f0e0b053c4..1f362cef65 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java @@ -2,6 +2,8 @@ package io.dataease.provider.query.pg; import com.alibaba.fastjson.JSONArray; import com.google.gson.Gson; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -180,7 +182,7 @@ public class PgQueryProvider extends QueryProvider { List wheres = new ArrayList<>(); if (customWheres != null) wheres.add(customWheres); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (whereTrees != null) wheres.add(whereTrees); @@ -1332,27 +1334,54 @@ public class PgQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java index 8c4a0c348f..07bab6ddfd 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java @@ -2,6 +2,8 @@ package io.dataease.provider.query.redshift; import com.alibaba.fastjson.JSONArray; import com.google.gson.Gson; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -1313,27 +1315,54 @@ public class RedshiftQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } private String sqlLimit(String sql, ChartViewWithBLOBs view) { diff --git a/core/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java b/core/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java index 3d98e2c9cb..d5efe0bdd4 100644 --- a/core/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java +++ b/core/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java @@ -2,6 +2,8 @@ package io.dataease.provider.query.sqlserver; import com.alibaba.fastjson.JSONArray; import com.google.gson.Gson; +import io.dataease.commons.exception.DEException; +import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs; import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.plugins.common.base.domain.DatasetTableFieldExample; @@ -110,94 +112,15 @@ public class SqlserverQueryProvider extends QueryProvider { return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null, fieldCustomFilter, rowPermissionsTree, sortFields, limit, keyword); } - /*@Override - public String createQuerySQLWithLimit(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter, List rowPermissionsTree, List sortFields, Long limit, String keyword) { - SQLObj tableObj = SQLObj.builder() - .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SqlServerSQLConstants.KEYWORD_TABLE, table)) - .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) - .build(); - setSchema(tableObj, ds); - - List xFields = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(fields)) { - for (int i = 0; i < fields.size(); i++) { - DatasetTableField f = fields.get(i); - String originField; - if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) { - // 解析origin name中有关联的字段生成sql表达式 - originField = calcFieldRegex(f.getOriginName(), tableObj); - } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) { - originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); - } else { - originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); - } - String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); - String fieldName; - // 处理横轴字段 - if (f.getDeExtractType() == DeTypeConstants.DE_TIME) { // 时间 转为 数值 - if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) { - fieldName = String.format(SqlServerSQLConstants.UNIX_TIMESTAMP, originField); - } else { - fieldName = originField; - } - } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) { - if (f.getDeType() == DeTypeConstants.DE_INT) { - fieldName = String.format(SqlServerSQLConstants.CONVERT, SqlServerSQLConstants.DEFAULT_INT_FORMAT, originField); - } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) { - fieldName = String.format(SqlServerSQLConstants.CONVERT, SqlServerSQLConstants.DEFAULT_FLOAT_FORMAT, originField); - } else if (f.getDeType() == DeTypeConstants.DE_TIME) { //字符串转时间 - fieldName = String.format(SqlServerSQLConstants.STRING_TO_DATE, originField, StringUtils.isNotEmpty(f.getDateFormat()) ? f.getDateFormat() : SqlServerSQLConstants.DEFAULT_DATE_FORMAT); - } else { - fieldName = originField; - } - } else { - if (f.getDeType() == DeTypeConstants.DE_TIME) { // 数值转时间 - String cast = String.format(SqlServerSQLConstants.LONG_TO_DATE, originField + "/1000"); - fieldName = String.format(SqlServerSQLConstants.FROM_UNIXTIME, cast); - } else if (f.getDeType() == DeTypeConstants.DE_INT) { - fieldName = String.format(SqlServerSQLConstants.CONVERT, SqlServerSQLConstants.DEFAULT_INT_FORMAT, originField); - } else { - fieldName = originField; - } - } - xFields.add(SQLObj.builder() - .fieldName(fieldName) - .fieldAlias(fieldAlias) - .build()); - } - } - - STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); - ST st_sql = stg.getInstanceOf("previewSql"); - st_sql.add("isGroup", isGroup); - if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); - if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); - String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); - // row permissions tree - String whereTrees = transFilterTrees(tableObj, rowPermissionsTree); - List wheres = new ArrayList<>(); - if (customWheres != null) wheres.add(customWheres); - if (whereTrees != null) wheres.add(whereTrees); - if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); - List xOrders = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(sortFields)) { - int step = fields.size(); - for (int i = step; i < (step + sortFields.size()); i++) { - DeSortField deSortField = sortFields.get(i - step); - SQLObj order = buildSortField(deSortField, tableObj, i); - xOrders.add(order); - } - } - if(limit != null){ - SQLObj limitFiled = SQLObj.builder().limitFiled(" top " + limit + " ").build(); - st_sql.add("limitFiled", limitFiled); - } - if (ObjectUtils.isNotEmpty(xOrders)) { - st_sql.add("orders", xOrders); - } - return st_sql.render(); - }*/ + private boolean anyFieldExceed(List fields) { + if (CollectionUtils.isEmpty(fields)) return false; + return fields.stream().anyMatch(field -> field.getDeExtractType().equals(DeTypeConstants.DE_STRING) && field.getSize() > 8000); + } + private boolean anySortFieldExceed(List fields) { + if (CollectionUtils.isEmpty(fields)) return false; + return fields.stream().anyMatch(field -> field.getDeExtractType().equals(DeTypeConstants.DE_STRING) && field.getSize() > 8000); + } @Override public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter, List rowPermissionsTree, List sortFields, Long limit, String keyword) { @@ -206,11 +129,46 @@ public class SqlserverQueryProvider extends QueryProvider { .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .build(); setSchema(tableObj, ds); + List exceedList = null; + if (anyFieldExceed(fields) || anySortFieldExceed(sortFields)) { + exceedList = new ArrayList<>(); + List calcFieldList = new ArrayList<>(fields); + if (CollectionUtils.isNotEmpty(sortFields)) { + calcFieldList.addAll(sortFields); + } + List sqlObjList = new ArrayList<>(); + sqlObjList.add(SQLObj.builder().fieldName("*").build()); + for (DatasetTableField f : calcFieldList) { + boolean exceed = f.getDeExtractType().equals(DeTypeConstants.DE_STRING) && f.getSize() > 8000; + if (!exceedList.contains(f.getOriginName()) && exceed) { + exceedList.add(f.getOriginName()); + String originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + String newOriginName = f.getOriginName() + "_exceed"; + String format = String.format(SqlServerSQLConstants.TO_STRING, originField, newOriginName); + sqlObjList.add(SQLObj.builder().fieldName(format).build()); + } + } + STGroup tabStg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = tabStg.getInstanceOf("previewSql"); + st_sql.add("isGroup", false); + st_sql.add("notUseAs", true); + st_sql.add("table", tableObj); + st_sql.add("groups", sqlObjList); + String render = st_sql.render(); + tableObj = SQLObj.builder() + .tableName(" (" + render + ") ") + .tableAlias(String.format(TABLE_ALIAS_PREFIX, "_exceed")) + .build(); + } + List xFields = new ArrayList<>(); if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { DatasetTableField f = fields.get(i); + if (CollectionUtils.isNotEmpty(exceedList) && exceedList.contains(f.getOriginName())) { + f.setOriginName(f.getOriginName() + "_exceed"); + } String originField; if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) { // 解析origin name中有关联的字段生成sql表达式 @@ -269,7 +227,7 @@ public class SqlserverQueryProvider extends QueryProvider { if (customWheres != null) wheres.add(customWheres); if (whereTrees != null) wheres.add(whereTrees); if (StringUtils.isNotBlank(keyword)) { - String keyWhere = "("+transKeywordFilterList(tableObj, xFields, keyword)+")"; + String keyWhere = "(" + transKeywordFilterList(tableObj, xFields, keyword) + ")"; wheres.add(keyWhere); } if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); @@ -278,11 +236,14 @@ public class SqlserverQueryProvider extends QueryProvider { int step = fields.size(); for (int i = step; i < (step + sortFields.size()); i++) { DeSortField deSortField = sortFields.get(i - step); + if (CollectionUtils.isNotEmpty(exceedList) && exceedList.contains(deSortField.getOriginName())) { + deSortField.setOriginName(deSortField.getOriginName() + "_exceed"); + } SQLObj order = buildSortField(deSortField, tableObj, i); xOrders.add(order); } } - if(ObjectUtils.isNotEmpty(limit)){ + if (ObjectUtils.isNotEmpty(limit)) { SQLObj limitFiled = SQLObj.builder().limitFiled(" top " + limit + " ").build(); st_sql.add("limitFiled", limitFiled); } @@ -1488,27 +1449,54 @@ public class SqlserverQueryProvider extends QueryProvider { } private String calcFieldRegex(String originField, SQLObj tableObj) { - originField = originField.replaceAll("[\\t\\n\\r]]", ""); - // 正则提取[xxx] - String regex = "\\[(.*?)]"; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(originField); - Set ids = new HashSet<>(); - while (matcher.find()) { - String id = matcher.group(1); - ids.add(id); + try { + int i = 0; + return buildCalcField(originField, tableObj, i); + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_ref")); } - if (CollectionUtils.isEmpty(ids)) { + return null; + } + + private String buildCalcField(String originField, SQLObj tableObj, int i) throws Exception { + try { + i++; + if (i > 100) { + DEException.throwException(Translator.get("i18n_field_circular_error")); + } + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + if (StringUtils.containsIgnoreCase(originField, ele.getId() + "")) { + // 计算字段允许二次引用,这里递归查询完整引用链 + if (Objects.equals(ele.getExtField(), 0)) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } else { + originField = originField.replaceAll("\\[" + ele.getId() + "]", ele.getOriginName()); + originField = buildCalcField(originField, tableObj, i); + } + } + } return originField; + } catch (Exception e) { + DEException.throwException(Translator.get("i18n_field_circular_error")); } - DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); - List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); - for (DatasetTableField ele : calcFields) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); - } - return originField; + return null; } @Override diff --git a/core/backend/src/main/java/io/dataease/service/CleaningRebotService.java b/core/backend/src/main/java/io/dataease/service/CleaningRebotService.java index 7b3f501c47..2288ee25de 100644 --- a/core/backend/src/main/java/io/dataease/service/CleaningRebotService.java +++ b/core/backend/src/main/java/io/dataease/service/CleaningRebotService.java @@ -1,6 +1,7 @@ package io.dataease.service; import io.dataease.ext.CleaningRebotMapper; +import io.dataease.service.dataset.DataSetTableTaskLogService; import io.dataease.service.message.SysMsgService; import io.dataease.service.sys.log.LogService; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +24,8 @@ public class CleaningRebotService { @Resource private SysMsgService sysMsgService; + @Resource + private DataSetTableTaskLogService dataSetTableTaskLogService; public void execute() { int floatDept = 0; @@ -43,5 +46,6 @@ public class CleaningRebotService { } logService.cleanDisusedLog(); sysMsgService.cleanDisusedMsg(); + dataSetTableTaskLogService.cleanLog(); } } diff --git a/core/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/core/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index e62b4833ef..414025dfac 100644 --- a/core/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/core/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -1379,6 +1379,9 @@ public class ChartViewService { if (StringUtils.isEmpty(cValue)) { continue; } + if (sum.equals(new BigDecimal(0))) { + continue; + } item[dataIndex] = new BigDecimal(cValue) .divide(sum, 8, RoundingMode.HALF_UP) .toString(); diff --git a/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 3b6b890970..8539ce7738 100644 --- a/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1115,20 +1115,25 @@ public class DataSetTableService { } } - private String handlePlainSelect(PlainSelect plainSelect, Select statementSelect, String dsType) throws Exception { - + private void handleSelectItems(PlainSelect plainSelect, String dsType) throws Exception { List selectItems = new ArrayList<>(); - plainSelect.getSelectItems().forEach(selectItem -> { - System.out.println(selectItem); - System.out.println(selectItem instanceof PlainSelect); - System.out.println(selectItem instanceof SubSelect); - + for (SelectItem selectItem : plainSelect.getSelectItems()) { + try { + SelectExpressionItem selectExpressionItem = (SelectExpressionItem) selectItem; + if (selectExpressionItem.getExpression() instanceof SubSelect) { + SubSelect subSelect = (SubSelect) selectExpressionItem.getExpression(); + Select select = (Select) CCJSqlParserUtil.parse(removeVariables(subSelect.getSelectBody().toString(), dsType)); + subSelect.setSelectBody(select.getSelectBody()); + ((SelectExpressionItem) selectItem).setExpression(subSelect); + } + } catch (Exception e) { + } selectItems.add(selectItem); - }); - - plainSelect.addSelectItems(selectItems); - + } + plainSelect.setSelectItems(selectItems); + } + private void handleFromItems(PlainSelect plainSelect, String dsType) throws Exception { FromItem fromItem = plainSelect.getFromItem(); if (fromItem instanceof SubSelect) { SelectBody selectBody = ((SubSelect) fromItem).getSelectBody(); @@ -1136,7 +1141,9 @@ public class DataSetTableService { Select subSelectTmp = (Select) CCJSqlParserUtil.parse(removeVariables(selectBody.toString(), dsType)); subSelect.setSelectBody(subSelectTmp.getSelectBody()); if (dsType.equals(DatasourceTypes.oracle.getType())) { - subSelect.setAlias(new Alias(fromItem.getAlias().toString(), false)); + if (fromItem.getAlias() != null) { + subSelect.setAlias(new Alias(fromItem.getAlias().toString(), false)); + } } else { if (fromItem.getAlias() == null) { throw new Exception("Failed to parse sql, Every derived table must have its own alias!"); @@ -1145,6 +1152,9 @@ public class DataSetTableService { } plainSelect.setFromItem(subSelect); } + } + + private void handleJoins(PlainSelect plainSelect, String dsType) throws Exception { List joins = plainSelect.getJoins(); if (joins != null) { List joinsList = new ArrayList<>(); @@ -1170,6 +1180,9 @@ public class DataSetTableService { } plainSelect.setJoins(joinsList); } + } + + private String handleWhere(PlainSelect plainSelect, Select statementSelect, String dsType) throws Exception { Expression expr = plainSelect.getWhere(); if (expr == null) { return handleWith(plainSelect, statementSelect, dsType); @@ -1211,6 +1224,14 @@ public class DataSetTableService { return builder.toString(); } + private String handlePlainSelect(PlainSelect plainSelect, Select statementSelect, String dsType) throws Exception { + handleSelectItems(plainSelect, dsType); + handleFromItems(plainSelect, dsType); + handleJoins(plainSelect, dsType); + return handleWhere(plainSelect, statementSelect, dsType); + } + + public Map getDBPreview(DataSetTableRequest dataSetTableRequest) throws Exception { Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); if (ds == null) { @@ -1639,8 +1660,17 @@ public class DataSetTableService { List fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); String[] array = fields.stream() - .map(f -> table + "." + f.getDataeaseName() + " AS " - + TableUtils.fieldName(tableId + "_" + f.getDataeaseName())) + .map(f -> { + String s = ""; + if (f == null) { + DEException.throwException( + Translator.get("i18n_ds_error")); + } else { + s = table + "." + f.getDataeaseName() + " AS " + + TableUtils.fieldName(tableId + "_" + f.getDataeaseName()); + } + return s; + }) .toArray(String[]::new); checkedInfo.put(table, array); checkedFields.addAll(fields); @@ -2766,13 +2796,13 @@ public class DataSetTableService { if (StringUtils.isEmpty(s)) { throw new RuntimeException(Translator.get("i18n_excel_empty_column")); } - if(hashSet.contains(s)){ + if (hashSet.contains(s)) { repeat.add(s); - }else { + } else { hashSet.add(s); } } - if(CollectionUtils.isNotEmpty(repeat)){ + if (CollectionUtils.isNotEmpty(repeat)) { DataEaseException.throwException(Translator.get("i18n_excel_field_repeat") + "" + String.valueOf(repeat)); } } diff --git a/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index 969461a17f..320ea16705 100644 --- a/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/core/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -1,6 +1,7 @@ package io.dataease.service.dataset; import cn.hutool.core.date.DateUtil; +import io.dataease.commons.constants.ParamConstants; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.ServletUtils; import io.dataease.controller.dataset.request.DataSetTaskInstanceGridRequest; @@ -13,6 +14,7 @@ import io.dataease.plugins.common.base.domain.DatasetTableTaskLog; import io.dataease.plugins.common.base.domain.DatasetTableTaskLogExample; import io.dataease.plugins.common.base.mapper.DatasetTableTaskLogMapper; import io.dataease.plugins.common.base.mapper.DatasetTableTaskMapper; +import io.dataease.service.system.SystemParameterService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.poi.hssf.usermodel.HSSFCell; @@ -29,6 +31,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import java.io.OutputStream; import java.net.URLEncoder; +import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.UUID; @@ -46,6 +49,8 @@ public class DataSetTableTaskLogService { private ExtDataSetTaskMapper extDataSetTaskMapper; @Resource private DatasetTableTaskMapper datasetTableTaskMapper; + @Resource + private SystemParameterService systemParameterService; public DatasetTableTaskLog save(DatasetTableTaskLog datasetTableTaskLog, Boolean hasTask) { if (hasTask && datasetTableTaskMapper.selectByPrimaryKey(datasetTableTaskLog.getTaskId()) == null) { @@ -210,5 +215,23 @@ public class DataSetTableTaskLogService { return example; } + private static final String LOG_RETENTION = "30"; + + public void cleanLog() { + String value = systemParameterService.getValue(ParamConstants.BASIC.DS_SYNC_LOG_TIME_OUT.getValue()); + value = StringUtils.isBlank(value) ? LOG_RETENTION : value; + int logRetention = Integer.parseInt(value); + Calendar instance = Calendar.getInstance(); + Calendar startInstance = (Calendar) instance.clone(); + startInstance.add(Calendar.DATE, -logRetention); + startInstance.set(Calendar.HOUR_OF_DAY, 0); + startInstance.set(Calendar.MINUTE, 0); + startInstance.set(Calendar.SECOND, 0); + startInstance.set(Calendar.MILLISECOND, -1); + long timeInMillis = startInstance.getTimeInMillis(); + DatasetTableTaskLogExample example = new DatasetTableTaskLogExample(); + example.createCriteria().andCreateTimeLessThan(timeInMillis); + datasetTableTaskLogMapper.deleteByExample(example); + } } diff --git a/core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index d8a0f693b1..a60555ba88 100644 --- a/core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -362,13 +362,14 @@ public class ExtractDataService { case add_scope: // 增量更新 try { + if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) { + throw new Exception("未进行全量同步"); + } if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) { extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null); } else { DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId); - if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) { - throw new Exception("未进行全量同步"); - } + execTime = System.currentTimeMillis(); if (datasetTableIncrementalConfig != null && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 diff --git a/core/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/core/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index d69165c1c8..8c363c9a22 100644 --- a/core/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/core/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -371,9 +371,11 @@ public class DatasourceService { String datasourceStatus = null; try { Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + System.out.println(datasourceProvider.getClass()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); datasourceStatus = datasourceProvider.checkStatus(datasourceRequest); + System.out.println(datasourceStatus); if (datasource.getType().equalsIgnoreCase("api")) { List apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(), new TypeToken>() { }.getType()); @@ -398,6 +400,7 @@ public class DatasourceService { return ResultHolder.success("Success"); } catch (Exception e) { + e.printStackTrace(); datasourceStatus = "Error"; return ResultHolder.error(Translator.get("I18N_DS_INVALID") + ": " + e.getMessage()); } finally { @@ -405,6 +408,7 @@ public class DatasourceService { record.setStatus(datasourceStatus); DatasourceExample example = new DatasourceExample(); example.createCriteria().andIdEqualTo(datasource.getId()); + System.out.println(new Gson().toJson(record)); datasourceMapper.updateByExampleSelective(record, example); } } diff --git a/core/backend/src/main/java/io/dataease/service/system/SystemParameterService.java b/core/backend/src/main/java/io/dataease/service/system/SystemParameterService.java index f4c5bc0207..2168abbcdc 100644 --- a/core/backend/src/main/java/io/dataease/service/system/SystemParameterService.java +++ b/core/backend/src/main/java/io/dataease/service/system/SystemParameterService.java @@ -82,6 +82,9 @@ public class SystemParameterService { if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.LOG_TIME_OUT.getValue())) { result.setLogTimeOut(param.getParamValue()); } + if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DS_SYNC_LOG_TIME_OUT.getValue())) { + result.setDsSyncLogTimeOut(param.getParamValue()); + } if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DEFAULT_LOGIN_TYPE.getValue())) { String paramValue = param.getParamValue(); result.setLoginType(StringUtils.isNotBlank(paramValue) ? Integer.parseInt(paramValue) : 0); diff --git a/core/backend/src/main/resources/i18n/messages_en_US.properties b/core/backend/src/main/resources/i18n/messages_en_US.properties index 48c85b8c64..2fee75cb4d 100644 --- a/core/backend/src/main/resources/i18n/messages_en_US.properties +++ b/core/backend/src/main/resources/i18n/messages_en_US.properties @@ -277,4 +277,6 @@ I18N_PANEL_PDF_TEMPLATE_ONLY_PIC=Default template only screenshot \u8FB9\u684610=Border 10 I18n_name_cant_empty=Name can not be empty! I18n_del_admin_tips=Forbidden to delete the admin account -I18N_NO_DRIVER_PERMISSION=Do not have permissions! \ No newline at end of file +I18N_NO_DRIVER_PERMISSION=Do not have permissions! +i18n_field_circular_error=Field error +i18n_field_circular_ref=Field has Circular Reference diff --git a/core/backend/src/main/resources/i18n/messages_zh_CN.properties b/core/backend/src/main/resources/i18n/messages_zh_CN.properties index ff5de5ab72..7a5e436e86 100644 --- a/core/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/core/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -268,4 +268,5 @@ I18N_PANEL_PDF_TEMPLATE_ONLY_PIC=\u9ED8\u8BA4\u6A21\u677F(\u53EA\u622A\u56FE) I18n_name_cant_empty=\u540D\u79F0\u4E0D\u80FD\u4E3A\u7A7A\uFF01 I18n_del_admin_tips=\u7981\u6B62\u5220\u9664admin\u8D26\u53F7 I18N_NO_DRIVER_PERMISSION=\u6ca1\u6709\u6743\u9650\uff01 - +i18n_field_circular_error=\u5B57\u6BB5\u89E3\u6790\u9519\u8BEF\uFF0C\u53EF\u80FD\u539F\u56E0\uFF1A\u5B57\u6BB5\u5DF2\u5220\u9664\u3001\u8BA1\u7B97\u5B57\u6BB5\u5F15\u7528\u5C42\u7EA7\u8FC7\u6DF1\u3001\u5B58\u5728\u5FAA\u73AF\u5F15\u7528\u7B49\uFF0C\u8BF7\u68C0\u67E5\u8868\u8282\u70B9\u548C\u5B57\u6BB5\u5E76\u91CD\u65B0\u7F16\u8F91\u3002 +i18n_field_circular_ref=\u5B57\u6BB5\u5B58\u5728\u5FAA\u73AF\u5F15\u7528 diff --git a/core/backend/src/main/resources/i18n/messages_zh_TW.properties b/core/backend/src/main/resources/i18n/messages_zh_TW.properties index 9898eb0b13..1489026291 100644 --- a/core/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/core/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -273,4 +273,6 @@ I18N_PANEL_PDF_TEMPLATE_ONLY_PIC=\u9ED8\u8A8D\u6A21\u677F(\u53EA\u622A\u5716) \u8FB9\u684610=\u908A\u6846 10 I18n_name_cant_empty=\u540D\u7A31\u4E0D\u80FD\u70BA\u7A7A\uFF01 I18n_del_admin_tips=\u7981\u6B62\u522A\u9664admin\u8CEC\u865F -I18N_NO_DRIVER_PERMISSION=\u6c92\u6709\u8a31\u53ef\u6b0a\uff01 \ No newline at end of file +I18N_NO_DRIVER_PERMISSION=\u6c92\u6709\u8a31\u53ef\u6b0a\uff01 +i18n_field_circular_error=\u5B57\u6BB5\u89E3\u6790\u932F\u8AA4\uFF0C\u53EF\u80FD\u539F\u56E0\uFF1A\u5B57\u6BB5\u5DF2\u522A\u9664\u3001\u8A08\u7B97\u5B57\u6BB5\u5F15\u7528\u5C64\u7D1A\u904E\u6DF1\u3001\u5B58\u5728\u5FAA\u74B0\u5F15\u7528\u7B49\uFF0C\u8ACB\u6AA2\u67E5\u8868\u7BC0\u9EDE\u548C\u5B57\u6BB5\u4E26\u91CD\u65B0\u7DE8\u8F2F\u3002 +i18n_field_circular_ref=\u5B57\u6BB5\u5B58\u5728\u5FAA\u74B0\u5F15\u7528 diff --git a/core/frontend/src/components/canvas/DeCanvas.vue b/core/frontend/src/components/canvas/DeCanvas.vue index 3ea8fbaa34..d97346a6d1 100644 --- a/core/frontend/src/components/canvas/DeCanvas.vue +++ b/core/frontend/src/components/canvas/DeCanvas.vue @@ -509,7 +509,7 @@ export default { this.$store.commit('setComponentWithId', this.currentFilterCom) this.$store.commit('recordSnapshot', 'sureFilter') this.$store.commit('setCurComponent', { component: this.currentFilterCom, index: this.curComponentIndex }) - bus.$emit('reset-default-value', this.currentFilterCom.id) + bus.$emit('reset-default-value', this.currentFilterCom) this.closeFilter() }, reFreshComponent(component) { diff --git a/core/frontend/src/components/canvas/components/editor/ComponentWrapper.vue b/core/frontend/src/components/canvas/components/editor/ComponentWrapper.vue index f3ad7c68cf..e498d062e6 100644 --- a/core/frontend/src/components/canvas/components/editor/ComponentWrapper.vue +++ b/core/frontend/src/components/canvas/components/editor/ComponentWrapper.vue @@ -399,13 +399,6 @@ export default { .component-active { z-index: 1; } - -.fullscreen { - transform: translate(0); - .main_view { - z-index: 0; - } -} .user-view { transform: translate(0); } diff --git a/core/frontend/src/components/canvas/components/editor/Preview.vue b/core/frontend/src/components/canvas/components/editor/Preview.vue index 357c632813..160735c00b 100644 --- a/core/frontend/src/components/canvas/components/editor/Preview.vue +++ b/core/frontend/src/components/canvas/components/editor/Preview.vue @@ -445,6 +445,7 @@ export default { this.canvasId === 'canvas-main' && bus.$on('pcChartDetailsDialog', this.openChartDetailsDialog) bus.$on('trigger-search-button', this.triggerSearchButton) bus.$on('trigger-reset-button', this.triggerResetButton) + bus.$on('trigger-filter-loaded', this.triggerFilterLoaded) this.initPdfTemplate() }, beforeDestroy() { @@ -460,10 +461,17 @@ export default { this.canvasId === 'canvas-main' && bus.$off('pcChartDetailsDialog', this.openChartDetailsDialog) bus.$off('trigger-search-button', this.triggerSearchButton) bus.$off('trigger-reset-button', this.triggerResetButton) + bus.$off('trigger-filter-loaded', this.triggerFilterLoaded) }, methods: { - filterLoaded(p) { + triggerFilterLoaded({ canvasIdStr, panelId, p }) { + if (this.panelInfo.id === panelId && !canvasIdStr.includes(this.canvasId)) { + this.filterLoaded(p, canvasIdStr) + } + }, + filterLoaded(p, canvasIdStr = '') { buildAfterFilterLoaded(this.filterMap, p) + bus.$emit('trigger-filter-loaded', { canvasIdStr: (canvasIdStr + this.canvasId), panelId: this.panelInfo.id, p }) }, getWrapperChildRefs() { return this.$refs['viewWrapperChild'] diff --git a/core/frontend/src/components/canvas/components/editor/ViewTrackBar.vue b/core/frontend/src/components/canvas/components/editor/ViewTrackBar.vue index e2a87a145f..76ab54df63 100644 --- a/core/frontend/src/components/canvas/components/editor/ViewTrackBar.vue +++ b/core/frontend/src/components/canvas/components/editor/ViewTrackBar.vue @@ -76,6 +76,9 @@ export default { .track-menu { border: #3a8ee6 1px solid; + position: absolute !important; + top: 0 !important; + left: 0 !important; } diff --git a/core/frontend/src/components/canvas/utils/utils.js b/core/frontend/src/components/canvas/utils/utils.js index 630adfcbb8..19ed237940 100644 --- a/core/frontend/src/components/canvas/utils/utils.js +++ b/core/frontend/src/components/canvas/utils/utils.js @@ -136,6 +136,7 @@ export function panelDataPrepare(componentData, componentStyle, callback) { } if (item.type === 'custom') { item.options.manualModify = false + item.options.loaded = false } if (item.filters && item.filters.length > 0) { item.filters = [] diff --git a/core/frontend/src/components/widget/deWidget/DeDate.vue b/core/frontend/src/components/widget/deWidget/DeDate.vue index ff355563cc..36b42c5827 100644 --- a/core/frontend/src/components/widget/deWidget/DeDate.vue +++ b/core/frontend/src/components/widget/deWidget/DeDate.vue @@ -418,8 +418,10 @@ export default { this.$refs.dateRef.hidePicker() } }, - resetDefaultValue(id) { - if (this.inDraw && this.manualModify && this.element.id === id) { + resetDefaultValue(ele) { + const id = ele.id + const eleVal = ele.options.value.toString() + if (this.inDraw && this.manualModify && this.element.id === id && this.values.toString() !== eleVal && this.defaultValueStr === eleVal) { if (!this.element.options.attrs.default.isDynamic) { this.values = this.fillValueDerfault() this.dateChange(this.values) diff --git a/core/frontend/src/components/widget/deWidget/DeInputSearch.vue b/core/frontend/src/components/widget/deWidget/DeInputSearch.vue index f2becf0ad6..cd63f8d1bc 100644 --- a/core/frontend/src/components/widget/deWidget/DeInputSearch.vue +++ b/core/frontend/src/components/widget/deWidget/DeInputSearch.vue @@ -99,8 +99,10 @@ export default { clearHandler() { this.value = null }, - resetDefaultValue(id) { - if (this.inDraw && this.manualModify && this.element.id === id) { + resetDefaultValue(ele) { + const id = ele.id + const eleVal = ele.options.value.toString() + if (this.inDraw && this.manualModify && this.element.id === id && this.value.toString() !== eleVal && this.defaultValueStr === eleVal) { this.value = this.fillValueDerfault() this.search() } diff --git a/core/frontend/src/components/widget/deWidget/DeNumberRange.vue b/core/frontend/src/components/widget/deWidget/DeNumberRange.vue index beb9410fdf..bc20b38d30 100644 --- a/core/frontend/src/components/widget/deWidget/DeNumberRange.vue +++ b/core/frontend/src/components/widget/deWidget/DeNumberRange.vue @@ -160,8 +160,10 @@ export default { this.form.min = null this.form.max = null }, - resetDefaultValue(id) { - if (this.inDraw && this.manualModify && this.element.id === id) { + resetDefaultValue(ele) { + const id = ele.id + const eleVal = ele.options.value.toString() + if (this.inDraw && this.manualModify && this.element.id === id && this.defaultValueStr === eleVal) { if (!this.element.options.value) { this.form.min = null this.form.max = null diff --git a/core/frontend/src/components/widget/deWidget/DeSelect.vue b/core/frontend/src/components/widget/deWidget/DeSelect.vue index 970ec23773..d8097e4361 100644 --- a/core/frontend/src/components/widget/deWidget/DeSelect.vue +++ b/core/frontend/src/components/widget/deWidget/DeSelect.vue @@ -345,8 +345,10 @@ export default { this.$refs.deSelect.$refs.visualSelect.blur() } }, - resetDefaultValue(id) { - if (this.inDraw && this.manualModify && this.element.id === id) { + resetDefaultValue(ele) { + const id = ele.id + const eleVal = ele.options.value.toString() + if (this.inDraw && this.manualModify && this.element.id === id && this.value.toString() !== eleVal && this.defaultValueStr === eleVal) { if (this.selectFirst) { this.fillFirstValue() this.firstChange(this.value) @@ -380,6 +382,8 @@ export default { componentId: this.element.id, val: (this.value && Array.isArray(this.value)) ? this.value.join(',') : this.value }) + this.element.options.loaded = true + this.$store.commit('setComponentWithId', this.element) } }, refreshLoad() { diff --git a/core/frontend/src/components/widget/deWidget/DeSelectGrid.vue b/core/frontend/src/components/widget/deWidget/DeSelectGrid.vue index 892719dfee..9fdd8ae81d 100644 --- a/core/frontend/src/components/widget/deWidget/DeSelectGrid.vue +++ b/core/frontend/src/components/widget/deWidget/DeSelectGrid.vue @@ -12,6 +12,8 @@ :size="size" prefix-icon="el-icon-search" clearable + @input="filterMethod" + @clear="filterMethod" />
@@ -31,7 +33,7 @@ v-model="value" @change="handleCheckedChange" > -