From a75734b948efbf92f3bd92ef3a2140a42a5f71f4 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 15 Jun 2021 17:51:18 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E9=87=8F=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E7=9A=84sql=EF=BC=8C=E5=A2=9E=E5=8A=A0=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/dataset/DataSetTableService.java | 63 ++++++++++++++++++- .../dataset/DataSetTableTaskService.java | 2 +- .../service/dataset/ExtractDataService.java | 21 ++++--- .../resources/i18n/messages_en_US.properties | 4 +- .../resources/i18n/messages_zh_CN.properties | 3 + .../resources/i18n/messages_zh_TW.properties | 4 +- .../src/views/dataset/data/UpdateInfo.vue | 2 +- 7 files changed, 85 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index fc4bf662fa..56d212fcc9 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -81,6 +81,9 @@ public class DataSetTableService { private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; @Resource private DatasetTableTaskLogMapper datasetTableTaskLogMapper; + private static String lastUpdateTime = "${__last_update_time__}"; + private static String currentUpdateTime = "${__current_update_time__}"; + @Value("${upload.file.path}") private String path; @@ -652,6 +655,7 @@ public class DataSetTableService { } else { return new DatasetTableIncrementalConfig(); } + } public DatasetTableIncrementalConfig incrementalConfig(String datasetTableId) { @@ -661,7 +665,7 @@ public class DataSetTableService { } - public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) { + public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception{ if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { return; } @@ -671,8 +675,65 @@ public class DataSetTableService { } else { datasetTableIncrementalConfigMapper.updateByPrimaryKey(datasetTableIncrementalConfig); } + checkColumes(datasetTableIncrementalConfig); } + private void checkColumes(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception { + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(datasetTableIncrementalConfig.getTableId()); + List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(datasetTable.getId()); + datasetTableFields.sort((o1, o2) -> { + if (o1.getOriginName() == null) { + return -1; + } + if (o2.getOriginName() == null) { + return 1; + } + return o1.getOriginName().compareTo(o2.getOriginName()); + }); + List originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); + Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 + String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) + .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); + datasourceRequest.setQuery(sql); + List sqlFileds = new ArrayList<>(); + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ + sqlFileds.add(filed); + }); + sort(sqlFileds); + if(!originNameFileds.equals(sqlFileds)){ + throw new Exception(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString()); + } + } + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 + String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) + .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); + datasourceRequest.setQuery(sql); + List sqlFileds = new ArrayList<>(); + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ + sqlFileds.add(filed); + }); + sort(sqlFileds); + if(!originNameFileds.equals(sqlFileds)){ + throw new Exception(Translator.get("i18n_sql_delete_not_matching") + sqlFileds.toString()); + } + } + } + + private void sort(List sqlFileds){ + sqlFileds.sort((o1, o2) -> { + if (o1 == null) { + return -1; + } + if (o2 == null) { + return 1; + } + return o1.compareTo(o2); + }); + } private void checkName(DatasetTable datasetTable) { // if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) { // return; diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index e696aaaeb4..a67bc741a4 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -24,7 +24,7 @@ import java.util.UUID; * @Date 2021/3/4 1:26 下午 */ @Service -@Transactional +@Transactional(rollbackFor = Exception.class) public class DataSetTableTaskService { @Resource private DatasetTableTaskMapper datasetTableTaskMapper; diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 5e37ea24a9..351a7a4870 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -245,7 +245,7 @@ public class ExtractDataService { datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } - if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql); @@ -253,7 +253,7 @@ public class ExtractDataService { extractData(datasetTable, "incremental_add"); } - if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 + if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTableTaskLogs.get(0).getStartTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql); @@ -563,23 +563,26 @@ public class ExtractDataService { } private String fetchSqlField(String sql, Datasource ds) throws Exception { - String tmpSql = sql; + String tmpSql = "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 0"; DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - if (tmpSql.trim().endsWith(";")) { - tmpSql = tmpSql.substring(0, tmpSql.length() - 1) + " limit 0"; - } else { - tmpSql = tmpSql + " limit 0"; - } datasourceRequest.setQuery(tmpSql); - ListdorisFileds = new ArrayList<>(); + List dorisFileds = new ArrayList<>(); datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ dorisFileds.add(DorisTableUtils.columnName(filed)); }); return String.join(",", dorisFileds); } + private String sqlFix(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) throws Exception { TransMeta transMeta = new TransMeta(); diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index 30e941a221..bcd8d9a172 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -254,4 +254,6 @@ i18n_dataset_delete=Data set is delete i18n_chart_delete=Chart is delete i18n_not_exec_add_sync=There is no completed synchronization task. Incremental synchronization cannot be performed i18n_excel_header_empty=Excel first row can not empty -i18n_custom_ds_delete=Custom dataset union data is deleted,can not display \ No newline at end of file +i18n_custom_ds_delete=Custom dataset union data is deleted,can not display +i18n_sql_add_not_matching=The data column of incremental SQL does not match the dataset, +i18n_sql_delete_not_matching=The data column of incremental delete SQL does not match the dataset, \ No newline at end of file diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index 4b0fd4d364..0d7141b769 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -255,3 +255,6 @@ i18n_chart_delete=当前用到的视图已被删除 i18n_not_exec_add_sync=没有已完成的同步任务，无法进行增量同步 i18n_excel_header_empty=Excel第一行为空 i18n_custom_ds_delete=自定义数据集所关联数据被删除，无法正常显示 +i18n_sql_add_not_matching=增量添加 sql 的数据列与数据集不匹配, +i18n_sql_delete_not_matching=增量删除 sql 的数据列与数据集不匹配, + diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index d8f81a16a3..42a79e08ed 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -256,4 +256,6 @@ i18n_dataset_delete=當前用到的數據集已被刪除 i18n_chart_delete=當前用到的視圖已被刪除 i18n_not_exec_add_sync=沒有已經完成的同步任務，無法進行增量同步 i18n_excel_header_empty=Excel第一行為空 -i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示 \ No newline at end of file +i18n_custom_ds_delete=自定義數據集所關聯數據被刪除，無法正常顯示 +i18n_sql_add_not_matching=增量添加 sql 的數據列與數據集不匹配, +i18n_sql_delete_not_matching=增量刪除 sql 的數據列與數據集不匹配, \ No newline at end of file diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index af5ca98396..2bc355cd16 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -472,7 +472,6 @@ export default { }) }, saveIncrementalConfig() { - this.update_setting = false if (this.incrementalUpdateType === 'incrementalAdd') { this.incrementalConfig.incrementalAdd = this.sql } else { @@ -485,6 +484,7 @@ export default { type: 'success', showClose: true }) + this.update_setting = false }) }, saveTask(task) {