From 796c2444db96467a3542cf46d59aef3887ff3779 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Sun, 1 May 2022 16:24:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E5=85=A8=E9=87=8F=E4=BB=BB=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=AE=8C=E6=88=90=E5=90=8E=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=A2=9E=E9=87=8F=EF=BC=8C=E5=8F=91=E7=8E=B0=E5=A2=9E?= =?UTF-8?q?=E9=87=8F=E4=B8=8D=E7=94=9F=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/chart/ChartGroupService.java | 4 - .../service/dataset/DataSetTableService.java | 7 +- .../dataset/DataSetTableTaskLogService.java | 6 ++ .../dataset/DataSetTableTaskService.java | 72 ++++++++++------ .../service/dataset/ExtractDataService.java | 84 ++++++++++--------- 5 files changed, 100 insertions(+), 73 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/chart/ChartGroupService.java b/backend/src/main/java/io/dataease/service/chart/ChartGroupService.java index f450b53f81..dc45766d01 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartGroupService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartGroupService.java @@ -31,10 +31,6 @@ public class ChartGroupService { @Resource private ExtChartGroupMapper extChartGroupMapper; @Resource - private ExtDataSetGroupMapper extDataSetGroupMapper; - @Resource - private SysAuthService sysAuthService; - @Resource private ExtChartViewMapper extChartViewMapper; public ChartGroupDTO save(ChartGroup chartGroup) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 635bfa35c2..78c4275b18 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -2245,8 +2245,7 @@ public class DataSetTableService { List jobStoppeddDatasetTables = new ArrayList<>(); datasetTableMapper.selectByExample(example).forEach(datasetTable -> { - if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains( - datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) { + if (StringUtils.isNotEmpty(datasetTable.getQrtzInstance()) && !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) { jobStoppeddDatasetTables.add(datasetTable); } }); @@ -2314,4 +2313,8 @@ public class DataSetTableService { saveTableField(datasetTable); return datasetTable; } + + public int updateByExampleSelective(DatasetTable record, DatasetTableExample example ){ + return datasetTableMapper.updateByExampleSelective(record, example); + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index bff2d699eb..274bf8a4d5 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -10,6 +10,7 @@ import io.dataease.dto.dataset.DataSetTaskLogDTO; import io.dataease.plugins.common.base.domain.DatasetTableTaskLog; import io.dataease.plugins.common.base.domain.DatasetTableTaskLogExample; import io.dataease.plugins.common.base.mapper.DatasetTableTaskLogMapper; +import io.dataease.plugins.common.base.mapper.DatasetTableTaskMapper; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -29,8 +30,13 @@ public class DataSetTableTaskLogService { private DatasetTableTaskLogMapper datasetTableTaskLogMapper; @Resource private ExtDataSetTaskMapper extDataSetTaskMapper; + @Resource + private DatasetTableTaskMapper datasetTableTaskMapper; public DatasetTableTaskLog save(DatasetTableTaskLog datasetTableTaskLog) { + if(datasetTableTaskMapper.selectByPrimaryKey(datasetTableTaskLog.getTaskId()) == null){ + return datasetTableTaskLog; + } if (StringUtils.isEmpty(datasetTableTaskLog.getId())) { datasetTableTaskLog.setId(UUID.randomUUID().toString()); datasetTableTaskLog.setCreateTime(System.currentTimeMillis()); diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index e8cda972b6..775f8d219d 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -108,26 +108,32 @@ public class DataSetTableTaskService { DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync")); } } - if (existSyncTask(dataSetTableService.get(datasetTableTask.getTableId()), datasetTableTask)) { + if (existSyncTask(datasetTableTask.getTableId(), datasetTableTask.getId())) { DataEaseException.throwException(Translator.get("i18n_sync_job_exists")); } } - private synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask) { - datasetTable.setSyncStatus(JobStatus.Underway.name()); + private synchronized boolean existSyncTask(String datasetTableId, String datasetTableTaskId) { + DatasetTable record = new DatasetTable(); + record.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); - example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name()); - example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull()); - Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusNotEqualTo(JobStatus.Underway.name()); + example.or(example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusIsNull()); + Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(record, example) == 0; if (!existSyncTask) { Long startTime = System.currentTimeMillis(); - datasetTableTask.setLastExecTime(startTime); - datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); - datasetTableTask.setStatus(TaskStatus.Exec.name()); - update(datasetTableTask); + + DatasetTableTask datasetTableTaskRecord = new DatasetTableTask(); + datasetTableTaskRecord.setLastExecTime(startTime); + datasetTableTaskRecord.setLastExecStatus(JobStatus.Underway.name()); + datasetTableTaskRecord.setStatus(TaskStatus.Exec.name()); + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTaskId); + updateByExampleSelective(datasetTableTaskRecord, datasetTableTaskExample); + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); - datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); - datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setTableId(datasetTableId); + datasetTableTaskLog.setTaskId(datasetTableTaskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setStartTime(startTime); datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); @@ -173,7 +179,7 @@ public class DataSetTableTaskService { } } - public void checkTaskIsStopped(DatasetTableTask datasetTableTask) { + public void checkTaskIsStopped(final DatasetTableTask datasetTableTask) { if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) { BaseGridRequest request = new BaseGridRequest(); ConditionEntity conditionEntity = new ConditionEntity(); @@ -186,24 +192,30 @@ public class DataSetTableTaskService { return; } if (dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); - update(datasetTableTask); + DatasetTableTask record = new DatasetTableTask(); + record.setStatus(TaskStatus.Stopped.name()); + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId()); + updateByExampleSelective(record, datasetTableTaskExample); return; } if (dataSetTaskDTOS.get(0).getNextExecTime() > datasetTableTask.getEndTime()) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); - update(datasetTableTask); + DatasetTableTask record = new DatasetTableTask(); + record.setStatus(TaskStatus.Stopped.name()); + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId()); + updateByExampleSelective(record, datasetTableTaskExample); } } } public void updateTaskStatus(DatasetTableTask datasetTableTask, JobStatus lastExecStatus) { - datasetTableTask.setLastExecStatus(lastExecStatus.name()); + DatasetTableTask recore = new DatasetTableTask(); + recore.setLastExecStatus(lastExecStatus.name()); if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); + recore.setStatus(TaskStatus.Stopped.name()); } else { -// datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId()); - datasetTableTask.setLastExecStatus(lastExecStatus.name()); + recore.setLastExecStatus(lastExecStatus.name()); if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) { BaseGridRequest request = new BaseGridRequest(); ConditionEntity conditionEntity = new ConditionEntity(); @@ -216,19 +228,25 @@ public class DataSetTableTaskService { return; } if (dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); + recore.setStatus(TaskStatus.Stopped.name()); } else { - datasetTableTask.setStatus(TaskStatus.Underway.name()); + recore.setStatus(TaskStatus.Underway.name()); } } else { - datasetTableTask.setStatus(TaskStatus.Underway.name()); + recore.setStatus(TaskStatus.Underway.name()); } } - update(datasetTableTask); + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId()); + updateByExampleSelective(recore, datasetTableTaskExample); } - public void update(DatasetTableTask datasetTableTask) { - datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask); + public DatasetTableTask selectByPrimaryKey(String id) { + return datasetTableTaskMapper.selectByPrimaryKey(id); + } + + public void updateByExampleSelective(DatasetTableTask datasetTableTask, DatasetTableTaskExample datasetTableTaskExample) { + datasetTableTaskMapper.updateByExampleSelective(datasetTableTask, datasetTableTaskExample); } public List list(DatasetTableTask datasetTableTask) { diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index dc24ed3a8f..5238e6a74c 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -65,6 +65,7 @@ import org.quartz.JobExecutionContext; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; + import javax.annotation.Resource; import java.io.*; import java.util.*; @@ -85,12 +86,6 @@ public class ExtractDataService { @Lazy private DataSetTableTaskService dataSetTableTaskService; @Resource - private DatasourceMapper datasourceMapper; - @Resource - private DatasetTableMapper datasetTableMapper; - @Resource - private DatasetTableTaskMapper datasetTableTaskMapper; - @Resource private DatasourceService datasourceService; @Resource private ExtChartViewMapper extChartViewMapper; @@ -133,24 +128,28 @@ public class ExtractDataService { " exit 1\n" + "fi\n"; - public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) { - datasetTable.setSyncStatus(JobStatus.Underway.name()); + public synchronized boolean existSyncTask(String datasetTableId, String datasetTableTaskId, Long startTime) { + DatasetTable datasetTableRecord = new DatasetTable(); + datasetTableRecord.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); - example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name()); - example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull()); - boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusNotEqualTo(JobStatus.Underway.name()); + example.or(example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusIsNull()); + boolean existSyncTask = dataSetTableService.updateByExampleSelective(datasetTableRecord, example) == 0; if (existSyncTask) { DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); - datasetTableTaskLog.setTaskId(datasetTableTask.getId()); - datasetTableTaskLog.setTableId(datasetTable.getId()); + datasetTableTaskLog.setTaskId(datasetTableTaskId); + datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name()); } else { - datasetTableTask.setLastExecTime(startTime); - datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); - datasetTableTask.setStatus(TaskStatus.Exec.name()); - dataSetTableTaskService.update(datasetTableTask); + DatasetTableTask record = new DatasetTableTask(); + record.setLastExecTime(startTime); + record.setLastExecStatus(JobStatus.Underway.name()); + record.setStatus(TaskStatus.Exec.name()); + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTaskId); + dataSetTableTaskService.updateByExampleSelective(record, datasetTableTaskExample); return false; } } @@ -179,7 +178,7 @@ public class ExtractDataService { return o1.getColumnIndex().compareTo(o2.getColumnIndex()); }); - DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); + DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); switch (updateType) { case all_scope: // 全量更新 try { @@ -195,7 +194,7 @@ public class ExtractDataService { } replaceTable(TableUtils.tableName(datasetTableId)); saveSuccessLog(datasetTableTaskLog); - updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); + updateTableStatus(datasetTableId, JobStatus.Completed, execTime); if (ops.equalsIgnoreCase("替换")) { List oldFileds = getDatasetTableFields(datasetTable.getId()); List toAdd = new ArrayList<>(); @@ -228,7 +227,7 @@ public class ExtractDataService { } } catch (Exception e) { saveErrorLog(datasetTableTaskLog, e); - updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); + updateTableStatus(datasetTableId, JobStatus.Error, null); dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); } finally { deleteFile("all_scope", datasetTableId); @@ -247,10 +246,10 @@ public class ExtractDataService { extractExcelDataForSimpleMode(datasetTable, "incremental_add"); } saveSuccessLog(datasetTableTaskLog); - updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); + updateTableStatus(datasetTableId, JobStatus.Completed, execTime); } catch (Exception e) { saveErrorLog(datasetTableTaskLog, e); - updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); + updateTableStatus(datasetTableId, JobStatus.Error, null); } finally { deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); @@ -273,7 +272,7 @@ public class ExtractDataService { LogUtil.error("Can not find DatasetTable: " + datasetTableId); return; } - DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); + DatasetTableTask datasetTableTask = dataSetTableTaskService.selectByPrimaryKey(taskId); if (datasetTableTask == null) { return; } @@ -284,19 +283,22 @@ public class ExtractDataService { } Long startTime = System.currentTimeMillis(); - if (existSyncTask(datasetTable, datasetTableTask, startTime)) { + if (existSyncTask(datasetTable.getId(), datasetTableTask.getId(), startTime)) { LogUtil.info("Skip synchronization task for dataset due to exist others, dataset ID : " + datasetTableId); return; } DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId, startTime); UpdateType updateType = UpdateType.valueOf(type); if (context != null) { - datasetTable.setQrtzInstance(context.getFireInstanceId()); - datasetTableMapper.updateByPrimaryKeySelective(datasetTable); + DatasetTable datasetTableRecord = new DatasetTable(); + datasetTableRecord.setQrtzInstance(context.getFireInstanceId()); + DatasetTableExample example = new DatasetTableExample(); + example.createCriteria().andIdEqualTo(datasetTableId); + dataSetTableService.updateByExampleSelective(datasetTableRecord, example); } Datasource datasource = new Datasource(); if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { - datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); + datasource = datasourceService.get(datasetTable.getDataSourceId()); } else { datasource.setType(datasetTable.getType()); } @@ -337,7 +339,7 @@ public class ExtractDataService { System.out.println(ignore.getMessage()); } try { - updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); + updateTableStatus(datasetTableId, lastExecStatus, execTime); } catch (Exception ignore) { System.out.println(ignore.getMessage()); } @@ -395,7 +397,7 @@ public class ExtractDataService { } catch (Exception ignore) { } try { - updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); + updateTableStatus(datasetTableId, lastExecStatus, execTime); } catch (Exception ignore) { } } @@ -459,9 +461,9 @@ public class ExtractDataService { String dataFile = null; String script = null; String streamLoadScript = ""; - if(kettleFilesKeep){ + if (kettleFilesKeep) { streamLoadScript = shellScript; - }else { + } else { streamLoadScript = shellScriptForDeleteFile; } switch (extractType) { @@ -586,14 +588,16 @@ public class ExtractDataService { }); } - private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) { - datasetTable.setSyncStatus(completed.name()); + private void updateTableStatus(String datasetTableId, JobStatus jobStatus, Long execTime) { + DatasetTable datasetTableRecord = new DatasetTable(); + datasetTableRecord.setId(datasetTableId); + datasetTableRecord.setSyncStatus(jobStatus.name()); if (execTime != null) { - datasetTable.setLastUpdateTime(execTime); + datasetTableRecord.setLastUpdateTime(execTime); } DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andIdEqualTo(datasetTableId); - datasetTableMapper.updateByExampleSelective(datasetTable, example); + dataSetTableService.updateByExampleSelective(datasetTableRecord, example); } private void saveSuccessLog(DatasetTableTaskLog datasetTableTaskLog) { @@ -706,7 +710,7 @@ public class ExtractDataService { for (ExcelSheetData sheet : excelXlsxReader.totalSheets) { if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) { for (List dataItem : sheet.getData()) { - if(dataItem.size()>0){ + if (dataItem.size() > 0) { data.add(dataItem.toArray(new String[dataItem.size()])); } } @@ -792,9 +796,9 @@ public class ExtractDataService { DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); String columns = columnFields + ",dataease_uuid"; String streamLoadScript = ""; - if(kettleFilesKeep){ + if (kettleFilesKeep) { streamLoadScript = shellScript; - }else { + } else { streamLoadScript = shellScriptForDeleteFile; } switch (extractType) { @@ -897,9 +901,9 @@ public class ExtractDataService { case StarRocks: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfiguration.class); dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfiguration.getHost().trim(), mysqlConfiguration.getDataBase().trim(), mysqlConfiguration.getPort().toString(), mysqlConfiguration.getUsername(), mysqlConfiguration.getPassword()); - if(StringUtils.isNotEmpty(mysqlConfiguration.getExtraParams()) && mysqlConfiguration.getExtraParams().split("&").length > 0){ + if (StringUtils.isNotEmpty(mysqlConfiguration.getExtraParams()) && mysqlConfiguration.getExtraParams().split("&").length > 0) { String[] params = mysqlConfiguration.getExtraParams().split("&"); - for(int i=0;i