dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 定时任务创建全量任务，执行完成后修改为增量，发现增量不生效

Browse Source
This commit is contained in:
taojinlong 2022-05-01 16:24:42 +08:00
parent ec9139bdf4
commit afd2a7ae3b
5 changed files with 100 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/main/java/io/dataease/service/chart/ChartGroupService.java
View File

@ -31,10 +31,6 @@ public class ChartGroupService {
@Resource
private ExtChartGroupMapper extChartGroupMapper;
@Resource
private ExtDataSetGroupMapper extDataSetGroupMapper;
@Resource
private SysAuthService sysAuthService;
@Resource
private ExtChartViewMapper extChartViewMapper;
public ChartGroupDTO save(ChartGroup chartGroup) {

7
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2245,8 +2245,7 @@ public class DataSetTableService {
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(
datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
if (StringUtils.isNotEmpty(datasetTable.getQrtzInstance()) && !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
jobStoppeddDatasetTables.add(datasetTable);
}
});
@ -2314,4 +2313,8 @@ public class DataSetTableService {
saveTableField(datasetTable);
return datasetTable;
}
public int updateByExampleSelective(DatasetTable record, DatasetTableExample example ){
return datasetTableMapper.updateByExampleSelective(record, example);
}
}

6
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java
View File

@ -10,6 +10,7 @@ import io.dataease.dto.dataset.DataSetTaskLogDTO;
import io.dataease.plugins.common.base.domain.DatasetTableTaskLog;
import io.dataease.plugins.common.base.domain.DatasetTableTaskLogExample;
import io.dataease.plugins.common.base.mapper.DatasetTableTaskLogMapper;
import io.dataease.plugins.common.base.mapper.DatasetTableTaskMapper;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -29,8 +30,13 @@ public class DataSetTableTaskLogService {
private DatasetTableTaskLogMapper datasetTableTaskLogMapper;
@Resource
private ExtDataSetTaskMapper extDataSetTaskMapper;
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;
public DatasetTableTaskLog save(DatasetTableTaskLog datasetTableTaskLog) {
if(datasetTableTaskMapper.selectByPrimaryKey(datasetTableTaskLog.getTaskId()) == null){
return datasetTableTaskLog;
}
if (StringUtils.isEmpty(datasetTableTaskLog.getId())) {
datasetTableTaskLog.setId(UUID.randomUUID().toString());
datasetTableTaskLog.setCreateTime(System.currentTimeMillis());

72
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -108,26 +108,32 @@ public class DataSetTableTaskService {
DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync"));
}
}
if (existSyncTask(dataSetTableService.get(datasetTableTask.getTableId()), datasetTableTask)) {
if (existSyncTask(datasetTableTask.getTableId(), datasetTableTask.getId())) {
DataEaseException.throwException(Translator.get("i18n_sync_job_exists"));
}
}
private synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask) {
datasetTable.setSyncStatus(JobStatus.Underway.name());
private synchronized boolean existSyncTask(String datasetTableId, String datasetTableTaskId) {
DatasetTable record = new DatasetTable();
record.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull());
Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0;
example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusIsNull());
Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(record, example) == 0;
if (!existSyncTask) {
Long startTime = System.currentTimeMillis();
datasetTableTask.setLastExecTime(startTime);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
datasetTableTask.setStatus(TaskStatus.Exec.name());
update(datasetTableTask);
DatasetTableTask datasetTableTaskRecord = new DatasetTableTask();
datasetTableTaskRecord.setLastExecTime(startTime);
datasetTableTaskRecord.setLastExecStatus(JobStatus.Underway.name());
datasetTableTaskRecord.setStatus(TaskStatus.Exec.name());
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTaskId);
updateByExampleSelective(datasetTableTaskRecord, datasetTableTaskExample);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableTask.getTableId());
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(datasetTableTaskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(startTime);
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
@ -173,7 +179,7 @@ public class DataSetTableTaskService {
}
}
public void checkTaskIsStopped(DatasetTableTask datasetTableTask) {
public void checkTaskIsStopped(final DatasetTableTask datasetTableTask) {
if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) {
BaseGridRequest request = new BaseGridRequest();
ConditionEntity conditionEntity = new ConditionEntity();
@ -186,24 +192,30 @@ public class DataSetTableTaskService {
return;
}
if (dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
update(datasetTableTask);
DatasetTableTask record = new DatasetTableTask();
record.setStatus(TaskStatus.Stopped.name());
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId());
updateByExampleSelective(record, datasetTableTaskExample);
return;
}
if (dataSetTaskDTOS.get(0).getNextExecTime() > datasetTableTask.getEndTime()) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
update(datasetTableTask);
DatasetTableTask record = new DatasetTableTask();
record.setStatus(TaskStatus.Stopped.name());
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId());
updateByExampleSelective(record, datasetTableTaskExample);
}
}
}
public void updateTaskStatus(DatasetTableTask datasetTableTask, JobStatus lastExecStatus) {
datasetTableTask.setLastExecStatus(lastExecStatus.name());
DatasetTableTask recore = new DatasetTableTask();
recore.setLastExecStatus(lastExecStatus.name());
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
recore.setStatus(TaskStatus.Stopped.name());
} else {
// datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
datasetTableTask.setLastExecStatus(lastExecStatus.name());
recore.setLastExecStatus(lastExecStatus.name());
if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) {
BaseGridRequest request = new BaseGridRequest();
ConditionEntity conditionEntity = new ConditionEntity();
@ -216,19 +228,25 @@ public class DataSetTableTaskService {
return;
}
if (dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
recore.setStatus(TaskStatus.Stopped.name());
} else {
datasetTableTask.setStatus(TaskStatus.Underway.name());
recore.setStatus(TaskStatus.Underway.name());
}
} else {
datasetTableTask.setStatus(TaskStatus.Underway.name());
recore.setStatus(TaskStatus.Underway.name());
}
}
update(datasetTableTask);
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTask.getId());
updateByExampleSelective(recore, datasetTableTaskExample);
}
public void update(DatasetTableTask datasetTableTask) {
datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask);
public DatasetTableTask selectByPrimaryKey(String id) {
return datasetTableTaskMapper.selectByPrimaryKey(id);
}
public void updateByExampleSelective(DatasetTableTask datasetTableTask, DatasetTableTaskExample datasetTableTaskExample) {
datasetTableTaskMapper.updateByExampleSelective(datasetTableTask, datasetTableTaskExample);
}
public List<DatasetTableTask> list(DatasetTableTask datasetTableTask) {

68
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -65,6 +65,7 @@ import org.quartz.JobExecutionContext;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.*;
import java.util.*;
@ -85,12 +86,6 @@ public class ExtractDataService {
@Lazy
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private DatasourceMapper datasourceMapper;
@Resource
private DatasetTableMapper datasetTableMapper;
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;
@Resource
private DatasourceService datasourceService;
@Resource
private ExtChartViewMapper extChartViewMapper;
@ -133,24 +128,28 @@ public class ExtractDataService {
" exit 1\n" +
"fi\n";
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) {
datasetTable.setSyncStatus(JobStatus.Underway.name());
public synchronized boolean existSyncTask(String datasetTableId, String datasetTableTaskId, Long startTime) {
DatasetTable datasetTableRecord = new DatasetTable();
datasetTableRecord.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull());
boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0;
example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusNotEqualTo(JobStatus.Underway.name());
example.or(example.createCriteria().andIdEqualTo(datasetTableId).andSyncStatusIsNull());
boolean existSyncTask = dataSetTableService.updateByExampleSelective(datasetTableRecord, example) == 0;
if (existSyncTask) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTaskId(datasetTableTask.getId());
datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setTaskId(datasetTableTaskId);
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
} else {
datasetTableTask.setLastExecTime(startTime);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
datasetTableTask.setStatus(TaskStatus.Exec.name());
dataSetTableTaskService.update(datasetTableTask);
DatasetTableTask record = new DatasetTableTask();
record.setLastExecTime(startTime);
record.setLastExecStatus(JobStatus.Underway.name());
record.setStatus(TaskStatus.Exec.name());
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
datasetTableTaskExample.createCriteria().andIdEqualTo(datasetTableTaskId);
dataSetTableTaskService.updateByExampleSelective(record, datasetTableTaskExample);
return false;
}
}
@ -195,7 +194,7 @@ public class ExtractDataService {
}
replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
updateTableStatus(datasetTableId, JobStatus.Completed, execTime);
if (ops.equalsIgnoreCase("替换")) {
List<DatasetTableField> oldFileds = getDatasetTableFields(datasetTable.getId());
List<DatasetTableField> toAdd = new ArrayList<>();
@ -228,7 +227,7 @@ public class ExtractDataService {
}
} catch (Exception e) {
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
updateTableStatus(datasetTableId, JobStatus.Error, null);
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
@ -247,10 +246,10 @@ public class ExtractDataService {
extractExcelDataForSimpleMode(datasetTable, "incremental_add");
}
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
updateTableStatus(datasetTableId, JobStatus.Completed, execTime);
} catch (Exception e) {
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
updateTableStatus(datasetTableId, JobStatus.Error, null);
} finally {
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
@ -273,7 +272,7 @@ public class ExtractDataService {
LogUtil.error("Can not find DatasetTable: " + datasetTableId);
return;
}
DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId);
DatasetTableTask datasetTableTask = dataSetTableTaskService.selectByPrimaryKey(taskId);
if (datasetTableTask == null) {
return;
}
@ -284,19 +283,22 @@ public class ExtractDataService {
}
Long startTime = System.currentTimeMillis();
if (existSyncTask(datasetTable, datasetTableTask, startTime)) {
if (existSyncTask(datasetTable.getId(), datasetTableTask.getId(), startTime)) {
LogUtil.info("Skip synchronization task for dataset due to exist others, dataset ID : " + datasetTableId);
return;
}
DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId, startTime);
UpdateType updateType = UpdateType.valueOf(type);
if (context != null) {
datasetTable.setQrtzInstance(context.getFireInstanceId());
datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
DatasetTable datasetTableRecord = new DatasetTable();
datasetTableRecord.setQrtzInstance(context.getFireInstanceId());
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTableId);
dataSetTableService.updateByExampleSelective(datasetTableRecord, example);
}
Datasource datasource = new Datasource();
if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) {
datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
datasource = datasourceService.get(datasetTable.getDataSourceId());
} else {
datasource.setType(datasetTable.getType());
}
@ -337,7 +339,7 @@ public class ExtractDataService {
System.out.println(ignore.getMessage());
}
try {
updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
updateTableStatus(datasetTableId, lastExecStatus, execTime);
} catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
@ -395,7 +397,7 @@ public class ExtractDataService {
} catch (Exception ignore) {
}
try {
updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
updateTableStatus(datasetTableId, lastExecStatus, execTime);
} catch (Exception ignore) {
}
}
@ -586,14 +588,16 @@ public class ExtractDataService {
});
}
private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) {
datasetTable.setSyncStatus(completed.name());
private void updateTableStatus(String datasetTableId, JobStatus jobStatus, Long execTime) {
DatasetTable datasetTableRecord = new DatasetTable();
datasetTableRecord.setId(datasetTableId);
datasetTableRecord.setSyncStatus(jobStatus.name());
if (execTime != null) {
datasetTable.setLastUpdateTime(execTime);
datasetTableRecord.setLastUpdateTime(execTime);
}
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdEqualTo(datasetTableId);
datasetTableMapper.updateByExampleSelective(datasetTable, example);
dataSetTableService.updateByExampleSelective(datasetTableRecord, example);
}
private void saveSuccessLog(DatasetTableTaskLog datasetTableTaskLog) {