dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 整理代码

Browse Source
This commit is contained in:
taojinlong 2022-03-16 15:44:59 +08:00
parent 1f5fd16564
commit 98dc2436a2
3 changed files with 35 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2188,9 +2188,9 @@ public class DataSetTableService {
+ qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()) + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp())
.map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); .map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>(); List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
DatasetTableExample example = new DatasetTableExample(); DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name());
datasetTableMapper.selectByExample(example).forEach(datasetTable -> { datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains( if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(
datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) { datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
@ -2202,6 +2202,7 @@ public class DataSetTableService {
return; return;
} }
//DatasetTable
DatasetTable record = new DatasetTable(); DatasetTable record = new DatasetTable();
record.setSyncStatus(JobStatus.Error.name()); record.setSyncStatus(JobStatus.Error.name());
example.clear(); example.clear();
@ -2209,6 +2210,14 @@ public class DataSetTableService {
.andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); .andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
datasetTableMapper.updateByExampleSelective(record, example); datasetTableMapper.updateByExampleSelective(record, example);
//Task
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();
criteria.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())).andStatusEqualTo(JobStatus.Underway.name());
List<DatasetTableTask> datasetTableTasks = dataSetTableTaskService.list(datasetTableTaskExample);
dataSetTableTaskService.updateTaskStatus(datasetTableTasks, JobStatus.Error);
//TaskLog
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo("Job stopped due to system error."); datasetTableTaskLog.setInfo("Job stopped due to system error.");
@ -2216,12 +2225,8 @@ public class DataSetTableService {
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample();
datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()) datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name())
.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); .andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
List<String> taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream()
.map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList());
datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample);
dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error);
for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) { for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) {
extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId());
extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId());

16
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -161,13 +161,13 @@ public class DataSetTableTaskService {
return datasetTableTaskMapper.selectByPrimaryKey(id); return datasetTableTaskMapper.selectByPrimaryKey(id);
} }
public void updateTaskStatus(List<String> taskIds, JobStatus lastExecStatus) {
if (CollectionUtils.isEmpty(taskIds)) { public List<DatasetTableTask> list(DatasetTableTaskExample example) {
return; return datasetTableTaskMapper.selectByExample(example);
} }
DatasetTableTaskExample example = new DatasetTableTaskExample();
example.createCriteria().andIdIn(taskIds);
List<DatasetTableTask> datasetTableTasks = datasetTableTaskMapper.selectByExample(example); public void updateTaskStatus(List<DatasetTableTask> datasetTableTasks, JobStatus lastExecStatus) {
for (DatasetTableTask tableTask : datasetTableTasks) { for (DatasetTableTask tableTask : datasetTableTasks) {
updateTaskStatus(tableTask, lastExecStatus); updateTaskStatus(tableTask, lastExecStatus);
} }
@ -202,7 +202,7 @@ public class DataSetTableTaskService {
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) { if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name()); datasetTableTask.setStatus(TaskStatus.Stopped.name());
} else { } else {
datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId()); // datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
datasetTableTask.setLastExecStatus(lastExecStatus.name()); datasetTableTask.setLastExecStatus(lastExecStatus.name());
if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) { if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) {
BaseGridRequest request = new BaseGridRequest(); BaseGridRequest request = new BaseGridRequest();

52
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -140,7 +140,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTableId(datasetTable.getId()); datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
return !CollectionUtils.isNotEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name()); return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
} else { } else {
datasetTableTask.setLastExecTime(startTime); datasetTableTask.setLastExecTime(startTime);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
@ -159,7 +159,7 @@ public class ExtractDataService {
return; return;
} }
UpdateType updateType = UpdateType.valueOf(type); UpdateType updateType = UpdateType.valueOf(type);
DatasetTableTaskLog datasetTableTaskLog;
if (datasetTableFields == null) { if (datasetTableFields == null) {
datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build()); datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
} }
@ -174,10 +174,10 @@ public class ExtractDataService {
return o1.getColumnIndex().compareTo(o2.getColumnIndex()); return o1.getColumnIndex().compareTo(o2.getColumnIndex());
}); });
DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
switch (updateType) { switch (updateType) {
case all_scope: // 全量更新 case all_scope: // 全量更新
try { try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields); createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields); createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
Long execTime = System.currentTimeMillis(); Long execTime = System.currentTimeMillis();
@ -222,7 +222,7 @@ public class ExtractDataService {
toDelete.forEach(datasetTableField -> dataSetTableFieldsService.delete(datasetTableField.getId())); toDelete.forEach(datasetTableField -> dataSetTableFieldsService.delete(datasetTableField.getId()));
} }
} catch (Exception e) { } catch (Exception e) {
saveErrorLog(datasetTableId, null, e); saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId))); dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally { } finally {
@ -233,7 +233,6 @@ public class ExtractDataService {
case add_scope: // 增量更新 case add_scope: // 增量更新
try { try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
Long execTime = System.currentTimeMillis(); Long execTime = System.currentTimeMillis();
if (!engineService.isSimpleMode()) { if (!engineService.isSimpleMode()) {
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
@ -245,7 +244,7 @@ public class ExtractDataService {
saveSuccessLog(datasetTableTaskLog); saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) { } catch (Exception e) {
saveErrorLog(datasetTableId, null, e); saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
} finally { } finally {
deleteFile("incremental_add", datasetTableId); deleteFile("incremental_add", datasetTableId);
@ -312,7 +311,7 @@ public class ExtractDataService {
msg = true; msg = true;
lastExecStatus = JobStatus.Completed; lastExecStatus = JobStatus.Completed;
} catch (Exception e) { } catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e); saveErrorLog(datasetTableTaskLog, e);
msg = false; msg = false;
lastExecStatus = JobStatus.Error; lastExecStatus = JobStatus.Error;
execTime = null; execTime = null;
@ -372,7 +371,7 @@ public class ExtractDataService {
msg = true; msg = true;
lastExecStatus = JobStatus.Completed; lastExecStatus = JobStatus.Completed;
} catch (Exception e) { } catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e); saveErrorLog(datasetTableTaskLog, e);
msg = false; msg = false;
lastExecStatus = JobStatus.Error; lastExecStatus = JobStatus.Error;
execTime = null; execTime = null;
@ -592,26 +591,14 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog); dataSetTableTaskLogService.save(datasetTableTaskLog);
} }
private void saveErrorLog(String datasetTableId, String taskId, Exception e) { private void saveErrorLog(DatasetTableTaskLog datasetTableTaskLog, Exception e) {
LogUtil.error("Extract data error: " + datasetTableId, e); LogUtil.error("Extract data error: " + datasetTableTaskLog.getTaskId(), e);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setInfo(e.getMessage());
datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setEndTime(System.currentTimeMillis());
if (StringUtils.isNotEmpty(taskId)) { dataSetTableTaskLogService.save(datasetTableTaskLog);
datasetTableTaskLog.setTaskId(taskId);
}
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog = datasetTableTaskLogs.get(0);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(e.getMessage());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
} }
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception { private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine(); Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
@ -671,14 +658,9 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Cron.name()); datasetTableTaskLog.setTriggerType(TriggerType.Cron.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); datasetTableTaskLog.setStartTime(System.currentTimeMillis());
if (CollectionUtils.isEmpty(datasetTableTaskLogs)) { dataSetTableTaskLogService.save(datasetTableTaskLog);
datasetTableTaskLog.setStartTime(System.currentTimeMillis()); return datasetTableTaskLog;
dataSetTableTaskLogService.save(datasetTableTaskLog);
return datasetTableTaskLog;
} else {
return datasetTableTaskLogs.get(0);
}
} }
private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId, Long startTime) { private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId, Long startTime) {
@ -687,7 +669,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
for (int i = 0; i < 10; i++) { for (int i = 0; i < 5; i++) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
return datasetTableTaskLogs.get(0); return datasetTableTaskLogs.get(0);