From 3af3e5798c1319645429a7518dfa377d277e7bb1 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Thu, 8 Jul 2021 14:00:06 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BB=BB=E5=8A=A1=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/constants/TaskStatus.java | 5 + .../commons/constants/TriggerType.java | 5 + .../dataease/dto/dataset/DataSetTaskDTO.java | 18 + .../dataset/DataSetTableTaskService.java | 48 +- .../service/dataset/ExtractDataService.java | 52 +- .../db/migration/V11__system_task.sql | 14 + frontend/src/icons/svg/task.svg | 1 + frontend/src/lang/en.js | 19 +- frontend/src/lang/tw.js | 19 +- frontend/src/lang/zh.js | 3 +- .../src/views/system/task/DatasetTaskList.vue | 558 ++++++++++++++++++ frontend/src/views/system/task/TaskRecord.vue | 181 ++++++ frontend/src/views/system/task/dataset.vue | 65 ++ frontend/src/views/system/task/index.vue | 21 + 14 files changed, 974 insertions(+), 35 deletions(-) create mode 100644 backend/src/main/java/io/dataease/commons/constants/TaskStatus.java create mode 100644 backend/src/main/java/io/dataease/commons/constants/TriggerType.java create mode 100644 backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java create mode 100644 backend/src/main/resources/db/migration/V11__system_task.sql create mode 100644 frontend/src/icons/svg/task.svg create mode 100644 frontend/src/views/system/task/DatasetTaskList.vue create mode 100644 frontend/src/views/system/task/TaskRecord.vue create mode 100644 frontend/src/views/system/task/dataset.vue create mode 100644 frontend/src/views/system/task/index.vue diff --git a/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java b/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java new file mode 100644 index 0000000000..148426c085 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java @@ -0,0 +1,5 @@ +package io.dataease.commons.constants; + +public enum TaskStatus { + Underway, Stopped +} diff --git a/backend/src/main/java/io/dataease/commons/constants/TriggerType.java b/backend/src/main/java/io/dataease/commons/constants/TriggerType.java new file mode 100644 index 0000000000..ce33114a8b --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/TriggerType.java @@ -0,0 +1,5 @@ +package io.dataease.commons.constants; + +public enum TriggerType { + Cron, Custom +} diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java new file mode 100644 index 0000000000..da22ebfbbe --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java @@ -0,0 +1,18 @@ +package io.dataease.dto.dataset; + +import io.dataease.base.domain.DatasetTableTask; +import io.dataease.base.domain.DatasetTableTaskLog; +import lombok.Getter; +import lombok.Setter; + +/** + * @Author gin + * @Date 2021/3/9 3:19 下午 + */ +@Getter +@Setter +public class DataSetTaskDTO extends DatasetTableTask { + private String datasetName; + private Long nextExecTime; + private String taskStatus; +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index 0f3e5e5839..391a92ac69 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -57,7 +57,7 @@ public class DataSetTableTaskService { public DatasetTableTask save(DataSetTaskRequest dataSetTaskRequest) throws Exception { checkName(dataSetTaskRequest); DatasetTableTask datasetTableTask = dataSetTaskRequest.getDatasetTableTask(); - if(!datasetTableTask.getType().equalsIgnoreCase("all_scope")){ + if(!datasetTableTask.getType().equalsIgnoreCase("add_scope")){ dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig()); } // check @@ -82,14 +82,24 @@ public class DataSetTableTaskService { datasetTableTask.setId(UUID.randomUUID().toString()); datasetTableTask.setCreateTime(System.currentTimeMillis()); datasetTableTask.setStatus(TaskStatus.Underway.name()); - if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { // SIMPLE 类型，提前占位 - execNow(datasetTableTask); - } datasetTableTaskMapper.insert(datasetTableTask); } else { datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask); } - scheduleService.addSchedule(datasetTableTask); + if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString()) && datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Underway.name())) { // SIMPLE 类型，提前占位 + execNow(datasetTableTask); + datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); + datasetTableTask.setLastExecTime(System.currentTimeMillis()); + update(datasetTableTask); + } + if(!datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){ + scheduleService.addSchedule(datasetTableTask); + }else { + if(datasetTableTask.getStatus().equalsIgnoreCase(JobStatus.Underway.name())){ + scheduleService.addSchedule(datasetTableTask); + } + } + return datasetTableTask; } @@ -100,17 +110,17 @@ public class DataSetTableTaskService { DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync")); } } - if (extractDataService.updateSyncStatusIsNone(dataSetTableService.get(datasetTableTask.getTableId()))) { + if (extractDataService.existSyncTask(dataSetTableService.get(datasetTableTask.getTableId()), null)) { DataEaseException.throwException(Translator.get("i18n_sync_job_exists")); - } else { //write log - DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); - datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); - datasetTableTaskLog.setTaskId(datasetTableTask.getId()); - datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - datasetTableTaskLog.setStartTime(System.currentTimeMillis()); - datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); - dataSetTableTaskLogService.save(datasetTableTaskLog); } + //write log + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); + datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + datasetTableTaskLog.setStartTime(System.currentTimeMillis()); + datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); + dataSetTableTaskLogService.save(datasetTableTaskLog); } public void delete(String id) { @@ -195,14 +205,16 @@ public class DataSetTableTaskService { } public void execTask(DatasetTableTask datasetTableTask) throws Exception{ + execNow(datasetTableTask); +// datasetTableTask.setStatus(TaskStatus.Underway.name()); + datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); + datasetTableTask.setLastExecTime(System.currentTimeMillis()); + update(datasetTableTask); + if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())){ scheduleService.fireNow(datasetTableTask); } if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){ - execNow(datasetTableTask); - datasetTableTask.setStatus(TaskStatus.Underway.name()); - datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); - update(datasetTableTask); scheduleService.addSchedule(datasetTableTask); } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index bbd3771d5c..008f2309bd 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -131,24 +131,42 @@ public class ExtractDataService { "fi\n" + "rm -rf %s\n"; - public synchronized boolean updateSyncStatusIsNone(DatasetTable datasetTable ){ + public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask){ datasetTable.setSyncStatus(JobStatus.Underway.name()); - DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name()); example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull()); - return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + if(existSyncTask){ + if(datasetTableTask != null){ + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setTableId(datasetTable.getId()); + datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); + if(CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())){ + return false; + } + } + return true; + }else { + return false; + } } public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = getDatasetTable(datasetTableId); if(datasetTable == null){ LogUtil.error("Can not find DatasetTable: " + datasetTableId); + return; } DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); - boolean isCronJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())); - if(updateSyncStatusIsNone(datasetTable) && isCronJob){ - LogUtil.info("Skip synchronization task for table : " + datasetTableId); + if(datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name()) && !datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){ + LogUtil.info("Skip synchronization task, task ID : " + datasetTableTask.getId()); + return; + } + if(existSyncTask(datasetTable, datasetTableTask)){ + LogUtil.info("Skip synchronization task for dataset, dataset ID : " + datasetTableId); return; } datasetTableTask.setLastExecTime(System.currentTimeMillis()); @@ -210,21 +228,25 @@ public class ExtractDataService { updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); dataSetTableTaskService.update(datasetTableTask); + }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); datasetTableTask.setLastExecStatus(JobStatus.Error.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } dataSetTableTaskService.update(datasetTableTask); + sendWebMsg(datasetTable, taskId,false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); deleteFile("all_scope", datasetTableId); }finally { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); - dataSetTableTaskService.update(datasetTableTask); - } } break; @@ -283,6 +305,9 @@ public class ExtractDataService { updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } dataSetTableTaskService.update(datasetTableTask); } }catch (Exception e){ @@ -290,14 +315,13 @@ public class ExtractDataService { sendWebMsg(datasetTable, taskId,false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); datasetTableTask.setLastExecStatus(JobStatus.Error.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } dataSetTableTaskService.update(datasetTableTask); deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); }finally { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTask.setStatus(TaskStatus.Stopped.name()); - dataSetTableTaskService.update(datasetTableTask); - } } break; } diff --git a/backend/src/main/resources/db/migration/V11__system_task.sql b/backend/src/main/resources/db/migration/V11__system_task.sql new file mode 100644 index 0000000000..fdd213b273 --- /dev/null +++ b/backend/src/main/resources/db/migration/V11__system_task.sql @@ -0,0 +1,14 @@ +INSERT INTO `sys_menu` VALUES (57, 1, 3, 1, '任务管理', 'sys-task', 'system/task/index', 2000, 'task', 'system-task', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL); +INSERT INTO `sys_menu` VALUES (58, 57, 0, 1, '数据集任务', 'sys-task-dataset', 'system/task/dataset', 1, 'task', 'dataset', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL); +ALTER TABLE `dataset_table_task` + ADD COLUMN `last_exec_time` BIGINT(13) NULL DEFAULT NULL COMMENT '上次执行时间' AFTER `create_time`, + ADD COLUMN `status` VARCHAR(50) NULL DEFAULT NULL COMMENT '任务状态' AFTER `last_exec_time`, + ADD COLUMN `last_exec_status` VARCHAR(50) NULL DEFAULT NULL COMMENT '上次执行结果' AFTER `last_exec_time`; + +update dataset_table_task set status='Underway' where rate='CRON'; +update dataset_table_task set status='Stopped' where rate='SIMPLE_COMPLETE'; + + +ALTER TABLE `dataset_table_task_log` ADD COLUMN `trigger_type` VARCHAR(45) NULL AFTER `create_time`; + + diff --git a/frontend/src/icons/svg/task.svg b/frontend/src/icons/svg/task.svg new file mode 100644 index 0000000000..e5847ca8fc --- /dev/null +++ b/frontend/src/icons/svg/task.svg @@ -0,0 +1 @@ + diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 7005db5a13..44c2078337 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -913,7 +913,24 @@ export default { m2: ' To', char_can_not_more_50: 'Dataset name can not more 50', task_add_title: 'Add Task', - task_edit_title: 'Edit Task' + task_edit_title: 'Edit Task', + task: { + list: 'Task list', + record: 'Execution record', + create: 'New task', + name: 'Task name', + last_exec_time: 'Last execution time', + next_exec_time: 'Next execution time', + last_exec_status: 'Last execution result', + task_status: 'Task status', + dataset: 'Data set', + search_by_name: 'Search by name', + underway: 'Running', + stopped: 'Stopped', + exec: 'Execute', + confirm_exec: 'Manual trigger execution？', + change_success: 'State switch successful' + } }, datasource: { datasource: 'Data Source', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index a710479e05..4113890173 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -913,7 +913,24 @@ export default { m2: ' 移動到', char_can_not_more_50: '數據集名稱不能超過50個字符', task_add_title: '添加任務', - task_edit_title: '編輯任務' + task_edit_title: '編輯任務', + task: { + list: '任務列表', + record: '執行紀錄', + create: '新建任務', + name: '任務名稱', + last_exec_time: '上次執行時間', + next_exec_time: '下次執行時間', + last_exec_status: '上次執行結果', + task_status: '任務狀態', + dataset: '數據集', + search_by_name: '根據名稱搜索', + underway: '運行中', + stopped: '停止', + exec: '執行', + confirm_exec: '手動觸發執行？', + change_success: '狀態切換成功' + } }, datasource: { datasource: '數據源', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 8109f5cbea..a2d1745262 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -928,7 +928,8 @@ export default { underway: '运行中', stopped: '停止', exec: '执行', - confirm_exec: '手动触发执行？' + confirm_exec: '手动触发执行？', + change_success: '状态切换成功' } }, datasource: { diff --git a/frontend/src/views/system/task/DatasetTaskList.vue b/frontend/src/views/system/task/DatasetTaskList.vue new file mode 100644 index 0000000000..c938a3119a --- /dev/null +++ b/frontend/src/views/system/task/DatasetTaskList.vue @@ -0,0 +1,558 @@ + + + + + diff --git a/frontend/src/views/system/task/TaskRecord.vue b/frontend/src/views/system/task/TaskRecord.vue new file mode 100644 index 0000000000..a9f3056c4f --- /dev/null +++ b/frontend/src/views/system/task/TaskRecord.vue @@ -0,0 +1,181 @@ + + + + + diff --git a/frontend/src/views/system/task/dataset.vue b/frontend/src/views/system/task/dataset.vue new file mode 100644 index 0000000000..fcdacfa1d9 --- /dev/null +++ b/frontend/src/views/system/task/dataset.vue @@ -0,0 +1,65 @@ + + + + + diff --git a/frontend/src/views/system/task/index.vue b/frontend/src/views/system/task/index.vue new file mode 100644 index 0000000000..0d8026c857 --- /dev/null +++ b/frontend/src/views/system/task/index.vue @@ -0,0 +1,21 @@ + + + +