diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java index e0f7a118e9..07a400d6d1 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java @@ -18,14 +18,20 @@ public class DatasetTableField implements Serializable { private String dataeaseName; + private String groupType; + private String type; private Integer size; private Integer deType; + private Integer deTypeFormat; + private Integer deExtractType; + private Integer extField; + private Boolean checked; private Integer columnIndex; diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java index 9d5214f0ec..60eb7802b8 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java @@ -454,6 +454,76 @@ public class DatasetTableFieldExample { return (Criteria) this; } + public Criteria andGroupTypeIsNull() { + addCriterion("group_type is null"); + return (Criteria) this; + } + + public Criteria andGroupTypeIsNotNull() { + addCriterion("group_type is not null"); + return (Criteria) this; + } + + public Criteria andGroupTypeEqualTo(String value) { + addCriterion("group_type =", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeNotEqualTo(String value) { + addCriterion("group_type <>", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeGreaterThan(String value) { + addCriterion("group_type >", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeGreaterThanOrEqualTo(String value) { + addCriterion("group_type >=", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeLessThan(String value) { + addCriterion("group_type <", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeLessThanOrEqualTo(String value) { + addCriterion("group_type <=", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeLike(String value) { + addCriterion("group_type like", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeNotLike(String value) { + addCriterion("group_type not like", value, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeIn(List values) { + addCriterion("group_type in", values, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeNotIn(List values) { + addCriterion("group_type not in", values, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeBetween(String value1, String value2) { + addCriterion("group_type between", value1, value2, "groupType"); + return (Criteria) this; + } + + public Criteria andGroupTypeNotBetween(String value1, String value2) { + addCriterion("group_type not between", value1, value2, "groupType"); + return (Criteria) this; + } + public Criteria andTypeIsNull() { addCriterion("`type` is null"); return (Criteria) this; @@ -644,6 +714,66 @@ public class DatasetTableFieldExample { return (Criteria) this; } + public Criteria andDeTypeFormatIsNull() { + addCriterion("de_type_format is null"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatIsNotNull() { + addCriterion("de_type_format is not null"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatEqualTo(Integer value) { + addCriterion("de_type_format =", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatNotEqualTo(Integer value) { + addCriterion("de_type_format <>", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatGreaterThan(Integer value) { + addCriterion("de_type_format >", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatGreaterThanOrEqualTo(Integer value) { + addCriterion("de_type_format >=", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatLessThan(Integer value) { + addCriterion("de_type_format <", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatLessThanOrEqualTo(Integer value) { + addCriterion("de_type_format <=", value, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatIn(List values) { + addCriterion("de_type_format in", values, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatNotIn(List values) { + addCriterion("de_type_format not in", values, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatBetween(Integer value1, Integer value2) { + addCriterion("de_type_format between", value1, value2, "deTypeFormat"); + return (Criteria) this; + } + + public Criteria andDeTypeFormatNotBetween(Integer value1, Integer value2) { + addCriterion("de_type_format not between", value1, value2, "deTypeFormat"); + return (Criteria) this; + } + public Criteria andDeExtractTypeIsNull() { addCriterion("de_extract_type is null"); return (Criteria) this; @@ -704,6 +834,66 @@ public class DatasetTableFieldExample { return (Criteria) this; } + public Criteria andExtFieldIsNull() { + addCriterion("ext_field is null"); + return (Criteria) this; + } + + public Criteria andExtFieldIsNotNull() { + addCriterion("ext_field is not null"); + return (Criteria) this; + } + + public Criteria andExtFieldEqualTo(Integer value) { + addCriterion("ext_field =", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldNotEqualTo(Integer value) { + addCriterion("ext_field <>", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldGreaterThan(Integer value) { + addCriterion("ext_field >", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldGreaterThanOrEqualTo(Integer value) { + addCriterion("ext_field >=", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldLessThan(Integer value) { + addCriterion("ext_field <", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldLessThanOrEqualTo(Integer value) { + addCriterion("ext_field <=", value, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldIn(List values) { + addCriterion("ext_field in", values, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldNotIn(List values) { + addCriterion("ext_field not in", values, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldBetween(Integer value1, Integer value2) { + addCriterion("ext_field between", value1, value2, "extField"); + return (Criteria) this; + } + + public Criteria andExtFieldNotBetween(Integer value1, Integer value2) { + addCriterion("ext_field not between", value1, value2, "extField"); + return (Criteria) this; + } + public Criteria andCheckedIsNull() { addCriterion("`checked` is null"); return (Criteria) this; diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableTask.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableTask.java index c6171d0de6..46da2b5eeb 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableTask.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableTask.java @@ -25,5 +25,11 @@ public class DatasetTableTask implements Serializable { private Long createTime; + private Long lastExecTime; + + private String status; + + private String lastExecStatus; + private static final long serialVersionUID = 1L; } \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskExample.java index 531cdce844..801373393a 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskExample.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskExample.java @@ -773,6 +773,206 @@ public class DatasetTableTaskExample { addCriterion("create_time not between", value1, value2, "createTime"); return (Criteria) this; } + + public Criteria andLastExecTimeIsNull() { + addCriterion("last_exec_time is null"); + return (Criteria) this; + } + + public Criteria andLastExecTimeIsNotNull() { + addCriterion("last_exec_time is not null"); + return (Criteria) this; + } + + public Criteria andLastExecTimeEqualTo(Long value) { + addCriterion("last_exec_time =", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeNotEqualTo(Long value) { + addCriterion("last_exec_time <>", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeGreaterThan(Long value) { + addCriterion("last_exec_time >", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeGreaterThanOrEqualTo(Long value) { + addCriterion("last_exec_time >=", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeLessThan(Long value) { + addCriterion("last_exec_time <", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeLessThanOrEqualTo(Long value) { + addCriterion("last_exec_time <=", value, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeIn(List values) { + addCriterion("last_exec_time in", values, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeNotIn(List values) { + addCriterion("last_exec_time not in", values, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeBetween(Long value1, Long value2) { + addCriterion("last_exec_time between", value1, value2, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andLastExecTimeNotBetween(Long value1, Long value2) { + addCriterion("last_exec_time not between", value1, value2, "lastExecTime"); + return (Criteria) this; + } + + public Criteria andStatusIsNull() { + addCriterion("`status` is null"); + return (Criteria) this; + } + + public Criteria andStatusIsNotNull() { + addCriterion("`status` is not null"); + return (Criteria) this; + } + + public Criteria andStatusEqualTo(String value) { + addCriterion("`status` =", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotEqualTo(String value) { + addCriterion("`status` <>", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThan(String value) { + addCriterion("`status` >", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusGreaterThanOrEqualTo(String value) { + addCriterion("`status` >=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThan(String value) { + addCriterion("`status` <", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLessThanOrEqualTo(String value) { + addCriterion("`status` <=", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusLike(String value) { + addCriterion("`status` like", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotLike(String value) { + addCriterion("`status` not like", value, "status"); + return (Criteria) this; + } + + public Criteria andStatusIn(List values) { + addCriterion("`status` in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotIn(List values) { + addCriterion("`status` not in", values, "status"); + return (Criteria) this; + } + + public Criteria andStatusBetween(String value1, String value2) { + addCriterion("`status` between", value1, value2, "status"); + return (Criteria) this; + } + + public Criteria andStatusNotBetween(String value1, String value2) { + addCriterion("`status` not between", value1, value2, "status"); + return (Criteria) this; + } + + public Criteria andLastExecStatusIsNull() { + addCriterion("last_exec_status is null"); + return (Criteria) this; + } + + public Criteria andLastExecStatusIsNotNull() { + addCriterion("last_exec_status is not null"); + return (Criteria) this; + } + + public Criteria andLastExecStatusEqualTo(String value) { + addCriterion("last_exec_status =", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusNotEqualTo(String value) { + addCriterion("last_exec_status <>", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusGreaterThan(String value) { + addCriterion("last_exec_status >", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusGreaterThanOrEqualTo(String value) { + addCriterion("last_exec_status >=", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusLessThan(String value) { + addCriterion("last_exec_status <", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusLessThanOrEqualTo(String value) { + addCriterion("last_exec_status <=", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusLike(String value) { + addCriterion("last_exec_status like", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusNotLike(String value) { + addCriterion("last_exec_status not like", value, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusIn(List values) { + addCriterion("last_exec_status in", values, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusNotIn(List values) { + addCriterion("last_exec_status not in", values, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusBetween(String value1, String value2) { + addCriterion("last_exec_status between", value1, value2, "lastExecStatus"); + return (Criteria) this; + } + + public Criteria andLastExecStatusNotBetween(String value1, String value2) { + addCriterion("last_exec_status not between", value1, value2, "lastExecStatus"); + return (Criteria) this; + } } public static class Criteria extends GeneratedCriteria { diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLog.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLog.java index 3419163d1b..3c5040feac 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLog.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLog.java @@ -19,6 +19,8 @@ public class DatasetTableTaskLog implements Serializable { private Long createTime; + private String triggerType; + private String info; private static final long serialVersionUID = 1L; diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLogExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLogExample.java index ed3337762d..82ca70c6ab 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLogExample.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableTaskLogExample.java @@ -563,6 +563,76 @@ public class DatasetTableTaskLogExample { addCriterion("create_time not between", value1, value2, "createTime"); return (Criteria) this; } + + public Criteria andTriggerTypeIsNull() { + addCriterion("trigger_type is null"); + return (Criteria) this; + } + + public Criteria andTriggerTypeIsNotNull() { + addCriterion("trigger_type is not null"); + return (Criteria) this; + } + + public Criteria andTriggerTypeEqualTo(String value) { + addCriterion("trigger_type =", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeNotEqualTo(String value) { + addCriterion("trigger_type <>", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeGreaterThan(String value) { + addCriterion("trigger_type >", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeGreaterThanOrEqualTo(String value) { + addCriterion("trigger_type >=", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeLessThan(String value) { + addCriterion("trigger_type <", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeLessThanOrEqualTo(String value) { + addCriterion("trigger_type <=", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeLike(String value) { + addCriterion("trigger_type like", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeNotLike(String value) { + addCriterion("trigger_type not like", value, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeIn(List values) { + addCriterion("trigger_type in", values, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeNotIn(List values) { + addCriterion("trigger_type not in", values, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeBetween(String value1, String value2) { + addCriterion("trigger_type between", value1, value2, "triggerType"); + return (Criteria) this; + } + + public Criteria andTriggerTypeNotBetween(String value1, String value2) { + addCriterion("trigger_type not between", value1, value2, "triggerType"); + return (Criteria) this; + } } public static class Criteria extends GeneratedCriteria { diff --git a/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml index cc2d33244a..1b3a36b2c8 100644 --- a/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml @@ -7,10 +7,13 @@ + + + @@ -74,8 +77,8 @@ - id, table_id, origin_name, `name`, dataease_name, `type`, `size`, de_type, de_extract_type, - `checked`, column_index, last_sync_time + id, table_id, origin_name, `name`, dataease_name, group_type, `type`, `size`, de_type, + de_type_format, de_extract_type, ext_field, `checked`, column_index, last_sync_time select @@ -108,11 +112,13 @@ insert into dataset_table_task (id, table_id, `name`, `type`, start_time, rate, cron, `end`, end_time, - create_time) + create_time, last_exec_time, `status`, + last_exec_status) values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{startTime,jdbcType=BIGINT}, #{rate,jdbcType=VARCHAR}, #{cron,jdbcType=VARCHAR}, #{end,jdbcType=VARCHAR}, #{endTime,jdbcType=BIGINT}, - #{createTime,jdbcType=BIGINT}) + #{createTime,jdbcType=BIGINT}, #{lastExecTime,jdbcType=BIGINT}, #{status,jdbcType=VARCHAR}, + #{lastExecStatus,jdbcType=VARCHAR}) insert into dataset_table_task @@ -147,6 +153,15 @@ create_time, + + last_exec_time, + + + `status`, + + + last_exec_status, + @@ -179,6 +194,15 @@ #{createTime,jdbcType=BIGINT}, + + #{lastExecTime,jdbcType=BIGINT}, + + + #{status,jdbcType=VARCHAR}, + + + #{lastExecStatus,jdbcType=VARCHAR}, + - SELECT dataset_table_task_log.*, - dataset_table_task.name + SELECT dataset_table_task_log.*, dataset_table_task.name FROM dataset_table_task_log LEFT JOIN dataset_table_task ON dataset_table_task_log.task_id = dataset_table_task.id - WHERE dataset_table_task_log.table_id = #{tableId,jdbcType=VARCHAR} - ORDER BY dataset_table_task_log.create_time desc + + + + + order by ${orderByClause} + + + ORDER BY dataset_table_task_log.create_time desc + + + + + diff --git a/backend/src/main/java/io/dataease/commons/constants/ScheduleType.java b/backend/src/main/java/io/dataease/commons/constants/ScheduleType.java index 18aeb65588..98b4fde4dd 100644 --- a/backend/src/main/java/io/dataease/commons/constants/ScheduleType.java +++ b/backend/src/main/java/io/dataease/commons/constants/ScheduleType.java @@ -1,5 +1,5 @@ package io.dataease.commons.constants; public enum ScheduleType { - CRON, SIMPLE, SIMPLE_COMPLETE + CRON, SIMPLE } diff --git a/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java b/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java new file mode 100644 index 0000000000..148426c085 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/TaskStatus.java @@ -0,0 +1,5 @@ +package io.dataease.commons.constants; + +public enum TaskStatus { + Underway, Stopped +} diff --git a/backend/src/main/java/io/dataease/commons/constants/TriggerType.java b/backend/src/main/java/io/dataease/commons/constants/TriggerType.java new file mode 100644 index 0000000000..ce33114a8b --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/TriggerType.java @@ -0,0 +1,5 @@ +package io.dataease.commons.constants; + +public enum TriggerType { + Cron, Custom +} diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java index 7edb03030e..391dc78a2e 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java @@ -1,8 +1,17 @@ package io.dataease.controller.dataset; +import com.github.pagehelper.Page; +import com.github.pagehelper.PageHelper; import io.dataease.base.domain.DatasetTableTask; +import io.dataease.commons.utils.PageUtils; +import io.dataease.commons.utils.Pager; import io.dataease.controller.request.dataset.DataSetTaskRequest; +import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.controller.sys.response.SysUserGridResponse; +import io.dataease.dto.dataset.DataSetTaskDTO; +import io.dataease.service.dataset.DataSetTableTaskLogService; import io.dataease.service.dataset.DataSetTableTaskService; +import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; @@ -17,6 +26,8 @@ import java.util.List; public class DataSetTableTaskController { @Resource private DataSetTableTaskService dataSetTableTaskService; + @Resource + private DataSetTableTaskLogService dataSetTableTaskLogService; @PostMapping("save") public DatasetTableTask save(@RequestBody DataSetTaskRequest dataSetTaskRequest) throws Exception { @@ -32,4 +43,28 @@ public class DataSetTableTaskController { public List list(@RequestBody DatasetTableTask datasetTableTask) { return dataSetTableTaskService.list(datasetTableTask); } + + @ApiOperation("查看数据集任务") + @PostMapping("/pageList/{goPage}/{pageSize}") + public Pager> taskList(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody BaseGridRequest request) { + Page page = PageHelper.startPage(goPage, pageSize, true); + + return PageUtils.setPageInfo(page, dataSetTableTaskService.taskList(request)); + } + + @PostMapping("/lastExecStatus") + public DataSetTaskDTO lastExecStatus(@RequestBody DataSetTaskDTO datasetTableTask) { + return dataSetTableTaskLogService.lastExecStatus(datasetTableTask); + } + + @PostMapping("/updateStatus") + public void updateStatus(@RequestBody DatasetTableTask datasetTableTask) { + dataSetTableTaskService.updateDatasetTableTaskStatus(datasetTableTask); + } + + @PostMapping("/execTask") + public void execTask(@RequestBody DatasetTableTask datasetTableTask) throws Exception{ + dataSetTableTaskService.execTask(datasetTableTask); + } + } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java index 3269b3eb8b..4b0123fe3c 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java @@ -5,6 +5,7 @@ import com.github.pagehelper.PageHelper; import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.commons.utils.PageUtils; import io.dataease.commons.utils.Pager; +import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.dto.dataset.DataSetTaskLogDTO; import io.dataease.service.dataset.DataSetTableTaskLogService; import org.springframework.web.bind.annotation.*; @@ -33,7 +34,7 @@ public class DataSetTableTaskLogController { } @PostMapping("list/{goPage}/{pageSize}") - public Pager> list(@RequestBody DatasetTableTaskLog request, @PathVariable int goPage, @PathVariable int pageSize) { + public Pager> list(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) { Page page = PageHelper.startPage(goPage, pageSize, true); return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request)); } diff --git a/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java b/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java index 2adf9f6453..2a48769ef6 100644 --- a/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java +++ b/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java @@ -19,4 +19,5 @@ public class ChartViewDTO extends ChartViewWithBLOBs { private Boolean isLeaf; private String pid; + private String sql; } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java new file mode 100644 index 0000000000..da22ebfbbe --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/DataSetTaskDTO.java @@ -0,0 +1,18 @@ +package io.dataease.dto.dataset; + +import io.dataease.base.domain.DatasetTableTask; +import io.dataease.base.domain.DatasetTableTaskLog; +import lombok.Getter; +import lombok.Setter; + +/** + * @Author gin + * @Date 2021/3/9 3:19 下午 + */ +@Getter +@Setter +public class DataSetTaskDTO extends DatasetTableTask { + private String datasetName; + private Long nextExecTime; + private String taskStatus; +} diff --git a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java index 14c8b9600b..c8597498a5 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java +++ b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java @@ -430,4 +430,10 @@ public class ScheduleManager { } return trigger.getFireTimeAfter(start); } + + public void fireNow(String jobName, String jobGroup) throws SchedulerException { + JobKey jobKey = new JobKey(jobName, jobGroup); + scheduler.triggerJob(jobKey); + } + } diff --git a/backend/src/main/java/io/dataease/listener/AppStartListener.java b/backend/src/main/java/io/dataease/listener/AppStartListener.java index 7acbdb007d..ffd313c23a 100644 --- a/backend/src/main/java/io/dataease/listener/AppStartListener.java +++ b/backend/src/main/java/io/dataease/listener/AppStartListener.java @@ -1,7 +1,9 @@ package io.dataease.listener; import io.dataease.base.domain.DatasetTableTask; +import io.dataease.commons.constants.NoticeConstants; import io.dataease.commons.constants.ScheduleType; +import io.dataease.commons.constants.TaskStatus; import io.dataease.service.ScheduleService; import io.dataease.service.dataset.DataSetTableTaskService; import org.apache.commons.lang3.StringUtils; @@ -40,8 +42,6 @@ public class AppStartListener implements ApplicationListener data = new ArrayList<>(); if (table.getMode() == 0) {// 直连 Datasource ds = datasourceService.get(table.getDataSourceId()); @@ -180,7 +181,6 @@ public class ChartViewService { throw new RuntimeException(Translator.get("i18n_datasource_delete")); } DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); - DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); @@ -201,20 +201,19 @@ public class ChartViewService { data = datasourceProvider.getData(datasourceRequest); /** * 直连不实用缓存 - String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery(); - Object cache; - if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) { - data = datasourceProvider.getData(datasourceRequest); - CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null); - }else { - data = (List) cache; - } + String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery(); + Object cache; + if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) { + data = datasourceProvider.getData(datasourceRequest); + CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null); + }else { + data = (List) cache; + } */ } else if (table.getMode() == 1) {// 抽取 // 连接doris，构建doris数据源查询 Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); - DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); String tableName = "ds_" + table.getId().replaceAll("-", "_"); datasourceRequest.setTable(tableName); @@ -243,12 +242,12 @@ public class ChartViewService { // 仪表板有参数不实用缓存 if (CollectionUtils.isNotEmpty(requestList.getFilter())) { data = datasourceProvider.getData(datasourceRequest); - }else { - try{ + } else { + try { data = cacheViewData(datasourceProvider, datasourceRequest, id); - }catch (Exception e) { + } catch (Exception e) { LogUtil.error(e); - }finally { + } finally { // 如果当前对象被锁 且 当前线程冲入次数 > 0 则释放锁 if (lock.isLocked() && lock.getHoldCount() > 0) { lock.unlock(); @@ -320,40 +319,42 @@ public class ChartViewService { ChartViewDTO dto = new ChartViewDTO(); BeanUtils.copyBean(dto, view); dto.setData(map); + dto.setSql(datasourceRequest.getQuery()); return dto; } /** * 避免缓存击穿 * 虽然流量不一定能够达到击穿的水平 + * * @param datasourceProvider * @param datasourceRequest * @param viewId * @return * @throws Exception */ - public List cacheViewData(DatasourceProvider datasourceProvider, DatasourceRequest datasourceRequest, String viewId) throws Exception{ - List result ; + public List cacheViewData(DatasourceProvider datasourceProvider, DatasourceRequest datasourceRequest, String viewId) throws Exception { + List result; Object cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, viewId); if (cache == null) { if (lock.tryLock()) {// 获取锁成功 - try{ + try { result = datasourceProvider.getData(datasourceRequest); if (result != null) { CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, viewId, result, null, null); } - }catch (Exception e) { + } catch (Exception e) { LogUtil.error(e); throw e; - }finally { + } finally { lock.unlock(); } - }else {//获取锁失败 + } else {//获取锁失败 Thread.sleep(100);//避免CAS自旋频率过大 占用cpu资源过高 result = cacheViewData(datasourceProvider, datasourceRequest, viewId); } - }else { - result = (List)cache; + } else { + result = (List) cache; } return result; } @@ -402,11 +403,11 @@ public class ChartViewService { public String chartCopy(String id) { String newChartId = UUID.randomUUID().toString(); - extChartViewMapper.chartCopy(newChartId,id); + extChartViewMapper.chartCopy(newChartId, id); return newChartId; } - public String searchAdviceSceneId(String panelId){ - return extChartViewMapper.searchAdviceSceneId(AuthUtils.getUser().getUserId().toString(),panelId); + public String searchAdviceSceneId(String panelId) { + return extChartViewMapper.searchAdviceSceneId(AuthUtils.getUser().getUserId().toString(), panelId); } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index fb69869a9d..36d023133e 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -260,7 +260,7 @@ public class DataSetTableService { List quota = new ArrayList<>(); fields.forEach(field -> { - if (field.getDeType() == 2 || field.getDeType() == 3) { + if (StringUtils.equalsIgnoreCase("q", field.getGroupType())) { quota.add(field); } else { dimension.add(field); @@ -277,6 +277,8 @@ public class DataSetTableService { .checked(true) .columnIndex(999) .deType(2) + .extField(1) + .groupType("q") .build(); quota.add(count); @@ -712,6 +714,8 @@ public class DataSetTableService { datasetTableField.setChecked(true); datasetTableField.setColumnIndex(i); datasetTableField.setLastSyncTime(syncTime); + datasetTableField.setExtField(0); + datasetTableField.setGroupType(datasetTableField.getDeType() < 2 ? "d" : "q"); dataSetTableFieldsService.save(datasetTableField); } } @@ -1192,12 +1196,17 @@ public class DataSetTableService { DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()).andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); + List taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream().map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList()); datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); + + DatasetTableTask datasetTableTask = new DatasetTableTask(); + datasetTableTask.setLastExecStatus(JobStatus.Error.name()); + dataSetTableTaskService.update(taskIds, datasetTableTask); + for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) { extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId()); extractDataService.deleteFile("incremental_delete", jobStoppeddDatasetTable.getId()); - } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index 286cae3cc5..c785cc3828 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -1,10 +1,16 @@ package io.dataease.service.dataset; +import io.dataease.base.domain.DatasetTableTask; import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.base.domain.DatasetTableTaskLogExample; import io.dataease.base.mapper.DatasetTableTaskLogMapper; +import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.ext.ExtDataSetTaskMapper; +import io.dataease.base.mapper.ext.query.GridExample; +import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.dto.dataset.DataSetTaskDTO; import io.dataease.dto.dataset.DataSetTaskLogDTO; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -38,8 +44,9 @@ public class DataSetTableTaskLogService { datasetTableTaskLogMapper.deleteByPrimaryKey(id); } - public List list(DatasetTableTaskLog request) { - return extDataSetTaskMapper.list(request); + public List list(BaseGridRequest request) { + GridExample gridExample = request.convertExample(); + return extDataSetTaskMapper.list(gridExample); } public void deleteByTaskId(String taskId){ @@ -57,6 +64,30 @@ public class DataSetTableTaskLogService { } public List select(DatasetTableTaskLog datasetTableTaskLog){ + DatasetTableTaskLogExample example = getDatasetTableTaskLogExample(datasetTableTaskLog); + example.setOrderByClause("create_time desc"); + return datasetTableTaskLogMapper.selectByExampleWithBLOBs(example); + } + + public DataSetTaskDTO lastExecStatus(DataSetTaskDTO dataSetTaskDTO){ + DatasetTableTaskLogExample example = new DatasetTableTaskLogExample(); + DatasetTableTaskLogExample.Criteria criteria = example.createCriteria(); + if(StringUtils.isNotEmpty(dataSetTaskDTO.getTableId())){ + criteria.andTableIdEqualTo(dataSetTaskDTO.getTableId()); + } + if(StringUtils.isNotEmpty(dataSetTaskDTO.getId())){ + criteria.andTaskIdEqualTo(dataSetTaskDTO.getId()); + } + example.setOrderByClause("create_time desc limit 1"); + List datasetTableTaskLogs = datasetTableTaskLogMapper.selectByExampleWithBLOBs(example); + if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){ + dataSetTaskDTO.setLastExecStatus(datasetTableTaskLogs.get(0).getStatus()); + dataSetTaskDTO.setLastExecTime(datasetTableTaskLogs.get(0).getCreateTime()); + } + return dataSetTaskDTO; + } + + private DatasetTableTaskLogExample getDatasetTableTaskLogExample(DatasetTableTaskLog datasetTableTaskLog) { DatasetTableTaskLogExample example = new DatasetTableTaskLogExample(); DatasetTableTaskLogExample.Criteria criteria = example.createCriteria(); if(StringUtils.isNotEmpty(datasetTableTaskLog.getStatus())){ @@ -68,7 +99,8 @@ public class DataSetTableTaskLogService { if(StringUtils.isNotEmpty(datasetTableTaskLog.getTaskId())){ criteria.andTaskIdEqualTo(datasetTableTaskLog.getTaskId()); } - example.setOrderByClause("create_time desc"); - return datasetTableTaskLogMapper.selectByExampleWithBLOBs(example); + return example; } + + } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index 55097ece70..391a92ac69 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -1,16 +1,26 @@ package io.dataease.service.dataset; +import com.google.gson.Gson; import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.DatasetTableTask; import io.dataease.base.domain.DatasetTableTaskExample; import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.base.mapper.DatasetTableTaskMapper; +import io.dataease.base.mapper.ext.ExtDataSetTaskMapper; +import io.dataease.base.mapper.ext.query.GridExample; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.constants.ScheduleType; +import io.dataease.commons.constants.TaskStatus; +import io.dataease.commons.constants.TriggerType; import io.dataease.controller.request.dataset.DataSetTaskRequest; +import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.controller.sys.response.SysUserGridResponse; +import io.dataease.controller.sys.response.SysUserRole; +import io.dataease.dto.dataset.DataSetTaskDTO; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; import io.dataease.service.ScheduleService; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.quartz.CronExpression; @@ -21,6 +31,7 @@ import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; /** * @Author gin @@ -40,12 +51,15 @@ public class DataSetTableTaskService { private DataSetTableService dataSetTableService; @Resource private ExtractDataService extractDataService; + @Resource + private ExtDataSetTaskMapper extDataSetTaskMapper; public DatasetTableTask save(DataSetTaskRequest dataSetTaskRequest) throws Exception { checkName(dataSetTaskRequest); DatasetTableTask datasetTableTask = dataSetTaskRequest.getDatasetTableTask(); - dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig()); - + if(!datasetTableTask.getType().equalsIgnoreCase("add_scope")){ + dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig()); + } // check if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(),"CRON")){ if (StringUtils.isNotEmpty(datasetTableTask.getCron())) { @@ -67,34 +81,48 @@ public class DataSetTableTaskService { if (StringUtils.isEmpty(datasetTableTask.getId())) { datasetTableTask.setId(UUID.randomUUID().toString()); datasetTableTask.setCreateTime(System.currentTimeMillis()); - // SIMPLE 类型，提前占位 - if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - if (datasetTableTask.getType().equalsIgnoreCase("add_scope")) { - DatasetTable datasetTable = dataSetTableService.get(datasetTableTask.getTableId()); - if (datasetTable.getLastUpdateTime() == 0 || datasetTable.getLastUpdateTime() == null) { - DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync")); - } - } - if (extractDataService.updateSyncStatusIsNone(dataSetTableService.get(datasetTableTask.getTableId()))) { - DataEaseException.throwException(Translator.get("i18n_sync_job_exists")); - } else { - //write log - DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); - datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); - datasetTableTaskLog.setTaskId(datasetTableTask.getId()); - datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - datasetTableTaskLog.setStartTime(System.currentTimeMillis()); - dataSetTableTaskLogService.save(datasetTableTaskLog); - } - } + datasetTableTask.setStatus(TaskStatus.Underway.name()); datasetTableTaskMapper.insert(datasetTableTask); } else { datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask); } - scheduleService.addSchedule(datasetTableTask); + if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString()) && datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Underway.name())) { // SIMPLE 类型，提前占位 + execNow(datasetTableTask); + datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); + datasetTableTask.setLastExecTime(System.currentTimeMillis()); + update(datasetTableTask); + } + if(!datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){ + scheduleService.addSchedule(datasetTableTask); + }else { + if(datasetTableTask.getStatus().equalsIgnoreCase(JobStatus.Underway.name())){ + scheduleService.addSchedule(datasetTableTask); + } + } + return datasetTableTask; } + private void execNow(DatasetTableTask datasetTableTask) { + if (datasetTableTask.getType().equalsIgnoreCase("add_scope")) { + DatasetTable datasetTable = dataSetTableService.get(datasetTableTask.getTableId()); + if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) { + DataEaseException.throwException(Translator.get("i18n_not_exec_add_sync")); + } + } + if (extractDataService.existSyncTask(dataSetTableService.get(datasetTableTask.getTableId()), null)) { + DataEaseException.throwException(Translator.get("i18n_sync_job_exists")); + } + //write log + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); + datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + datasetTableTaskLog.setStartTime(System.currentTimeMillis()); + datasetTableTaskLog.setTriggerType(TriggerType.Custom.name()); + dataSetTableTaskLogService.save(datasetTableTaskLog); + } + public void delete(String id) { DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(id); datasetTableTaskMapper.deleteByPrimaryKey(id); @@ -121,19 +149,34 @@ public class DataSetTableTaskService { } public void update(DatasetTableTask datasetTableTask) { - datasetTableTaskMapper.updateByPrimaryKey(datasetTableTask); + datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask); + } + + public void update(List taskIds, DatasetTableTask datasetTableTask) { + if (CollectionUtils.isEmpty(taskIds)){ + return; + } + DatasetTableTaskExample example = new DatasetTableTaskExample(); + example.createCriteria().andIdIn(taskIds); + datasetTableTaskMapper.updateByExampleSelective(datasetTableTask, example); } public List list(DatasetTableTask datasetTableTask) { DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria(); - if (StringUtils.isNotEmpty(datasetTableTask.getTableId())) { + if (datasetTableTask != null && StringUtils.isNotEmpty(datasetTableTask.getTableId())) { criteria.andTableIdEqualTo(datasetTableTask.getTableId()); } datasetTableTaskExample.setOrderByClause("create_time desc,name asc"); return datasetTableTaskMapper.selectByExample(datasetTableTaskExample); } + public List taskList(BaseGridRequest request) { + GridExample gridExample = request.convertExample(); + List dataSetTaskDTOS = extDataSetTaskMapper.taskList(gridExample); + return dataSetTaskDTOS; + } + private void checkName(DataSetTaskRequest dataSetTaskRequest) { DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria(); @@ -151,4 +194,29 @@ public class DataSetTableTaskService { throw new RuntimeException(Translator.get("i18n_task_name_repeat")); } } + + public void updateDatasetTableTaskStatus(DatasetTableTask datasetTableTask){ + DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample(); + DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria(); + criteria.andIdEqualTo(datasetTableTask.getId()); + DatasetTableTask record = new DatasetTableTask(); + record.setStatus(datasetTableTask.getStatus()); + datasetTableTaskMapper.updateByExampleSelective(record, datasetTableTaskExample); + } + + public void execTask(DatasetTableTask datasetTableTask) throws Exception{ + execNow(datasetTableTask); +// datasetTableTask.setStatus(TaskStatus.Underway.name()); + datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); + datasetTableTask.setLastExecTime(System.currentTimeMillis()); + update(datasetTableTask); + + if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())){ + scheduleService.fireNow(datasetTableTask); + } + if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){ + scheduleService.addSchedule(datasetTableTask); + } + + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 9e262a7022..008f2309bd 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -6,10 +6,7 @@ import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.DatasourceMapper; import io.dataease.base.mapper.ext.ExtChartViewMapper; -import io.dataease.commons.constants.JdbcConstants; -import io.dataease.commons.constants.JobStatus; -import io.dataease.commons.constants.ScheduleType; -import io.dataease.commons.constants.UpdateType; +import io.dataease.commons.constants.*; import io.dataease.commons.model.AuthURD; import io.dataease.commons.utils.*; import io.dataease.datasource.constants.DatasourceTypes; @@ -134,28 +131,48 @@ public class ExtractDataService { "fi\n" + "rm -rf %s\n"; - public synchronized boolean updateSyncStatusIsNone(DatasetTable datasetTable ){ + public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask){ datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); - example.createCriteria().andIdEqualTo(datasetTable.getId()); - datasetTableMapper.selectByExample(example); - example.clear(); example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name()); example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull()); - return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + if(existSyncTask){ + if(datasetTableTask != null){ + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setTableId(datasetTable.getId()); + datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); + if(CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())){ + return false; + } + } + return true; + }else { + return false; + } } public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = getDatasetTable(datasetTableId); if(datasetTable == null){ LogUtil.error("Can not find DatasetTable: " + datasetTableId); - } - DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); - boolean isCronJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())); - if(updateSyncStatusIsNone(datasetTable) && isCronJob){ - LogUtil.info("Skip synchronization task for table : " + datasetTableId); return; } + DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); + if(datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name()) && !datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())){ + LogUtil.info("Skip synchronization task, task ID : " + datasetTableTask.getId()); + return; + } + if(existSyncTask(datasetTable, datasetTableTask)){ + LogUtil.info("Skip synchronization task for dataset, dataset ID : " + datasetTableId); + return; + } + datasetTableTask.setLastExecTime(System.currentTimeMillis()); + datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); + dataSetTableTaskService.update(datasetTableTask); + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); UpdateType updateType = UpdateType.valueOf(type); if(context != null){ @@ -207,20 +224,29 @@ public class ExtractDataService { sendWebMsg(datasetTable, taskId,true); -// deleteFile("all_scope", datasetTableId); + deleteFile("all_scope", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); + +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } + datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); + dataSetTableTaskService.update(datasetTableTask); + }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); + datasetTableTask.setLastExecStatus(JobStatus.Error.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } + dataSetTableTaskService.update(datasetTableTask); + sendWebMsg(datasetTable, taskId,false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); -// deleteFile("all_scope", datasetTableId); + deleteFile("all_scope", datasetTableId); }finally { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); - dataSetTableTaskService.update(datasetTableTask); - } } break; @@ -274,22 +300,28 @@ public class ExtractDataService { sendWebMsg(datasetTable, taskId,true); -// deleteFile("incremental_add", datasetTableId); -// deleteFile("incremental_delete", datasetTableId); + deleteFile("incremental_add", datasetTableId); + deleteFile("incremental_delete", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); + datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } + dataSetTableTaskService.update(datasetTableTask); } }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); sendWebMsg(datasetTable, taskId,false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); -// deleteFile("incremental_add", datasetTableId); -// deleteFile("incremental_delete", datasetTableId); + datasetTableTask.setLastExecStatus(JobStatus.Error.name()); +// if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { +// datasetTableTask.setStatus(TaskStatus.Stopped.name()); +// } + dataSetTableTaskService.update(datasetTableTask); + deleteFile("incremental_add", datasetTableId); + deleteFile("incremental_delete", datasetTableId); }finally { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); - dataSetTableTaskService.update(datasetTableTask); - } } break; } @@ -435,6 +467,7 @@ public class ExtractDataService { datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + datasetTableTaskLog.setTriggerType(TriggerType.Cron.name()); List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); if(CollectionUtils.isEmpty(datasetTableTaskLogs)){ datasetTableTaskLog.setStartTime(System.currentTimeMillis()); diff --git a/backend/src/main/resources/db/migration/V12__alter_updata_ds_table_field.sql b/backend/src/main/resources/db/migration/V12__alter_updata_ds_table_field.sql new file mode 100644 index 0000000000..4cb02e5c54 --- /dev/null +++ b/backend/src/main/resources/db/migration/V12__alter_updata_ds_table_field.sql @@ -0,0 +1,7 @@ +ALTER TABLE `dataset_table_field` ADD COLUMN `group_type` VARCHAR(50) COMMENT '维度/指标标识 d:维度，q:指标' AFTER `dataease_name`; +ALTER TABLE `dataset_table_field` ADD COLUMN `de_type_format` int(10) COMMENT '类型格式' AFTER `de_type`; +ALTER TABLE `dataset_table_field` ADD COLUMN `ext_field` int(10) COMMENT '是否扩展字段 0否 1是' AFTER `de_extract_type`; + +UPDATE `dataset_table_field` SET group_type='d' where `de_type` IN (0,1); +UPDATE `dataset_table_field` SET group_type='q' where `de_type` IN (2,3); +UPDATE `dataset_table_field` SET ext_field=0; \ No newline at end of file diff --git a/backend/src/main/resources/db/migration/V13__system_task.sql b/backend/src/main/resources/db/migration/V13__system_task.sql new file mode 100644 index 0000000000..efbdc47965 --- /dev/null +++ b/backend/src/main/resources/db/migration/V13__system_task.sql @@ -0,0 +1,14 @@ +INSERT INTO `sys_menu` VALUES (57, 1, 3, 1, '任务管理', 'sys-task', 'system/task/index', 2000, 'task', 'system-task', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL); +INSERT INTO `sys_menu` VALUES (58, 57, 0, 1, '数据集任务', 'sys-task-dataset', 'system/task/dataset', 1, 'task', 'dataset', b'0', b'0', b'0', NULL, NULL, NULL, NULL, NULL); +ALTER TABLE `dataset_table_task` + ADD COLUMN `last_exec_time` BIGINT(13) NULL DEFAULT NULL COMMENT '上次执行时间' AFTER `create_time`, + ADD COLUMN `status` VARCHAR(50) NULL DEFAULT NULL COMMENT '任务状态' AFTER `last_exec_time`, + ADD COLUMN `last_exec_status` VARCHAR(50) NULL DEFAULT NULL COMMENT '上次执行结果' AFTER `last_exec_time`; + +update dataset_table_task set status='Underway'; +update dataset_table_task set rate='SIMPLE' where rate='SIMPLE_COMPLETE'; + + +ALTER TABLE `dataset_table_task_log` ADD COLUMN `trigger_type` VARCHAR(45) NULL AFTER `create_time`; +update dataset_table_task_log set trigger_type='Cron'; + diff --git a/backend/src/main/resources/generatorConfig.xml b/backend/src/main/resources/generatorConfig.xml index 74f351b5ce..3a6f7816c8 100644 --- a/backend/src/main/resources/generatorConfig.xml +++ b/backend/src/main/resources/generatorConfig.xml @@ -74,7 +74,7 @@ - +
diff --git a/frontend/src/api/dataset/dataset.js b/frontend/src/api/dataset/dataset.js index 7045b0b9da..f5100ed445 100644 --- a/frontend/src/api/dataset/dataset.js +++ b/frontend/src/api/dataset/dataset.js @@ -128,4 +128,21 @@ export function isKettleRunning(showLoading = true) { }) } +export function taskList(spage, size, data) { + return request({ + url: '/dataset/group/isKettleRunning', + method: 'post', + loading: showLoading + }) +} + +export function datasetTaskList(page, size, data) { + return request({ + url: '/dataset/task/pageList/' + page + '/' + size, + method: 'post', + data, + loading: true + }) +} + export default { loadTable, getScene, addGroup, delGroup, addTable, delTable, groupTree } diff --git a/frontend/src/components/canvas/components/Editor/ContextMenu.vue b/frontend/src/components/canvas/components/Editor/ContextMenu.vue index 2806cec0df..e3fddce02d 100644 --- a/frontend/src/components/canvas/components/Editor/ContextMenu.vue +++ b/frontend/src/components/canvas/components/Editor/ContextMenu.vue @@ -94,6 +94,7 @@ export default { this.deleteCurCondition() this.$store.commit('deleteComponent') this.$store.commit('recordSnapshot') + this.$store.commit('setCurComponent', { component: null, index: null }) }, deleteCurCondition() { diff --git a/frontend/src/components/canvas/components/Editor/index.vue b/frontend/src/components/canvas/components/Editor/index.vue index 6119004f42..6a3909799e 100644 --- a/frontend/src/components/canvas/components/Editor/index.vue +++ b/frontend/src/components/canvas/components/Editor/index.vue @@ -28,7 +28,7 @@ class-name-active="de-drag-active" :class="{'gap_class':canvasStyleData.panel.gap==='yes'}" :snap="true" - :snap-tolerance="1" + :snap-tolerance="2" :change-style="customStyle" @refLineParams="getRefLineParams" > @@ -84,6 +84,12 @@ height: item.lineLength, }" /> + @@ -521,7 +527,7 @@ export default { } }, getRefLineParams(params) { - // console.log(params) + // console.log(JSON.stringify(params)) const { vLine, hLine } = params this.vLine = vLine this.hLine = hLine diff --git a/frontend/src/icons/svg/field_location.svg b/frontend/src/icons/svg/field_location.svg new file mode 100644 index 0000000000..7def81f700 --- /dev/null +++ b/frontend/src/icons/svg/field_location.svg @@ -0,0 +1 @@ + diff --git a/frontend/src/icons/svg/task.svg b/frontend/src/icons/svg/task.svg new file mode 100644 index 0000000000..e5847ca8fc --- /dev/null +++ b/frontend/src/icons/svg/task.svg @@ -0,0 +1 @@ + diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 6ddc5c15da..555a33eba1 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -129,6 +129,7 @@ export default { password_error: 'The password can not be less than 8 digits' }, commons: { + success: 'Success', switch_lang: 'Switch Language Success', close: 'Close', icon: 'Icon', @@ -920,7 +921,26 @@ export default { char_can_not_more_50: 'Dataset name can not more 50', task_add_title: 'Add Task', task_edit_title: 'Edit Task', - sync_latter: 'Sync latter' + sync_latter: 'Sync latter', + task: { + list: 'Task list', + record: 'Execution record', + create: 'New task', + name: 'Task name', + last_exec_time: 'Last execution time', + next_exec_time: 'Next execution time', + last_exec_status: 'Last execution result', + task_status: 'Task status', + dataset: 'Data set', + search_by_name: 'Search by name', + underway: 'Running', + stopped: 'Stopped', + exec: 'Execute', + confirm_exec: 'Manual trigger execution？', + change_success: 'State switch successful' + }, + field_group_type: 'Type', + location: 'Location' }, datasource: { datasource: 'Data Source', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 7099ffd7b0..081d10fbd6 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -129,6 +129,7 @@ export default { password_error: '密碼不小於 8 位' }, commons: { + success: '成功', switch_lang: '切換語言成功', close: '關閉', icon: '圖標', @@ -920,7 +921,26 @@ export default { char_can_not_more_50: '數據集名稱不能超過50個字符', task_add_title: '添加任務', task_edit_title: '編輯任務', - sync_latter: '稍後同步' + sync_latter: '稍後同步', + task: { + list: '任務列表', + record: '執行紀錄', + create: '新建任務', + name: '任務名稱', + last_exec_time: '上次執行時間', + next_exec_time: '下次執行時間', + last_exec_status: '上次執行結果', + task_status: '任務狀態', + dataset: '數據集', + search_by_name: '根據名稱搜索', + underway: '運行中', + stopped: '停止', + exec: '執行', + confirm_exec: '手動觸發執行？', + change_success: '狀態切換成功' + }, + field_group_type: '分類', + location: '地理位置' }, datasource: { datasource: '數據源', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index c5b3cff9ad..c53c83bd07 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -129,6 +129,7 @@ export default { password_error: '密码不小于 8 位' }, commons: { + success: '成功', switch_lang: '切换语言成功', close: '关闭', icon: '图标', @@ -920,7 +921,26 @@ export default { char_can_not_more_50: '数据集名称不能超过50个字符', task_add_title: '添加任务', task_edit_title: '编辑任务', - sync_latter: '稍后同步' + sync_latter: '稍后同步', + task: { + list: '任务列表', + record: '执行记录', + create: '新建任务', + name: '任务名称', + last_exec_time: '上次执行时间', + next_exec_time: '下次执行时间', + last_exec_status: '上次执行结果', + task_status: '任务状态', + dataset: '数据集', + search_by_name: '根据名称搜索', + underway: '运行中', + stopped: '停止', + exec: '执行', + confirm_exec: '手动触发执行？', + change_success: '状态切换成功' + }, + field_group_type: '分类', + location: '地理位置' }, datasource: { datasource: '数据源', diff --git a/frontend/src/styles/index.scss b/frontend/src/styles/index.scss index 93ee1c8c13..a93b6726da 100644 --- a/frontend/src/styles/index.scss +++ b/frontend/src/styles/index.scss @@ -195,7 +195,7 @@ div:focus { .el-table__row { :hover { cursor: pointer; - } + } } td { padding: 3px 0 !important; @@ -235,6 +235,10 @@ div:focus { @extend %field-icon; color: #37b4aa; } +.field-icon-location{ + @extend %field-icon; + color: #F56C6C; +} .ds-icon-scene{ width: 13px; height: 13px; diff --git a/frontend/src/utils/webMsg.js b/frontend/src/utils/webMsg.js index 0f57d12276..b251540641 100644 --- a/frontend/src/utils/webMsg.js +++ b/frontend/src/utils/webMsg.js @@ -1,7 +1,4 @@ export const msgTypes = [ - // { value: -1, label: '全部类型' }, - // { value: 0, label: '仪表板分享' }, - // { value: 1, label: '数据集同步' } { value: -1, label: 'webmsg.all_type' }, { value: 0, label: 'webmsg.panel_type' }, { value: 1, label: 'webmsg.dataset_type' } diff --git a/frontend/src/views/chart/components/drag-item/DimensionItem.vue b/frontend/src/views/chart/components/drag-item/DimensionItem.vue index a8f032f17e..b160a32e56 100644 --- a/frontend/src/views/chart/components/drag-item/DimensionItem.vue +++ b/frontend/src/views/chart/components/drag-item/DimensionItem.vue @@ -1,12 +1,23 @@ - + + + + + + diff --git a/frontend/src/views/dataset/data/UnionView.vue b/frontend/src/views/dataset/data/UnionView.vue index c31b8ac35e..f238094cc8 100644 --- a/frontend/src/views/dataset/data/UnionView.vue +++ b/frontend/src/views/dataset/data/UnionView.vue @@ -69,6 +69,9 @@ + + + {{ item.name }} @@ -115,6 +118,9 @@ + + + {{ item.name }} diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index 5c11a97687..239a4f0c69 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -472,7 +472,6 @@ export default { post('/dataset/table/incrementalConfig', { tableId: this.table.id }).then(response => { this.incrementalConfig = response.data this.incrementalUpdateType = 'incrementalAdd' - // console.log(this.sql) if (this.incrementalConfig.incrementalAdd) { this.sql = this.incrementalConfig.incrementalAdd } else { @@ -555,7 +554,8 @@ export default { } }, listTaskLog(loading = true) { - post('/dataset/taskLog/list/' + this.page.currentPage + '/' + this.page.pageSize, { tableId: this.table.id }, loading).then(response => { + const params = {"conditions":[{"field":"dataset_table_task.table_id","operator":"eq","value": this.table.id}],"orders":[]} + post('/dataset/taskLog/list/' + this.page.currentPage + '/' + this.page.pageSize, params, loading).then(response => { this.taskLogData = response.data.listObject this.page.total = response.data.itemCount }) diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index d2f97bb8ee..123fc85924 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -101,8 +101,7 @@ export default { 'configuration.username': [{ required: true, message: this.$t('datasource.please_input_user_name'), trigger: 'blur' }], 'configuration.password': [{ required: true, message: this.$t('datasource.please_input_password'), trigger: 'change' }], 'configuration.host': [{ required: true, message: this.$t('datasource.please_input_host'), trigger: 'change' }], - 'configuration.port': [{ required: true, message: this.$t('datasource.please_input_port'), trigger: 'change' }], - 'configuration.schema': [{ required: true, message: this.$t('datasource.please_choose_schema'), trigger: 'change' }] + 'configuration.port': [{ required: true, message: this.$t('datasource.please_input_port'), trigger: 'change' }] }, allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' }, { name: 'oracle', label: 'Oracle', type: 'jdbc' }], schemas: [], @@ -148,7 +147,7 @@ export default { this.$refs.dsForm.resetFields() }, save() { - if (!this.form.configuration.schema) { + if (!this.form.configuration.schema && this.form.type === 'oracle') { this.$message.error(this.$t('datasource.please_choose_schema')) return } @@ -174,7 +173,7 @@ export default { data.configuration = JSON.stringify(data.configuration) getSchema(data).then(res => { this.schemas = res.data - this.$success(this.$t('datasource.validate_success')) + this.$success(this.$t('commons.success')) }) } else { return false @@ -182,6 +181,10 @@ export default { }) }, validaDatasource() { + if (!this.form.configuration.schema && this.form.type === 'oracle') { + this.$message.error(this.$t('datasource.please_choose_schema')) + return + } this.$refs.dsForm.validate(valid => { if (valid) { const data = JSON.parse(JSON.stringify(this.form)) diff --git a/frontend/src/views/system/task/DatasetTaskList.vue b/frontend/src/views/system/task/DatasetTaskList.vue new file mode 100644 index 0000000000..c938a3119a --- /dev/null +++ b/frontend/src/views/system/task/DatasetTaskList.vue @@ -0,0 +1,558 @@ + + + + + diff --git a/frontend/src/views/system/task/TaskRecord.vue b/frontend/src/views/system/task/TaskRecord.vue new file mode 100644 index 0000000000..a9f3056c4f --- /dev/null +++ b/frontend/src/views/system/task/TaskRecord.vue @@ -0,0 +1,181 @@ + + + + + diff --git a/frontend/src/views/system/task/dataset.vue b/frontend/src/views/system/task/dataset.vue new file mode 100644 index 0000000000..fcdacfa1d9 --- /dev/null +++ b/frontend/src/views/system/task/dataset.vue @@ -0,0 +1,65 @@ + + + + + diff --git a/frontend/src/views/system/task/index.vue b/frontend/src/views/system/task/index.vue new file mode 100644 index 0000000000..0d8026c857 --- /dev/null +++ b/frontend/src/views/system/task/index.vue @@ -0,0 +1,21 @@ + + + + diff --git a/frontend/src/views/system/user/index.vue b/frontend/src/views/system/user/index.vue index 1701abc348..43bff39217 100644 --- a/frontend/src/views/system/user/index.vue +++ b/frontend/src/views/system/user/index.vue @@ -12,7 +12,6 @@ >