From 92549c13d43783b6c5bbd367ed2cde3402e22dfd Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 28 May 2021 12:19:14 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E6=A3=80=E6=B5=8B=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/dataease/base/domain/DatasetTable.java | 2 + .../base/domain/DatasetTableExample.java | 70 +++++++++++++++++++ .../base/mapper/DatasetTableMapper.xml | 28 ++++++-- .../datasource/service/DatasourceService.java | 2 +- .../dataease/job/sechedule/DeScheduleJob.java | 1 + .../job/sechedule/ExtractDataJob.java | 2 +- .../service/dataset/DataSetTableService.java | 59 ++++++++++++---- .../service/dataset/ExtractDataService.java | 7 +- .../db/migration/V6__alter_table.sql | 3 +- 9 files changed, 150 insertions(+), 24 deletions(-) diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTable.java b/backend/src/main/java/io/dataease/base/domain/DatasetTable.java index 9a0590bba7..e88c0febff 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTable.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTable.java @@ -21,6 +21,8 @@ public class DatasetTable implements Serializable { private Long createTime; + private String qrtzInstance; + private String syncStatus; private String info; diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java index b229db8021..b0dad0931d 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java @@ -644,6 +644,76 @@ public class DatasetTableExample { return (Criteria) this; } + public Criteria andQrtzInstanceIsNull() { + addCriterion("qrtz_instance is null"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceIsNotNull() { + addCriterion("qrtz_instance is not null"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceEqualTo(String value) { + addCriterion("qrtz_instance =", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotEqualTo(String value) { + addCriterion("qrtz_instance <>", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceGreaterThan(String value) { + addCriterion("qrtz_instance >", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceGreaterThanOrEqualTo(String value) { + addCriterion("qrtz_instance >=", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLessThan(String value) { + addCriterion("qrtz_instance <", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLessThanOrEqualTo(String value) { + addCriterion("qrtz_instance <=", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLike(String value) { + addCriterion("qrtz_instance like", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotLike(String value) { + addCriterion("qrtz_instance not like", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceIn(List values) { + addCriterion("qrtz_instance in", values, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotIn(List values) { + addCriterion("qrtz_instance not in", values, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceBetween(String value1, String value2) { + addCriterion("qrtz_instance between", value1, value2, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotBetween(String value1, String value2) { + addCriterion("qrtz_instance not between", value1, value2, "qrtzInstance"); + return (Criteria) this; + } + public Criteria andSyncStatusIsNull() { addCriterion("sync_status is null"); return (Criteria) this; diff --git a/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml index f465f58ec3..35c7c22c34 100644 --- a/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml @@ -10,6 +10,7 @@ + @@ -74,7 +75,8 @@ - id, `name`, scene_id, data_source_id, `type`, `mode`, create_by, create_time, sync_status + id, `name`, scene_id, data_source_id, `type`, `mode`, create_by, create_time, qrtz_instance, + sync_status info @@ -130,12 +132,12 @@ insert into dataset_table (id, `name`, scene_id, data_source_id, `type`, `mode`, - create_by, create_time, sync_status, - info) + create_by, create_time, qrtz_instance, + sync_status, info) values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{sceneId,jdbcType=VARCHAR}, #{dataSourceId,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{mode,jdbcType=INTEGER}, - #{createBy,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{syncStatus,jdbcType=VARCHAR}, - #{info,jdbcType=LONGVARCHAR}) + #{createBy,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{qrtzInstance,jdbcType=VARCHAR}, + #{syncStatus,jdbcType=VARCHAR}, #{info,jdbcType=LONGVARCHAR}) insert into dataset_table @@ -164,6 +166,9 @@ create_time, + + qrtz_instance, + sync_status, @@ -196,6 +201,9 @@ #{createTime,jdbcType=BIGINT}, + + #{qrtzInstance,jdbcType=VARCHAR}, + #{syncStatus,jdbcType=VARCHAR}, @@ -237,6 +245,9 @@ create_time = #{record.createTime,jdbcType=BIGINT}, + + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, + sync_status = #{record.syncStatus,jdbcType=VARCHAR}, @@ -258,6 +269,7 @@ `mode` = #{record.mode,jdbcType=INTEGER}, create_by = #{record.createBy,jdbcType=VARCHAR}, create_time = #{record.createTime,jdbcType=BIGINT}, + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, sync_status = #{record.syncStatus,jdbcType=VARCHAR}, info = #{record.info,jdbcType=LONGVARCHAR} @@ -274,6 +286,7 @@ `mode` = #{record.mode,jdbcType=INTEGER}, create_by = #{record.createBy,jdbcType=VARCHAR}, create_time = #{record.createTime,jdbcType=BIGINT}, + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, sync_status = #{record.syncStatus,jdbcType=VARCHAR} @@ -303,6 +316,9 @@ create_time = #{createTime,jdbcType=BIGINT}, + + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, + sync_status = #{syncStatus,jdbcType=VARCHAR}, @@ -321,6 +337,7 @@ `mode` = #{mode,jdbcType=INTEGER}, create_by = #{createBy,jdbcType=VARCHAR}, create_time = #{createTime,jdbcType=BIGINT}, + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, sync_status = #{syncStatus,jdbcType=VARCHAR}, info = #{info,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=VARCHAR} @@ -334,6 +351,7 @@ `mode` = #{mode,jdbcType=INTEGER}, create_by = #{createBy,jdbcType=VARCHAR}, create_time = #{createTime,jdbcType=BIGINT}, + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, sync_status = #{syncStatus,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR} diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index 721f1a6ff4..d151efc6fd 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -145,7 +145,7 @@ public class DatasourceService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); datasourceProvider.initDataSource(datasourceRequest); - LogUtil.error("Succsss to init datasource connection pool: " + datasource.getName()); + LogUtil.info("Succsss to init datasource connection pool: " + datasource.getName()); }catch (Exception e){ LogUtil.error("Failed to init datasource connection pool: " + datasource.getName(), e); } diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java index 3fa52a44d3..ecbf81d1e6 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java @@ -1,5 +1,6 @@ package io.dataease.job.sechedule; +import com.google.gson.Gson; import io.dataease.commons.utils.LogUtil; import org.quartz.*; diff --git a/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java index d6411e3b96..b8f03d5a75 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java @@ -15,7 +15,7 @@ public class ExtractDataJob extends DeScheduleJob{ @Override void businessExecute(JobExecutionContext context) { - extractDataService.extractData(datasetTableId, taskId, updateType); + extractDataService.extractData(datasetTableId, taskId, updateType, context); } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index e2c3910b4d..2e950b79cb 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1,11 +1,10 @@ package io.dataease.service.dataset; +import com.fit2cloud.quartz.anno.QuartzScheduled; import com.google.gson.Gson; import io.dataease.base.domain.*; -import io.dataease.base.mapper.DatasetTableIncrementalConfigMapper; -import io.dataease.base.mapper.DatasetTableMapper; -import io.dataease.base.mapper.DatasourceMapper; +import io.dataease.base.mapper.*; import io.dataease.base.mapper.ext.ExtDataSetTableMapper; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.utils.*; @@ -75,6 +74,10 @@ public class DataSetTableService { private DataSetTableUnionService dataSetTableUnionService; @Resource private DataSetTableTaskLogService dataSetTableTaskLogService; + @Resource + private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; + @Resource + private DatasetTableTaskLogMapper datasetTableTaskLogMapper; @Value("${upload.file.path}") private String path; @@ -106,7 +109,7 @@ public class DataSetTableService { saveTableField(datasetTable); if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { commonThreadPool.addTask(() -> { - extractDataService.extractData(datasetTable.getId(), null, "all_scope"); + extractDataService.extractData(datasetTable.getId(), null, "all_scope", null); }); } } @@ -867,22 +870,48 @@ public class DataSetTableService { } public Boolean checkDorisTableIsExists(String id) throws Exception { -// Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); -// JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); -// DatasourceRequest datasourceRequest = new DatasourceRequest(); -// datasourceRequest.setDatasource(dorisDatasource); -// QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); -// datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id))); -// List data = jdbcProvider.getData(datasourceRequest); -// return CollectionUtils.isNotEmpty(data); - return true; + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(dorisDatasource); + QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); + datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id))); + List data = jdbcProvider.getData(datasourceRequest); + return CollectionUtils.isNotEmpty(data); } + @QuartzScheduled(cron = "0 0/3 * * * ?") public void updateDatasetTableStatus(){ - DatasetTable record = new DatasetTable(); - record.setSyncStatus(JobStatus.Completed.name()); + List qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); + List activeQrtzInstances = qrtzSchedulerStates.stream().filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > System.currentTimeMillis()).map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); + List jobStoppeddDatasetTables = new ArrayList<>(); + DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); + + datasetTableMapper.selectByExample(example).forEach(datasetTable -> { + if(StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))){ + jobStoppeddDatasetTables.add(datasetTable); + } + }); + + if(CollectionUtils.isEmpty(jobStoppeddDatasetTables)){ + return; + } + + DatasetTable record = new DatasetTable(); + record.setSyncStatus(JobStatus.Completed.name()); + example.clear(); + example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()).andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); datasetTableMapper.updateByExampleSelective(record, example); + + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setStatus(JobStatus.Error.name()); + datasetTableTaskLog.setInfo("Job stopped due to system error."); + + DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); + datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()).andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); + datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index e953b80976..a0d3510bc3 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -63,6 +63,7 @@ import org.pentaho.di.trans.steps.textfileoutput.TextFileOutputMeta; import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef; import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta; import org.pentaho.di.www.SlaveServerJobStatus; +import org.quartz.JobExecutionContext; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -183,7 +184,7 @@ public class ExtractDataService { return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; } - public void extractData(String datasetTableId, String taskId, String type) { + public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = dataSetTableService.get(datasetTableId); if(updateSyncStatus(datasetTable)){ LogUtil.info("Skip synchronization task for table : " + datasetTableId); @@ -193,6 +194,10 @@ public class ExtractDataService { UpdateType updateType = UpdateType.valueOf(type); Datasource datasource = new Datasource(); try { + if(context != null){ + datasetTable.setQrtzInstance(context.getFireInstanceId()); + datasetTableMapper.updateByPrimaryKeySelective(datasetTable); + } if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); } else { diff --git a/backend/src/main/resources/db/migration/V6__alter_table.sql b/backend/src/main/resources/db/migration/V6__alter_table.sql index 9551f19b98..37106309dd 100644 --- a/backend/src/main/resources/db/migration/V6__alter_table.sql +++ b/backend/src/main/resources/db/migration/V6__alter_table.sql @@ -1 +1,2 @@ -ALTER TABLE `dataease`.`dataset_table` ADD COLUMN `sync_status` VARCHAR(45) NULL AFTER `create_time`; +ALTER TABLE `dataset_table` ADD COLUMN `sync_status` VARCHAR(45) NULL AFTER `create_time`; +ALTER TABLE `dataset_table` ADD COLUMN `qrtz_instance` VARCHAR(1024) NULL AFTER `create_time`; From e955d2bfff32630bf452b4de5b39e89df8f260dd Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 12:40:19 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(frontend):user=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=A7=92=E8=89=B2=E5=AD=97=E6=AE=B5=EF=BC=9B?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E6=B7=BB=E5=8A=A0=E6=8A=A5=E9=94=99?= =?UTF-8?q?fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/service/DatasourceService.java | 3 ++- .../resources/i18n/messages_en_US.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + frontend/src/views/system/datasource/form.vue | 6 ++++-- frontend/src/views/system/user/index.vue | 18 ++++++++++++++++++ 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index d151efc6fd..099862fcbe 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -18,6 +18,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.DatasourceDTO; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.i18n.Translator; import io.dataease.service.dataset.DataSetGroupService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -49,7 +50,7 @@ public class DatasourceService { DatasourceExample example = new DatasourceExample(); example.createCriteria().andNameEqualTo(datasource.getName()); if (CollectionUtils.isNotEmpty(datasourceMapper.selectByExample(example))) { - DEException.throwException("Exist data connection with the same name "); + DEException.throwException(Translator.get("i18n_ds_name_exists")); } long currentTimeMillis = System.currentTimeMillis(); datasource.setId(UUID.randomUUID().toString()); diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index 457b55ba55..c3d50d92fe 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -241,4 +241,5 @@ i18n_union_field_exists=The same field can't in two dataset i18n_cron_time_error=Start time can't greater then end time i18n_auth_source_be_canceled=This Auth Resource Already Be Canceled i18n_username_exists=ID is already exists +i18n_ds_name_exists=Datasource name exists diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index aa40d38b40..8ac7dde23b 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -243,3 +243,4 @@ i18n_union_field_exists=两个数据集之间关联不能出现多次相同字 i18n_cron_time_error=开始时间不能大于结束时间 i18n_auth_source_be_canceled=当前资源授权权限已经被取消 i18n_username_exists=用户ID已存在 +i18n_ds_name_exists=数据源名称已存在 diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index 8d6b357d76..b19c84ce96 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -243,3 +243,4 @@ i18n_union_field_exists=兩個數據集之間關聯不能出現多次相同字 i18n_cron_time_error=開始時間不能大於結束時間 i18n_auth_source_be_canceled=當前資源授權權限已經被取消 i18n_username_exists=用戶ID已存在 +i18n_ds_name_exists=數據源名稱已存在 diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index 34dbdeeb61..ee62f4ae68 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -94,8 +94,10 @@ export default { this.$refs.dsForm.validate(valid => { if (valid) { const method = this.formType === 'add' ? addDs : editDs - this.form.configuration = JSON.stringify(this.form.configuration) - method(this.form).then(res => { + // this.form.configuration = JSON.stringify(this.form.configuration) + const form = JSON.parse(JSON.stringify(this.form)) + form.configuration = JSON.stringify(form.configuration) + method(form).then(res => { this.$success(this.$t('commons.save_success')) this.backToList() }) diff --git a/frontend/src/views/system/user/index.vue b/frontend/src/views/system/user/index.vue index 6b5b9204d6..74fc75cf0c 100644 --- a/frontend/src/views/system/user/index.vue +++ b/frontend/src/views/system/user/index.vue @@ -25,6 +25,24 @@
{{ scope.row.dept && scope.row.dept.deptName }}
+ + +