From bb76675ec120e0c5dd3fbb1deb8c5075e2d0f80b Mon Sep 17 00:00:00 2001 From: taojinlong Date: Thu, 27 May 2021 18:14:43 +0800 Subject: [PATCH 01/23] =?UTF-8?q?fix:=20=E5=90=8C=E6=AD=A5=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/dataset/ExtractDataService.java | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index a47abc7ee0..e953b80976 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -174,12 +174,19 @@ public class ExtractDataService { jdbcProvider.exec(datasourceRequest); } - public void extractData(String datasetTableId, String taskId, String type) { - DatasetTable datasetTable = dataSetTableService.get(datasetTableId); + + private synchronized boolean updateSyncStatus(DatasetTable datasetTable ){ datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); - example.createCriteria().andIdEqualTo(datasetTableId); - if (datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0) { + example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusEqualTo(JobStatus.Completed.name()); + example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull(); + return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; + } + + public void extractData(String datasetTableId, String taskId, String type) { + DatasetTable datasetTable = dataSetTableService.get(datasetTableId); + if(updateSyncStatus(datasetTable)){ + LogUtil.info("Skip synchronization task for table : " + datasetTableId); return; } DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); @@ -270,7 +277,7 @@ public class ExtractDataService { dataSetTableTaskService.update(datasetTableTask); } datasetTable.setSyncStatus(JobStatus.Completed.name()); - example.clear(); + DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andIdEqualTo(datasetTableId); datasetTableMapper.updateByExampleSelective(datasetTable, example); } @@ -619,26 +626,27 @@ public class ExtractDataService { } public boolean isKettleRunning() { - try { - if (!InetAddress.getByName(carte).isReachable(1000)) { - return false; - } - HttpClient httpClient; - HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); - HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); - clientBuilder.setConnectionTimeout(1); - clientBuilder.setCredentials(user, passwd); - httpClient = clientBuilder.build(); - HttpResponse httpResponse = httpClient.execute(getMethod); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - if (statusCode != -1 && statusCode < 400) { - return true; - } else { - return false; - } - } catch (Exception e) { - return false; - } + return true; +// try { +// if (!InetAddress.getByName(carte).isReachable(1000)) { +// return false; +// } +// HttpClient httpClient; +// HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); +// HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); +// clientBuilder.setConnectionTimeout(1); +// clientBuilder.setCredentials(user, passwd); +// httpClient = clientBuilder.build(); +// HttpResponse httpResponse = httpClient.execute(getMethod); +// int statusCode = httpResponse.getStatusLine().getStatusCode(); +// if (statusCode != -1 && statusCode < 400) { +// return true; +// } else { +// return false; +// } +// } catch (Exception e) { +// return false; +// } } private static String alterColumnTypeCode = " if(\"FILED\".equalsIgnoreCase(filed)){\n" + From 3075f93c8421f19b4317e4f9cbbdf774a40b343e Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 12:05:09 +0800 Subject: [PATCH 02/23] =?UTF-8?q?feat(user):=E7=94=A8=E6=88=B7ID=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E6=A0=A1=E9=AA=8Cfix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/dataease/service/sys/SysUserService.java | 15 +++++++++++++++ .../main/resources/i18n/messages_en_US.properties | 1 + .../main/resources/i18n/messages_zh_CN.properties | 1 + .../main/resources/i18n/messages_zh_TW.properties | 1 + 4 files changed, 18 insertions(+) diff --git a/backend/src/main/java/io/dataease/service/sys/SysUserService.java b/backend/src/main/java/io/dataease/service/sys/SysUserService.java index e7db5dd9cd..eb1d6ac33f 100644 --- a/backend/src/main/java/io/dataease/service/sys/SysUserService.java +++ b/backend/src/main/java/io/dataease/service/sys/SysUserService.java @@ -19,6 +19,7 @@ import io.dataease.controller.sys.request.SysUserPwdRequest; import io.dataease.controller.sys.request.SysUserStateRequest; import io.dataease.controller.sys.response.SysUserGridResponse; import io.dataease.controller.sys.response.SysUserRole; +import io.dataease.i18n.Translator; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -78,6 +79,7 @@ public class SysUserService { @Transactional public int save(SysUserCreateRequest request) { + checkUsername(request); SysUser user = BeanUtils.copyBean(new SysUser(), request); long now = System.currentTimeMillis(); user.setCreateTime(now); @@ -106,6 +108,7 @@ public class SysUserService { @CacheEvict(value = AuthConstants.USER_CACHE_NAME, key = "'user' + #request.userId") @Transactional public int update(SysUserCreateRequest request) { + checkUsername(request); if (StringUtils.isEmpty(request.getPassword())) { request.setPassword(null); } @@ -235,4 +238,16 @@ public class SysUserService { sysUserMapper.updateByPrimaryKeySelective(sysUser); } + private void checkUsername(SysUserCreateRequest request) { + SysUserExample sysUserExample = new SysUserExample(); + SysUserExample.Criteria criteria = sysUserExample.createCriteria(); + if (request.getUserId() != null) { + criteria.andUserIdNotEqualTo(request.getUserId()); + } + criteria.andUsernameEqualTo(request.getUsername()); + List sysUsers = sysUserMapper.selectByExample(sysUserExample); + if (CollectionUtils.isNotEmpty(sysUsers)) { + throw new RuntimeException(Translator.get("i18n_username_exists")); + } + } } diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index 4a1e306b1d..457b55ba55 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -240,4 +240,5 @@ i18n_union_already_exists=Union relation already exists i18n_union_field_exists=The same field can't in two dataset i18n_cron_time_error=Start time can't greater then end time i18n_auth_source_be_canceled=This Auth Resource Already Be Canceled +i18n_username_exists=ID is already exists diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index 4dbc419e57..aa40d38b40 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -242,3 +242,4 @@ i18n_union_already_exists=关联关系已存在 i18n_union_field_exists=两个数据集之间关联不能出现多次相同字段 i18n_cron_time_error=开始时间不能大于结束时间 i18n_auth_source_be_canceled=当前资源授权权限已经被取消 +i18n_username_exists=用户ID已存在 diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index 5983b903c1..8d6b357d76 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -242,3 +242,4 @@ i18n_union_already_exists=關聯關系已存在 i18n_union_field_exists=兩個數據集之間關聯不能出現多次相同字段 i18n_cron_time_error=開始時間不能大於結束時間 i18n_auth_source_be_canceled=當前資源授權權限已經被取消 +i18n_username_exists=用戶ID已存在 From 8d39ec1b4895f4098610eab25b75bc5e31fef126 Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 12:08:44 +0800 Subject: [PATCH 03/23] =?UTF-8?q?feat(datasource):desc=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E9=9D=9E=E5=BF=85=E5=A1=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/main/resources/db/migration/V2__dataease_ddl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/resources/db/migration/V2__dataease_ddl.sql b/backend/src/main/resources/db/migration/V2__dataease_ddl.sql index 846ba41197..b21b7ca9c6 100644 --- a/backend/src/main/resources/db/migration/V2__dataease_ddl.sql +++ b/backend/src/main/resources/db/migration/V2__dataease_ddl.sql @@ -40,7 +40,7 @@ DROP TABLE IF EXISTS `datasource`; CREATE TABLE `datasource` ( `id` varchar(50) NOT NULL DEFAULT '' COMMENT 'ID', `name` varchar(50) NOT NULL COMMENT '名称', - `desc` varchar(50) NOT NULL COMMENT '描述', + `desc` varchar(50) COMMENT '描述', `type` varchar(50) NOT NULL COMMENT '类型', `configuration` longtext NOT NULL COMMENT '详细信息', `create_time` bigint(13) NOT NULL COMMENT 'Create timestamp', From 92549c13d43783b6c5bbd367ed2cde3402e22dfd Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 28 May 2021 12:19:14 +0800 Subject: [PATCH 04/23] =?UTF-8?q?fix:=20=E6=A3=80=E6=B5=8B=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/dataease/base/domain/DatasetTable.java | 2 + .../base/domain/DatasetTableExample.java | 70 +++++++++++++++++++ .../base/mapper/DatasetTableMapper.xml | 28 ++++++-- .../datasource/service/DatasourceService.java | 2 +- .../dataease/job/sechedule/DeScheduleJob.java | 1 + .../job/sechedule/ExtractDataJob.java | 2 +- .../service/dataset/DataSetTableService.java | 59 ++++++++++++---- .../service/dataset/ExtractDataService.java | 7 +- .../db/migration/V6__alter_table.sql | 3 +- 9 files changed, 150 insertions(+), 24 deletions(-) diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTable.java b/backend/src/main/java/io/dataease/base/domain/DatasetTable.java index 9a0590bba7..e88c0febff 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTable.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTable.java @@ -21,6 +21,8 @@ public class DatasetTable implements Serializable { private Long createTime; + private String qrtzInstance; + private String syncStatus; private String info; diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java index b229db8021..b0dad0931d 100644 --- a/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java +++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableExample.java @@ -644,6 +644,76 @@ public class DatasetTableExample { return (Criteria) this; } + public Criteria andQrtzInstanceIsNull() { + addCriterion("qrtz_instance is null"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceIsNotNull() { + addCriterion("qrtz_instance is not null"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceEqualTo(String value) { + addCriterion("qrtz_instance =", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotEqualTo(String value) { + addCriterion("qrtz_instance <>", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceGreaterThan(String value) { + addCriterion("qrtz_instance >", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceGreaterThanOrEqualTo(String value) { + addCriterion("qrtz_instance >=", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLessThan(String value) { + addCriterion("qrtz_instance <", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLessThanOrEqualTo(String value) { + addCriterion("qrtz_instance <=", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceLike(String value) { + addCriterion("qrtz_instance like", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotLike(String value) { + addCriterion("qrtz_instance not like", value, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceIn(List values) { + addCriterion("qrtz_instance in", values, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotIn(List values) { + addCriterion("qrtz_instance not in", values, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceBetween(String value1, String value2) { + addCriterion("qrtz_instance between", value1, value2, "qrtzInstance"); + return (Criteria) this; + } + + public Criteria andQrtzInstanceNotBetween(String value1, String value2) { + addCriterion("qrtz_instance not between", value1, value2, "qrtzInstance"); + return (Criteria) this; + } + public Criteria andSyncStatusIsNull() { addCriterion("sync_status is null"); return (Criteria) this; diff --git a/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml index f465f58ec3..35c7c22c34 100644 --- a/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/DatasetTableMapper.xml @@ -10,6 +10,7 @@ + @@ -74,7 +75,8 @@ - id, `name`, scene_id, data_source_id, `type`, `mode`, create_by, create_time, sync_status + id, `name`, scene_id, data_source_id, `type`, `mode`, create_by, create_time, qrtz_instance, + sync_status info @@ -130,12 +132,12 @@ insert into dataset_table (id, `name`, scene_id, data_source_id, `type`, `mode`, - create_by, create_time, sync_status, - info) + create_by, create_time, qrtz_instance, + sync_status, info) values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{sceneId,jdbcType=VARCHAR}, #{dataSourceId,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{mode,jdbcType=INTEGER}, - #{createBy,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{syncStatus,jdbcType=VARCHAR}, - #{info,jdbcType=LONGVARCHAR}) + #{createBy,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{qrtzInstance,jdbcType=VARCHAR}, + #{syncStatus,jdbcType=VARCHAR}, #{info,jdbcType=LONGVARCHAR}) insert into dataset_table @@ -164,6 +166,9 @@ create_time, + + qrtz_instance, + sync_status, @@ -196,6 +201,9 @@ #{createTime,jdbcType=BIGINT}, + + #{qrtzInstance,jdbcType=VARCHAR}, + #{syncStatus,jdbcType=VARCHAR}, @@ -237,6 +245,9 @@ create_time = #{record.createTime,jdbcType=BIGINT}, + + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, + sync_status = #{record.syncStatus,jdbcType=VARCHAR}, @@ -258,6 +269,7 @@ `mode` = #{record.mode,jdbcType=INTEGER}, create_by = #{record.createBy,jdbcType=VARCHAR}, create_time = #{record.createTime,jdbcType=BIGINT}, + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, sync_status = #{record.syncStatus,jdbcType=VARCHAR}, info = #{record.info,jdbcType=LONGVARCHAR} @@ -274,6 +286,7 @@ `mode` = #{record.mode,jdbcType=INTEGER}, create_by = #{record.createBy,jdbcType=VARCHAR}, create_time = #{record.createTime,jdbcType=BIGINT}, + qrtz_instance = #{record.qrtzInstance,jdbcType=VARCHAR}, sync_status = #{record.syncStatus,jdbcType=VARCHAR} @@ -303,6 +316,9 @@ create_time = #{createTime,jdbcType=BIGINT}, + + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, + sync_status = #{syncStatus,jdbcType=VARCHAR}, @@ -321,6 +337,7 @@ `mode` = #{mode,jdbcType=INTEGER}, create_by = #{createBy,jdbcType=VARCHAR}, create_time = #{createTime,jdbcType=BIGINT}, + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, sync_status = #{syncStatus,jdbcType=VARCHAR}, info = #{info,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=VARCHAR} @@ -334,6 +351,7 @@ `mode` = #{mode,jdbcType=INTEGER}, create_by = #{createBy,jdbcType=VARCHAR}, create_time = #{createTime,jdbcType=BIGINT}, + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR}, sync_status = #{syncStatus,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR} diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index 721f1a6ff4..d151efc6fd 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -145,7 +145,7 @@ public class DatasourceService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); datasourceProvider.initDataSource(datasourceRequest); - LogUtil.error("Succsss to init datasource connection pool: " + datasource.getName()); + LogUtil.info("Succsss to init datasource connection pool: " + datasource.getName()); }catch (Exception e){ LogUtil.error("Failed to init datasource connection pool: " + datasource.getName(), e); } diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java index 3fa52a44d3..ecbf81d1e6 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java @@ -1,5 +1,6 @@ package io.dataease.job.sechedule; +import com.google.gson.Gson; import io.dataease.commons.utils.LogUtil; import org.quartz.*; diff --git a/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java index d6411e3b96..b8f03d5a75 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java @@ -15,7 +15,7 @@ public class ExtractDataJob extends DeScheduleJob{ @Override void businessExecute(JobExecutionContext context) { - extractDataService.extractData(datasetTableId, taskId, updateType); + extractDataService.extractData(datasetTableId, taskId, updateType, context); } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index e2c3910b4d..2e950b79cb 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1,11 +1,10 @@ package io.dataease.service.dataset; +import com.fit2cloud.quartz.anno.QuartzScheduled; import com.google.gson.Gson; import io.dataease.base.domain.*; -import io.dataease.base.mapper.DatasetTableIncrementalConfigMapper; -import io.dataease.base.mapper.DatasetTableMapper; -import io.dataease.base.mapper.DatasourceMapper; +import io.dataease.base.mapper.*; import io.dataease.base.mapper.ext.ExtDataSetTableMapper; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.utils.*; @@ -75,6 +74,10 @@ public class DataSetTableService { private DataSetTableUnionService dataSetTableUnionService; @Resource private DataSetTableTaskLogService dataSetTableTaskLogService; + @Resource + private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; + @Resource + private DatasetTableTaskLogMapper datasetTableTaskLogMapper; @Value("${upload.file.path}") private String path; @@ -106,7 +109,7 @@ public class DataSetTableService { saveTableField(datasetTable); if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { commonThreadPool.addTask(() -> { - extractDataService.extractData(datasetTable.getId(), null, "all_scope"); + extractDataService.extractData(datasetTable.getId(), null, "all_scope", null); }); } } @@ -867,22 +870,48 @@ public class DataSetTableService { } public Boolean checkDorisTableIsExists(String id) throws Exception { -// Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); -// JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); -// DatasourceRequest datasourceRequest = new DatasourceRequest(); -// datasourceRequest.setDatasource(dorisDatasource); -// QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); -// datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id))); -// List data = jdbcProvider.getData(datasourceRequest); -// return CollectionUtils.isNotEmpty(data); - return true; + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(dorisDatasource); + QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType()); + datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id))); + List data = jdbcProvider.getData(datasourceRequest); + return CollectionUtils.isNotEmpty(data); } + @QuartzScheduled(cron = "0 0/3 * * * ?") public void updateDatasetTableStatus(){ - DatasetTable record = new DatasetTable(); - record.setSyncStatus(JobStatus.Completed.name()); + List qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); + List activeQrtzInstances = qrtzSchedulerStates.stream().filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > System.currentTimeMillis()).map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); + List jobStoppeddDatasetTables = new ArrayList<>(); + DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); + + datasetTableMapper.selectByExample(example).forEach(datasetTable -> { + if(StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))){ + jobStoppeddDatasetTables.add(datasetTable); + } + }); + + if(CollectionUtils.isEmpty(jobStoppeddDatasetTables)){ + return; + } + + DatasetTable record = new DatasetTable(); + record.setSyncStatus(JobStatus.Completed.name()); + example.clear(); + example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()).andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); datasetTableMapper.updateByExampleSelective(record, example); + + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setStatus(JobStatus.Error.name()); + datasetTableTaskLog.setInfo("Job stopped due to system error."); + + DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); + datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()).andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); + datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index e953b80976..a0d3510bc3 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -63,6 +63,7 @@ import org.pentaho.di.trans.steps.textfileoutput.TextFileOutputMeta; import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef; import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta; import org.pentaho.di.www.SlaveServerJobStatus; +import org.quartz.JobExecutionContext; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -183,7 +184,7 @@ public class ExtractDataService { return datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; } - public void extractData(String datasetTableId, String taskId, String type) { + public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = dataSetTableService.get(datasetTableId); if(updateSyncStatus(datasetTable)){ LogUtil.info("Skip synchronization task for table : " + datasetTableId); @@ -193,6 +194,10 @@ public class ExtractDataService { UpdateType updateType = UpdateType.valueOf(type); Datasource datasource = new Datasource(); try { + if(context != null){ + datasetTable.setQrtzInstance(context.getFireInstanceId()); + datasetTableMapper.updateByPrimaryKeySelective(datasetTable); + } if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) { datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); } else { diff --git a/backend/src/main/resources/db/migration/V6__alter_table.sql b/backend/src/main/resources/db/migration/V6__alter_table.sql index 9551f19b98..37106309dd 100644 --- a/backend/src/main/resources/db/migration/V6__alter_table.sql +++ b/backend/src/main/resources/db/migration/V6__alter_table.sql @@ -1 +1,2 @@ -ALTER TABLE `dataease`.`dataset_table` ADD COLUMN `sync_status` VARCHAR(45) NULL AFTER `create_time`; +ALTER TABLE `dataset_table` ADD COLUMN `sync_status` VARCHAR(45) NULL AFTER `create_time`; +ALTER TABLE `dataset_table` ADD COLUMN `qrtz_instance` VARCHAR(1024) NULL AFTER `create_time`; From e955d2bfff32630bf452b4de5b39e89df8f260dd Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 12:40:19 +0800 Subject: [PATCH 05/23] =?UTF-8?q?feat(frontend):user=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=A7=92=E8=89=B2=E5=AD=97=E6=AE=B5=EF=BC=9B?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E6=B7=BB=E5=8A=A0=E6=8A=A5=E9=94=99?= =?UTF-8?q?fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/service/DatasourceService.java | 3 ++- .../resources/i18n/messages_en_US.properties | 1 + .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + frontend/src/views/system/datasource/form.vue | 6 ++++-- frontend/src/views/system/user/index.vue | 18 ++++++++++++++++++ 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index d151efc6fd..099862fcbe 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -18,6 +18,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.DatasourceDTO; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.i18n.Translator; import io.dataease.service.dataset.DataSetGroupService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -49,7 +50,7 @@ public class DatasourceService { DatasourceExample example = new DatasourceExample(); example.createCriteria().andNameEqualTo(datasource.getName()); if (CollectionUtils.isNotEmpty(datasourceMapper.selectByExample(example))) { - DEException.throwException("Exist data connection with the same name "); + DEException.throwException(Translator.get("i18n_ds_name_exists")); } long currentTimeMillis = System.currentTimeMillis(); datasource.setId(UUID.randomUUID().toString()); diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index 457b55ba55..c3d50d92fe 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -241,4 +241,5 @@ i18n_union_field_exists=The same field can't in two dataset i18n_cron_time_error=Start time can't greater then end time i18n_auth_source_be_canceled=This Auth Resource Already Be Canceled i18n_username_exists=ID is already exists +i18n_ds_name_exists=Datasource name exists diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index aa40d38b40..8ac7dde23b 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -243,3 +243,4 @@ i18n_union_field_exists=两个数据集之间关联不能出现多次相同字 i18n_cron_time_error=开始时间不能大于结束时间 i18n_auth_source_be_canceled=当前资源授权权限已经被取消 i18n_username_exists=用户ID已存在 +i18n_ds_name_exists=数据源名称已存在 diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index 8d6b357d76..b19c84ce96 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -243,3 +243,4 @@ i18n_union_field_exists=兩個數據集之間關聯不能出現多次相同字 i18n_cron_time_error=開始時間不能大於結束時間 i18n_auth_source_be_canceled=當前資源授權權限已經被取消 i18n_username_exists=用戶ID已存在 +i18n_ds_name_exists=數據源名稱已存在 diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index 34dbdeeb61..ee62f4ae68 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -94,8 +94,10 @@ export default { this.$refs.dsForm.validate(valid => { if (valid) { const method = this.formType === 'add' ? addDs : editDs - this.form.configuration = JSON.stringify(this.form.configuration) - method(this.form).then(res => { + // this.form.configuration = JSON.stringify(this.form.configuration) + const form = JSON.parse(JSON.stringify(this.form)) + form.configuration = JSON.stringify(form.configuration) + method(form).then(res => { this.$success(this.$t('commons.save_success')) this.backToList() }) diff --git a/frontend/src/views/system/user/index.vue b/frontend/src/views/system/user/index.vue index 6b5b9204d6..74fc75cf0c 100644 --- a/frontend/src/views/system/user/index.vue +++ b/frontend/src/views/system/user/index.vue @@ -25,6 +25,24 @@
{{ scope.row.dept && scope.row.dept.deptName }}
+ + + @@ -57,8 +58,8 @@ export default { initData() { this.resetData() if (this.table.id) { - this.table.row = 10 - post('/dataset/table/getPreviewData/1/10', this.table).then(response => { + this.table.row = 100 + post('/dataset/table/getPreviewData/1/100', this.table).then(response => { this.fields = response.data.fields this.data = response.data.data const datas = this.data From b4e503612aeb491fdc27a5d642ea732233e08e75 Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 16:12:37 +0800 Subject: [PATCH 10/23] =?UTF-8?q?feat(=E8=A7=86=E5=9B=BE):=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E8=A7=86=E5=9B=BE=EF=BC=8C=E9=A2=84=E8=A7=88=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=A1=A8=E5=A4=B4=E5=A2=9E=E5=8A=A0=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E7=B1=BB=E5=9E=8Bicon=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/views/dataset/common/DatasetTableData.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/dataset/common/DatasetTableData.vue b/frontend/src/views/dataset/common/DatasetTableData.vue index 383b0e89e9..89b2e73dde 100644 --- a/frontend/src/views/dataset/common/DatasetTableData.vue +++ b/frontend/src/views/dataset/common/DatasetTableData.vue @@ -14,11 +14,17 @@ :key="field.dataeaseName" min-width="200px" :field="field.dataeaseName" - :title="field.name" :resizable="true" - /> + > + + - {{ $t('chart.preview_100_data') }} + {{ $t('chart.preview_100_data') }} From 1965ff3eabc1016c7456e61926b46e14883d0c4b Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 28 May 2021 16:31:07 +0800 Subject: [PATCH 11/23] =?UTF-8?q?feat(frontend):=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E3=80=81=E8=AD=A6=E5=91=8A=E3=80=81=E9=94=99=E8=AF=AF=E7=AD=89?= =?UTF-8?q?=E8=89=B2=E5=80=BC=E4=BF=AE=E6=94=B9=E4=B8=8Eelement-ui?= =?UTF-8?q?=EF=BC=8Cfit2cloud-ui=E4=B8=80=E8=87=B4=EF=BC=8C=E5=8E=9F?= =?UTF-8?q?=E5=85=88=E7=9A=84=E5=A4=AA=E4=BA=AE=EF=BC=9B=E8=A7=92=E8=89=B2?= =?UTF-8?q?i18n=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/styles/variables.scss | 6 +++--- frontend/src/views/system/role/index.vue | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/styles/variables.scss b/frontend/src/styles/variables.scss index 3272b606f7..4dfe4ad8d9 100644 --- a/frontend/src/styles/variables.scss +++ b/frontend/src/styles/variables.scss @@ -5,9 +5,9 @@ /* theme color */ $--color-primary: #0a7be0; -$--color-success: #13ce66; -$--color-warning: #ffba00; -$--color-danger: #ff4949; +$--color-success: #67C23A; +$--color-warning: #E6A23C; +$--color-danger: #F56C6C; // $--color-info: #1E1E1E; $--button-font-weight: 400; diff --git a/frontend/src/views/system/role/index.vue b/frontend/src/views/system/role/index.vue index a4c7d7c521..e04aae98c6 100644 --- a/frontend/src/views/system/role/index.vue +++ b/frontend/src/views/system/role/index.vue @@ -251,13 +251,13 @@ export default { this.$refs.menu.setCheckedKeys(this.menuIds) }, handleDelete(row) { - this.$confirm(this.$t('commons.confirm_delete') + ': ' + row.name + '？', this.$t('role.tips'), { + this.$confirm(this.$t('role.confirm_delete') + ': ' + row.name + '？', this.$t('role.tips'), { confirmButtonText: this.$t('commons.confirm'), cancelButtonText: this.$t('commons.cancel'), type: 'warning' }).then(() => { delRole(row.roleId).then(res => { - this.$success(this.$t('commons.modify_success')) + this.$success(this.$t('commons.delete_success')) this.search() }) }).catch(() => { From f99865cf8d5129c3f1e513db7055b26274e2d7f0 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 28 May 2021 16:37:37 +0800 Subject: [PATCH 12/23] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=AB=8B=E5=8D=B3=E6=89=A7=E8=A1=8C=E6=97=B6=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/dataset/DataSetTableService.java | 16 ++++++------- .../dataset/DataSetTableTaskLogService.java | 16 +++++++++++++ .../dataset/DataSetTableTaskService.java | 23 +++++++++++++++++-- .../service/dataset/ExtractDataService.java | 18 +++++++++++---- .../resources/i18n/messages_en_US.properties | 2 +- .../resources/i18n/messages_zh_CN.properties | 1 + .../resources/i18n/messages_zh_TW.properties | 1 + frontend/src/lang/en.js | 4 ++-- frontend/src/lang/tw.js | 4 ++-- frontend/src/lang/zh.js | 4 ++-- .../src/views/dataset/data/UpdateInfo.vue | 19 +++++++-------- 11 files changed, 77 insertions(+), 31 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 2e950b79cb..14005e8ba9 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -268,18 +268,18 @@ public class DataSetTableService { e.printStackTrace(); } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - List datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId()); - if (CollectionUtils.isEmpty(datasetTableTaskLogs)) { + if (StringUtils.isEmpty(datasetTable.getSyncStatus()) || datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) { map.put("status", "warnning"); map.put("msg", Translator.get("i18n_processing_data")); dataSetPreviewPage.setTotal(0); - }else if (datasetTableTaskLogs.get(0).getStatus().equalsIgnoreCase(JobStatus.Underway.name())) { - map.put("status", "warnning"); - map.put("msg", Translator.get("i18n_processing_data")); - dataSetPreviewPage.setTotal(0); - }else if (datasetTableTaskLogs.get(0).getStatus().equalsIgnoreCase(JobStatus.Error.name())) { + }else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) { + List datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId()); map.put("status", "error"); - map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo()); + if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){ + map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo()); + }else { + map.put("msg", "Failed to extract data."); + } dataSetPreviewPage.setTotal(0); }else { Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index b0072fc786..b4ee6d2929 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -4,6 +4,7 @@ import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.base.domain.DatasetTableTaskLogExample; import io.dataease.base.mapper.DatasetTableTaskLogMapper; import io.dataease.base.mapper.ext.ExtDataSetTaskMapper; +import io.dataease.commons.constants.JobStatus; import io.dataease.dto.dataset.DataSetTaskLogDTO; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -55,4 +56,19 @@ public class DataSetTableTaskLogService { criteria.andTableIdEqualTo(datasetId); return datasetTableTaskLogMapper.selectByExampleWithBLOBs(datasetTableTaskLogExample); } + + public List select(DatasetTableTaskLog datasetTableTaskLog){ + DatasetTableTaskLogExample example = new DatasetTableTaskLogExample(); + DatasetTableTaskLogExample.Criteria criteria = example.createCriteria(); + if(StringUtils.isNotEmpty(datasetTableTaskLog.getStatus())){ + criteria.andStatusEqualTo(datasetTableTaskLog.getStatus()); + } + if(StringUtils.isNotEmpty(datasetTableTaskLog.getTableId())){ + criteria.andTableIdEqualTo(datasetTableTaskLog.getTableId()); + } + if(StringUtils.isNotEmpty(datasetTableTaskLog.getTaskId())){ + criteria.andTaskIdEqualTo(datasetTableTaskLog.getTaskId()); + } + return datasetTableTaskLogMapper.selectByExampleWithBLOBs(example); + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index 85b5fa00a4..e99650b809 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -2,7 +2,10 @@ package io.dataease.service.dataset; import io.dataease.base.domain.DatasetTableTask; import io.dataease.base.domain.DatasetTableTaskExample; +import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.base.mapper.DatasetTableTaskMapper; +import io.dataease.commons.constants.JobStatus; +import io.dataease.commons.constants.ScheduleType; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.i18n.Translator; import io.dataease.service.ScheduleService; @@ -33,10 +36,12 @@ public class DataSetTableTaskService { @Resource @Lazy private DataSetTableService dataSetTableService; - + @Resource + private ExtractDataService extractDataService; public DatasetTableTask save(DataSetTaskRequest dataSetTaskRequest) throws Exception { - dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig()); DatasetTableTask datasetTableTask = dataSetTaskRequest.getDatasetTableTask(); + dataSetTableService.saveIncrementalConfig(dataSetTaskRequest.getDatasetTableIncrementalConfig()); + // check if (StringUtils.isNotEmpty(datasetTableTask.getCron())) { if (!CronExpression.isValidExpression(datasetTableTask.getCron())) { @@ -54,6 +59,20 @@ public class DataSetTableTaskService { if (StringUtils.isEmpty(datasetTableTask.getId())) { datasetTableTask.setId(UUID.randomUUID().toString()); datasetTableTask.setCreateTime(System.currentTimeMillis()); + // SIMPLE 类型，提前占位 + if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){ + if(extractDataService.updateSyncStatus(dataSetTableService.get(datasetTableTask.getTableId()))){ + throw new Exception(Translator.get("i18n_sync_job_exists")); + }else { + //write log + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + datasetTableTaskLog.setTableId(datasetTableTask.getTableId()); + datasetTableTaskLog.setTaskId(datasetTableTask.getId()); + datasetTableTaskLog.setStatus(JobStatus.Underway.name()); + datasetTableTaskLog.setStartTime(System.currentTimeMillis()); + dataSetTableTaskLogService.save(datasetTableTaskLog); + } + } datasetTableTaskMapper.insert(datasetTableTask); } else { datasetTableTaskMapper.updateByPrimaryKeySelective(datasetTableTask); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index a0d3510bc3..78a32116e8 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -3,6 +3,7 @@ package io.dataease.service.dataset; import com.google.gson.Gson; import io.dataease.base.domain.*; import io.dataease.base.mapper.DatasetTableMapper; +import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.DatasourceMapper; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.constants.ScheduleType; @@ -90,13 +91,17 @@ public class ExtractDataService { @Resource private DataSetTableFieldsService dataSetTableFieldsService; @Resource + @Lazy private DataSetTableTaskLogService dataSetTableTaskLogService; @Resource + @Lazy private DataSetTableTaskService dataSetTableTaskService; @Resource private DatasourceMapper datasourceMapper; @Resource private DatasetTableMapper datasetTableMapper; + @Resource + private DatasetTableTaskMapper datasetTableTaskMapper; private static String lastUpdateTime = "${__last_update_time__}"; private static String currentUpdateTime = "${__current_update_time__}"; @@ -176,7 +181,7 @@ public class ExtractDataService { } - private synchronized boolean updateSyncStatus(DatasetTable datasetTable ){ + public synchronized boolean updateSyncStatus(DatasetTable datasetTable ){ datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusEqualTo(JobStatus.Completed.name()); @@ -186,7 +191,9 @@ public class ExtractDataService { public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = dataSetTableService.get(datasetTableId); - if(updateSyncStatus(datasetTable)){ + DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); + boolean isSIMPLEJob = (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())); + if(updateSyncStatus(datasetTable) && !isSIMPLEJob){ LogUtil.info("Skip synchronization task for table : " + datasetTableId); return; } @@ -276,7 +283,6 @@ public class ExtractDataService { datasetTableTaskLog.setEndTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); } finally { - DatasetTableTask datasetTableTask = dataSetTableTaskService.get(taskId); if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); dataSetTableTaskService.update(datasetTableTask); @@ -292,8 +298,10 @@ public class ExtractDataService { datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - datasetTableTaskLog.setStartTime(System.currentTimeMillis()); - dataSetTableTaskLogService.save(datasetTableTaskLog); + if(CollectionUtils.isEmpty(dataSetTableTaskLogService.select(datasetTableTaskLog))){ + datasetTableTaskLog.setStartTime(System.currentTimeMillis()); + dataSetTableTaskLogService.save(datasetTableTaskLog); + } } private void extractData(DatasetTable datasetTable, String extractType) throws Exception { diff --git a/backend/src/main/resources/i18n/messages_en_US.properties b/backend/src/main/resources/i18n/messages_en_US.properties index c3d50d92fe..a89f27a5a3 100644 --- a/backend/src/main/resources/i18n/messages_en_US.properties +++ b/backend/src/main/resources/i18n/messages_en_US.properties @@ -242,4 +242,4 @@ i18n_cron_time_error=Start time can't greater then end time i18n_auth_source_be_canceled=This Auth Resource Already Be Canceled i18n_username_exists=ID is already exists i18n_ds_name_exists=Datasource name exists - +i18n_sync_job_exists=There is already a synchronization task running, please try again later diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index 8ac7dde23b..5a3a318e9e 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -244,3 +244,4 @@ i18n_cron_time_error=开始时间不能大于结束时间 i18n_auth_source_be_canceled=当前资源授权权限已经被取消 i18n_username_exists=用户ID已存在 i18n_ds_name_exists=数据源名称已存在 +i18n_sync_job_exists=已经有同步任务在运行，稍后重试 diff --git a/backend/src/main/resources/i18n/messages_zh_TW.properties b/backend/src/main/resources/i18n/messages_zh_TW.properties index b19c84ce96..49c3d8d98a 100644 --- a/backend/src/main/resources/i18n/messages_zh_TW.properties +++ b/backend/src/main/resources/i18n/messages_zh_TW.properties @@ -244,3 +244,4 @@ i18n_cron_time_error=開始時間不能大於結束時間 i18n_auth_source_be_canceled=當前資源授權權限已經被取消 i18n_username_exists=用戶ID已存在 i18n_ds_name_exists=數據源名稱已存在 +i18n_sync_job_exists=已經有同步任務在運行，稍後重試 diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 226822e2bb..1e00b88d08 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -732,13 +732,13 @@ export default { error: 'Error', completed: 'Completed', underway: 'underway', - task_update: 'Regular update', + task_update: 'Update Setting', update_type: 'Update Type', all_scope: 'Full update', add_scope: 'Incremental update', select_data_time: 'Select date time', execute_rate: 'Execution frequency', - execute_once: 'Only once', + execute_once: 'Execution Now', cron_config: 'Expression setting', no_limit: 'No limit', set_end_time: 'Set the end time', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 6f8cae65d9..2352f89100 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -732,13 +732,13 @@ export default { error: '錯誤', completed: '完成', underway: '執行中', - task_update: '定時更新', + task_update: '更新設置', update_type: '更新方式', all_scope: '全量更新', add_scope: '增量更新', select_data_time: '選擇日期時間', execute_rate: '執行頻率', - execute_once: '只執行一次', + execute_once: '立即執行', cron_config: '表達時設定', no_limit: '無限制', set_end_time: '設定結束時間', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 4f706be994..e206e2cb63 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -732,13 +732,13 @@ export default { error: '错误', completed: '完成', underway: '执行中', - task_update: '定时更新', + task_update: '更新设置', update_type: '更新方式', all_scope: '全量更新', add_scope: '增量更新', select_data_time: '选择日期时间', execute_rate: '执行频率', - execute_once: '只执行一次', + execute_once: '立即执行', cron_config: '表达式设定', no_limit: '无限制', set_end_time: '设定结束时间', diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index 3723b1cefd..19033303d8 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -116,14 +116,6 @@ /> - - - + - + + + + Date: Fri, 28 May 2021 17:48:05 +0800 Subject: [PATCH 13/23] =?UTF-8?q?feat(frontend):1.=E8=A7=92=E8=89=B2?= =?UTF-8?q?=E5=88=86=E9=A1=B5=E7=9B=B8=E5=85=B3bug=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=EF=BC=9B2.=E5=86=85=E7=BD=AE=E8=A7=92=E8=89=B2=E7=A6=81?= =?UTF-8?q?=E6=AD=A2=E7=BC=96=E8=BE=91=E5=92=8C=E5=88=A0=E9=99=A4=EF=BC=9B?= =?UTF-8?q?3.=E5=8F=B3=E4=B8=8A=E8=A7=92=E4=B8=8B=E6=8B=89=E8=8F=9C?= =?UTF-8?q?=E5=8D=95=E6=96=87=E6=A1=88=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/lang/en.js | 5 +++-- frontend/src/lang/tw.js | 5 +++-- frontend/src/lang/zh.js | 5 +++-- frontend/src/layout/components/Topbar.vue | 2 +- frontend/src/views/system/role/form.vue | 2 +- frontend/src/views/system/role/index.vue | 10 +++++++--- frontend/src/views/system/user/personPwd.vue | 2 +- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index a687fc8d5a..44634f221f 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -278,7 +278,7 @@ export default { default_module: 'Default Module' }, datasource: 'Datasource', - char_can_not_more_50: 'Name can not more 50 char', + char_can_not_more_50: 'Can not more 50 char', share_success: 'Share Success' }, documentation: { @@ -435,7 +435,8 @@ export default { current_user: 'Current User', origin_passwd: 'Origin Password', new_passwd: 'New Password', - confirm_passwd: 'Confirm Password' + confirm_passwd: 'Confirm Password', + change_password: 'Change Password' }, role: { menu_authorization: 'Menu Authorization', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index a8b24afed5..8330f734c8 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -278,7 +278,7 @@ export default { default_module: '默认模块' }, datasource: '數據源', - char_can_not_more_50: '名稱不能超過50字符', + char_can_not_more_50: '不能超過50字符', share_success: '分享成功' }, documentation: { @@ -435,7 +435,8 @@ export default { current_user: '當前用戶', origin_passwd: '原始密碼', new_passwd: '新密碼', - confirm_passwd: '確認密碼' + confirm_passwd: '確認密碼', + change_password: '修改密碼' }, role: { menu_authorization: '菜單授權', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index e0246ec79d..2feb5ef0f3 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -278,7 +278,7 @@ export default { default_module: '默认模块' }, datasource: '数据源', - char_can_not_more_50: '名称不能超过50字符', + char_can_not_more_50: '不能超过50字符', share_success: '分享成功' }, documentation: { @@ -435,7 +435,8 @@ export default { current_user: '当前用户', origin_passwd: '原始密码', new_passwd: '新密码', - confirm_passwd: '确认密码' + confirm_passwd: '确认密码', + change_password: '修改密码' }, role: { menu_authorization: '菜单授权', diff --git a/frontend/src/layout/components/Topbar.vue b/frontend/src/layout/components/Topbar.vue index 0f73ef356d..4f5bccaedb 100644 --- a/frontend/src/layout/components/Topbar.vue +++ b/frontend/src/layout/components/Topbar.vue @@ -47,7 +47,7 @@ {{ $t('commons.personal_info') }} - {{ $t('user.reset_password') }} + {{ $t('user.change_password') }} {{ $t('commons.help_documentation') }} diff --git a/frontend/src/views/system/role/form.vue b/frontend/src/views/system/role/form.vue index fd7619736d..7a9930f673 100644 --- a/frontend/src/views/system/role/form.vue +++ b/frontend/src/views/system/role/form.vue @@ -30,7 +30,7 @@ export default { name: [ { required: true, trigger: 'blur', validator: this.roleValidator } ], - code: [{ required: true, message: '请输入代码', trigger: 'blur' }] + description: [{ max: 50, message: this.$t('commons.char_can_not_more_50'), trigger: 'blur' }] }, roles: [], originName: null diff --git a/frontend/src/views/system/role/index.vue b/frontend/src/views/system/role/index.vue index e04aae98c6..2045fee5a2 100644 --- a/frontend/src/views/system/role/index.vue +++ b/frontend/src/views/system/role/index.vue @@ -18,6 +18,7 @@ +