dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 自定义数据源检测时间

Browse Source
This commit is contained in:
taojinlong 2022-09-14 16:33:47 +08:00
parent 9cb834ecab
commit 674abcb203
12 changed files with 188 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/commons/constants/ParamConstants.java
View File

@ -110,6 +110,8 @@ public interface ParamConstants {
enum BASIC implements ParamConstants {
FRONT_TIME_OUT("basic.frontTimeOut"),
MSG_TIME_OUT("basic.msgTimeOut"),
DS_CHECK_INTERVAL("basic.dsCheckInterval"),
DS_CHECK_INTERVAL_TYPE("basic.dsCheckIntervalType"),
DEFAULT_LOGIN_TYPE("basic.loginType"),
OPEN_HOME_PAGE("ui.openHomePage"),

4
backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java
View File

@ -22,5 +22,9 @@ public class BasicInfo implements Serializable {
private String templateAccessKey;
@ApiModelProperty("显示模板市场")
private String openMarketPage;
@ApiModelProperty("数据源检测时间间隔")
private String dsCheckInterval;
@ApiModelProperty("数据源检测时间间隔类型")
private String dsCheckIntervalType;
}

4
backend/src/main/java/io/dataease/job/sechedule/Schedular.java
View File

@ -22,9 +22,9 @@ public class Schedular {
dataSetTableService.updateDatasetTableStatus();
}
@QuartzScheduled(cron = "0 0/30 * * * ?")
@QuartzScheduled(cron = "0 0/3 * * * ?")
public void updateDatasourceStatus() {
datasourceService.updateDatasourceStatus();
datasourceService.checkDatasourceJob();
}
@QuartzScheduled(cron = "0 0/30 * * * ?")

2
backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
View File

@ -22,8 +22,10 @@ public class DataSourceInitStartListener implements ApplicationListener<Applicat
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
datasourceService.initAllDataSourceConnectionPool();
datasourceService.initDsCheckJob();
dataSetTableService.updateDatasetTableStatus();
engineService.initSimpleEngine();
}

1
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -790,6 +790,7 @@ public class ExtractDataService {
Thread.sleep(1000);
}
if (jobStatus.getStatusDescription().equals("Finished")) {
LogUtil.info(datasetTable.getId()+ ": " + jobStatus.getLoggingString());
return;
} else {
DataEaseException.throwException(jobStatus.getLoggingString());

86
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -9,7 +9,10 @@ import io.dataease.auth.annotation.DeCleaner;
import io.dataease.commons.constants.RedisConstants;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.controller.sys.response.BasicInfo;
import io.dataease.dto.TaskInstance;
import io.dataease.ext.ExtDataSourceMapper;
import io.dataease.ext.ExtTaskInstanceMapper;
import io.dataease.ext.UtilMapper;
import io.dataease.ext.query.GridExample;
import io.dataease.commons.constants.DePermissionType;
import io.dataease.commons.constants.SysAuthConstants;
@ -31,17 +34,20 @@ import io.dataease.i18n.Translator;
import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.base.mapper.DatasetTableMapper;
import io.dataease.plugins.common.base.mapper.DatasourceMapper;
import io.dataease.plugins.common.base.mapper.QrtzSchedulerStateMapper;
import io.dataease.plugins.common.constants.DatasetType;
import io.dataease.plugins.common.constants.DatasourceCalculationMode;
import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.dto.datasource.DataSourceType;
import io.dataease.plugins.common.dto.datasource.TableDesc;
import io.dataease.plugins.common.entity.GlobalTaskEntity;
import io.dataease.plugins.common.request.datasource.DatasourceRequest;
import io.dataease.plugins.config.SpringContextUtil;
import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.ApiProvider;
import io.dataease.service.ScheduleService;
import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.message.DeMsgutil;
import io.dataease.service.sys.SysAuthService;
@ -49,6 +55,7 @@ import io.dataease.service.system.SystemParameterService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
@ -77,6 +84,14 @@ public class DatasourceService {
private Environment env;
@Resource
private SystemParameterService systemParameterService;
@Autowired
private ScheduleService scheduleService;
@Resource
private QrtzSchedulerStateMapper qrtzSchedulerStateMapper;
@Resource
private UtilMapper utilMapper;
@Resource
private ExtTaskInstanceMapper extTaskInstanceMapper;
public Collection<DataSourceType> types() {
Collection<DataSourceType> types = new ArrayList<>();
@ -432,6 +447,24 @@ public class DatasourceService {
}
}
public void checkDatasourceJob() {
List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null);
List<String> activeQrtzInstances = qrtzSchedulerStates.stream()
.filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime()
+ qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp())
.map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<TaskInstance> taskInstances = extTaskInstanceMapper.select();
taskInstances.forEach(taskInstance -> {
if (StringUtils.isNotEmpty(taskInstance.getQrtzInstance()) && !activeQrtzInstances.contains(taskInstance.getQrtzInstance().substring(0, taskInstance.getQrtzInstance().length() - 13))) {
TaskInstance update = new TaskInstance();
update.setTaskId("Datasource_check_status");
extTaskInstanceMapper.update(update);
}
});
}
public void updateDatasourceStatus() {
List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample());
datasources.forEach(datasource -> checkAndUpdateDatasourceStatus(datasource, true));
@ -496,4 +529,57 @@ public class DatasourceService {
DeMsgutil.sendMsg(userId, typeId, content, gson.toJson(param));
});
}
public void updateDatasourceStatusJob(BasicInfo basicInfo, List<SystemParameter> parameters) {
String type = "";
Integer interval = 30;
boolean changeDsCheckTime = false;
basicInfo.getDsCheckInterval();
basicInfo.getDsCheckIntervalType();
for (SystemParameter parameter : parameters) {
if (parameter.getParamKey().equalsIgnoreCase("basic.dsCheckInterval") && !parameter.getParamValue().equalsIgnoreCase(basicInfo.getDsCheckInterval())) {
changeDsCheckTime = true;
interval = Integer.valueOf(parameter.getParamValue());
}
if (parameter.getParamKey().equalsIgnoreCase("basic.dsCheckIntervalType") && !parameter.getParamValue().equalsIgnoreCase(basicInfo.getDsCheckInterval())) {
changeDsCheckTime = true;
type = parameter.getParamValue();
}
}
if(!changeDsCheckTime){
return;
}
addJob(type, interval);
}
private void addJob(String type, Integer interval) {
String cron = "";
switch (type){
case "hour":
cron = "0 0 0/hour * * ? *".replace("hour", interval.toString());
break;
default:
cron = "0 0/minute * * * ? *".replace("minute", interval.toString());
}
GlobalTaskEntity globalTask = new GlobalTaskEntity();
globalTask.setCron(cron);
globalTask.setCreateTime(System.currentTimeMillis());
globalTask.setJobKey("Datasource_check_status");
globalTask.setTaskName("Datasource check status");
globalTask.setTaskType("dsTaskHandler");
globalTask.setStartTime(System.currentTimeMillis());
try {
scheduleService.addSchedule(globalTask);
}catch (Exception e){
e.printStackTrace();
}
}
public void initDsCheckJob(){
BasicInfo basicInfo = systemParameterService.basicInfo();
addJob(basicInfo.getDsCheckIntervalType(), Integer.valueOf(basicInfo.getDsCheckInterval()));
}
}

26
backend/src/main/java/io/dataease/service/system/SystemParameterService.java
View File

@ -8,7 +8,6 @@ import io.dataease.controller.sys.response.BasicInfo;
import io.dataease.dto.SystemParameterDTO;
import io.dataease.exception.DataEaseException;
import io.dataease.plugins.common.base.domain.FileMetadata;
import io.dataease.plugins.common.base.domain.SysParamAssist;
import io.dataease.plugins.common.base.domain.SystemParameter;
import io.dataease.plugins.common.base.domain.SystemParameterExample;
import io.dataease.plugins.common.base.mapper.SystemParameterMapper;
@ -17,10 +16,11 @@ import io.dataease.plugins.xpack.cas.dto.CasSaveResult;
import io.dataease.plugins.xpack.cas.service.CasXpackService;
import io.dataease.plugins.xpack.display.service.DisplayXpackService;
import io.dataease.service.FileService;
import io.dataease.service.datasource.DatasourceService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
@ -29,7 +29,6 @@ import javax.annotation.Resource;
import javax.imageio.ImageIO;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
@ -47,6 +46,9 @@ public class SystemParameterService {
private ExtSystemParameterMapper extSystemParameterMapper;
@Resource
private FileService fileService;
@Resource
@Lazy
private DatasourceService datasourceService;
public String searchEmail() {
return extSystemParameterMapper.email();
@ -86,7 +88,12 @@ public class SystemParameterService {
if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.TEMPLATE_ACCESS_KEY.getValue())) {
result.setTemplateAccessKey(param.getParamValue());
}
if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DS_CHECK_INTERVAL.getValue())) {
result.setDsCheckInterval(param.getParamValue());
}
if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DS_CHECK_INTERVAL_TYPE.getValue())) {
result.setDsCheckIntervalType(param.getParamValue());
}
}
}
return result;
@ -109,6 +116,7 @@ public class SystemParameterService {
@Transactional
public CasSaveResult editBasic(List<SystemParameter> parameters) {
CasSaveResult casSaveResult = afterSwitchDefaultLogin(parameters);
BasicInfo basicInfo = basicInfo();
for (int i = 0; i < parameters.size(); i++) {
SystemParameter parameter = parameters.get(i);
SystemParameterExample example = new SystemParameterExample();
@ -121,9 +129,11 @@ public class SystemParameterService {
}
example.clear();
}
datasourceService.updateDatasourceStatusJob(basicInfo, parameters);
return casSaveResult;
}
@Transactional
public void resetCas() {
Map<String, CasXpackService> beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((CasXpackService.class));
@ -188,7 +198,6 @@ public class SystemParameterService {
}
public String getVersion() {
return System.getenv("MS_VERSION");
}
@ -302,10 +311,11 @@ public class SystemParameterService {
}
}
public BasicInfo templateMarketInfo(){
public BasicInfo templateMarketInfo() {
BasicInfo basicInfo = new BasicInfo();
List<SystemParameter> result = this.getParamList("basic.template");
if(CollectionUtils.isNotEmpty(result)){
if (CollectionUtils.isNotEmpty(result)) {
result.stream().forEach(param -> {
if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.TEMPLATE_MARKET_ULR.getValue())) {
basicInfo.setTemplateMarketUlr(param.getParamValue());
@ -315,7 +325,7 @@ public class SystemParameterService {
}
});
}
if(StringUtils.isEmpty(basicInfo.getTemplateMarketUlr())|| StringUtils.isEmpty(basicInfo.getTemplateAccessKey())){
if (StringUtils.isEmpty(basicInfo.getTemplateMarketUlr()) || StringUtils.isEmpty(basicInfo.getTemplateAccessKey())) {
DataEaseException.throwException("Please check market setting info");
}
return basicInfo;

17
backend/src/main/resources/db/migration/V40__1.15.sql
View File

@ -8,3 +8,20 @@ CREATE TABLE `sys_external_token` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;
UPDATE `sys_menu` set `component` = 'system/datasource/DsForm' where `component` = 'system/datasource/form';
INSERT INTO `system_parameter`(`param_key`, `param_value`, `type`, `sort`) VALUES ('basic.dsCheckInterval', 20, 'text', 1);
INSERT INTO `system_parameter`(`param_key`, `param_value`, `type`, `sort`) VALUES ('basic.dsCheckIntervalType', 'minute', 'text', 1);
CREATE TABLE `task_instance` (
`task_id` VARCHAR(128) NOT NULL COMMENT '任务ID',
`execute_time` bigint(13) DEFAULT NULL COMMENT '执行时间',
`finish_time` bigint(13) DEFAULT NULL COMMENT '完成时间',
`status` VARCHAR(128) DEFAULT NULL COMMENT '状态',
`info` longtext COMMENT '执行信息',
`qrtz_instance` VARCHAR(128) DEFAULT NULL COMMENT '任务实例ID',
PRIMARY KEY (`task_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;
INSERT INTO `task_instance` (`task_id`) VALUES ('Datasource_check_status');

1
frontend/src/lang/en.js
View File

@ -794,6 +794,7 @@ export default {
no_more_than: 'Size no more than',
request_timeout: 'Request timeout',
message_retention_time: 'Message retention time',
ds_check_time: 'Data source detection interval',
test_mail_recipient: 'Used only as a test mail recipient',
to_enable_tsl: 'If the SMTP port is 587, you usually need to enable TSL',
to_enable_ssl: 'If the SMTP port is 465, you usually need to enable SSL',

1
frontend/src/lang/tw.js
View File

@ -794,6 +794,7 @@ export default {
no_more_than: '大小不超過',
request_timeout: '請求超時時間',
message_retention_time: '消息保留時間',
ds_check_time: '数据源檢測時間間隔',
test_mail_recipient: '僅用來作為測試郵件收件人',
to_enable_tsl: '如果SMTP埠是587，通常需要啟用TSL',
to_enable_ssl: '如果SMTP埠是465，通常需要啟用SSL',

1
frontend/src/lang/zh.js
View File

@ -794,6 +794,7 @@ export default {
no_more_than: '大小不超过',
request_timeout: '请求超时时间',
message_retention_time: '消息保留时间',
ds_check_time: '数据源检测时间间隔',
test_mail_recipient: '仅用来作为测试邮件收件人',
to_enable_tsl: '如果SMTP端口是587，通常需要启用TSL',
to_enable_ssl: '如果SMTP端口是465，通常需要启用SSL',

55
frontend/src/views/system/SysParam/BasicSetting.vue
View File

@ -31,6 +31,22 @@
slot="append">{{ $t('components.day') }}</template></el-input>
</el-form-item>
<el-form-item :label="$t('system_parameter_setting.ds_check_time')" >
<el-form :inline="true" :disabled="show">
<el-form-item >
<el-input v-model="formInline.dsCheckInterval" type="number" min="1" @change="onSimpleCronChange()" />
</el-form-item>
<el-form-item class="form-item">
<el-select v-model="formInline.dsCheckIntervalType" filterable size="mini" @change="onSimpleCronChange()">
<el-option :label="$t('cron.minute_default')" value="minute" />
<el-option :label="$t('cron.hour_default')" value="hour" />
</el-select>
</el-form-item>
<el-form-item class="form-item" :label="$t('cron.every_exec')" />
</el-form>
</el-form-item>
<el-form-item v-if="loginTypes.length > 1" :label="$t('system_parameter_setting.login_type')" prop="loginType">
<el-radio-group v-model="formInline.loginType">
<el-radio :label="0" size="mini">{{
@ -175,6 +191,18 @@ export default {
type: "text",
sort: 3,
},
{
paramKey: "basic.dsCheckInterval",
paramValue: this.formInline.dsCheckInterval,
type: "text",
sort: 4,
},
{
paramKey: "basic.dsCheckIntervalType",
paramValue: this.formInline.dsCheckIntervalType,
type: "text",
sort: 5,
},
{
paramKey: "ui.openHomePage",
paramValue: this.formInline.openHomePage,
@ -234,7 +262,7 @@ export default {
window.location.reload();
} else {
this.openMessageSuccess("commons.save_failed", 'error');
}
}
});
},
cancel() {
@ -244,6 +272,29 @@ export default {
this.show = true;
this.query();
},
onSimpleCronChange() {
if (this.formInline.dsCheckIntervalType === 'minute') {
if (this.formInline.dsCheckInterval < 1 || this.formInline.dsCheckInterval > 59) {
this.$message({ message: this.$t('cron.minute_limit'), type: 'warning', showClose: true })
this.taskForm.extraData.simple_cron_value = 59
}
return
}
if (this.formInline.dsCheckIntervalType === 'hour') {
if (this.formInline.dsCheckInterval < 1 || this.formInline.dsCheckInterval > 23) {
this.$message({ message: this.$t('cron.hour_limit'), type: 'warning', showClose: true })
this.taskForm.extraData.simple_cron_value = 23
}
return
}
if (this.formInline.dsCheckIntervalType === 'day') {
if (this.formInline.dsCheckInterval < 1 || this.formInline.dsCheckInterval > 31) {
this.$message({ message: this.$t('cron.day_limit'), type: 'warning', showClose: true })
this.taskForm.extraData.simple_cron_value = 31
}
return
}
},
},
};
</script>
@ -273,4 +324,4 @@ export default {
margin-left: 2px;
}
}
</style>
</style>