diff --git a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java index f2f84c1b5b..6a32dff31a 100644 --- a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java +++ b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java @@ -68,7 +68,8 @@ public class ShiroServiceImpl implements ShiroService { filterChainDefinitionMap.put("/**/*.json", ANON); filterChainDefinitionMap.put("/system/ui/**", ANON); - filterChainDefinitionMap.put("/system/file/**", ANON); + filterChainDefinitionMap.put("/system/filedown/**", ANON); + filterChainDefinitionMap.put("/system/showpicture/**", ANON); filterChainDefinitionMap.put("/**/*.js", ANON); filterChainDefinitionMap.put("/**/*.css", ANON); filterChainDefinitionMap.put("/**/*.map", ANON); diff --git a/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java b/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java index 6b5c6aacd0..6eef4b8d2c 100644 --- a/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java +++ b/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java @@ -110,6 +110,8 @@ public interface ParamConstants { enum BASIC implements ParamConstants { FRONT_TIME_OUT("basic.frontTimeOut"), MSG_TIME_OUT("basic.msgTimeOut"), + DS_CHECK_INTERVAL("basic.dsCheckInterval"), + DS_CHECK_INTERVAL_TYPE("basic.dsCheckIntervalType"), DEFAULT_LOGIN_TYPE("basic.loginType"), OPEN_HOME_PAGE("ui.openHomePage"), diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index b01b6bea37..aca169acb0 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -70,11 +70,11 @@ public class DataSetTableController { @ApiOperation("修改") @PostMapping("alter") @DeLog( - operatetype = SysLogConstants.OPERATE_TYPE.MODIFY, - sourcetype = SysLogConstants.SOURCE_TYPE.DATASET, - value = "id", - positionIndex = 0, - positionKey = "sceneId" + operatetype = SysLogConstants.OPERATE_TYPE.MODIFY, + sourcetype = SysLogConstants.SOURCE_TYPE.DATASET, + value = "id", + positionIndex = 0, + positionKey = "sceneId" ) public void alter(@RequestBody DataSetTableRequest request) throws Exception { dataSetTableService.alter(request); @@ -135,6 +135,16 @@ public class DataSetTableController { return dataSetTableService.getPreviewData(dataSetTableRequest, page, pageSize, null); } + @ApiOperation("db数据库表预览数据") + @PostMapping("dbPreview") + @DePermissions(value = { + @DePermission(type = DePermissionType.DATASET, value = "id", level = ResourceAuthLevel.DATASET_LEVEL_USE), + @DePermission(type = DePermissionType.DATASOURCE, value = "dataSourceId", level = ResourceAuthLevel.DATASOURCE_LEVEL_USE) + }, logical = Logical.AND) + public Map getDBPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { + return dataSetTableService.getDBPreview(dataSetTableRequest); + } + @ApiOperation("根据sql查询预览数据") @PostMapping("sqlPreview") @DePermissions(value = { @@ -212,7 +222,13 @@ public class DataSetTableController { @ApiOperation("根据仪表板视图ID查询数据集变量") @PostMapping("/paramsWithIds/{type}") - List paramsWithIds(@PathVariable String type, @RequestBody List viewIds){ + List paramsWithIds(@PathVariable String type, @RequestBody List viewIds) { return dataSetTableService.paramsWithIds(type, viewIds); - }; + } + + @ApiOperation("根据数据集文件夹ID查询数据集名称") + @PostMapping("/getDatasetNameFromGroup/{sceneId}") + public List getDatasetNameFromGroup(@PathVariable String sceneId) { + return dataSetTableService.getDatasetNameFromGroup(sceneId); + } } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java index 14944e9f4e..00910fd834 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskController.java @@ -4,7 +4,6 @@ import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; import com.github.xiaoymin.knife4j.annotations.ApiSupport; import io.dataease.auth.annotation.DePermission; -import io.dataease.plugins.common.base.domain.DatasetTableTask; import io.dataease.commons.constants.DePermissionType; import io.dataease.commons.constants.ResourceAuthLevel; import io.dataease.commons.utils.PageUtils; @@ -12,6 +11,7 @@ import io.dataease.commons.utils.Pager; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.dto.dataset.DataSetTaskDTO; +import io.dataease.plugins.common.base.domain.DatasetTableTask; import io.dataease.service.dataset.DataSetTableTaskLogService; import io.dataease.service.dataset.DataSetTableTaskService; import io.swagger.annotations.Api; @@ -62,6 +62,14 @@ public class DataSetTableTaskController { return dataSetTableTaskService.list(datasetTableTask); } + @DePermission(type = DePermissionType.DATASET, value = "tableId", level = ResourceAuthLevel.DATASET_LEVEL_MANAGE) + @ApiOperation("分页查询") + @PostMapping("list/{goPage}/{pageSize}") + public Pager> list(@RequestBody DatasetTableTask datasetTableTask, @PathVariable int goPage, @PathVariable int pageSize) { + Page page = PageHelper.startPage(goPage, pageSize, true); + return PageUtils.setPageInfo(page, dataSetTableTaskService.list(datasetTableTask)); + } + @ApiOperation("分页查询") @PostMapping("/pageList/{goPage}/{pageSize}") public Pager> taskList(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody BaseGridRequest request) { diff --git a/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java b/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java index 144f12bdd9..63543ee308 100644 --- a/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java +++ b/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java @@ -1,6 +1,6 @@ package io.dataease.controller.sys; -import io.dataease.plugins.common.base.domain.FileMetadata; + import io.dataease.plugins.common.base.domain.SystemParameter; import io.dataease.commons.constants.ParamConstants; import io.dataease.controller.sys.response.BasicInfo; @@ -8,6 +8,7 @@ import io.dataease.controller.sys.response.MailInfo; import io.dataease.dto.SystemParameterDTO; import io.dataease.listener.DatasetCheckListener; import io.dataease.listener.util.CacheUtils; +import io.dataease.plugins.common.util.GlobalFileUtil; import io.dataease.plugins.xpack.cas.dto.CasSaveResult; import io.dataease.service.FileService; import io.dataease.service.system.EmailService; @@ -22,7 +23,7 @@ import springfox.documentation.annotations.ApiIgnore; import javax.annotation.Resource; import java.io.IOException; -import java.net.URLEncoder; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -108,21 +109,16 @@ public class SystemParameterController { return new ResponseEntity<>(bytes, headers, HttpStatus.OK); } - @GetMapping("/file/down/{fileId}/{fileName}") - public ResponseEntity down(@PathVariable("fileId") String fileId, @PathVariable("fileName") String fileName) throws Exception{ + @GetMapping("/filedown/{fileId}/{fileName}") + public ResponseEntity down(@PathVariable("fileId") String fileId, @PathVariable("fileName") String fileName) throws Exception { - FileMetadata fileMetadata = fileService.getFileMetadataById(fileId); - String type = fileMetadata.getType(); - if (!StringUtils.endsWith(fileName.toUpperCase(), type.toUpperCase())) { - fileName += ("." + type); - } - byte[] bytes = fileService.loadFileAsBytes(fileId); - ByteArrayResource bar = new ByteArrayResource(bytes); - final HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - ContentDisposition contentDisposition = ContentDisposition.parse("attachment; filename=" + URLEncoder.encode(fileName, "UTF-8")); - headers.setContentDisposition(contentDisposition); - return new ResponseEntity<>(bar, headers, HttpStatus.OK); + return GlobalFileUtil.down(fileId, fileName); + } + + @GetMapping(value = "/showpicture/{fileId}", produces = {MediaType.IMAGE_JPEG_VALUE, MediaType.IMAGE_PNG_VALUE}) + public ResponseEntity showPicture(@PathVariable("fileId") String fileId) throws Exception { + + return GlobalFileUtil.showPicture(fileId); } @PostMapping(value = "/save/ui", consumes = {"multipart/form-data"}) diff --git a/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java b/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java index 4a4ca4f3b8..7038a03406 100644 --- a/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java +++ b/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java @@ -22,5 +22,9 @@ public class BasicInfo implements Serializable { private String templateAccessKey; @ApiModelProperty("显示模板市场") private String openMarketPage; + @ApiModelProperty("数据源检测时间间隔") + private String dsCheckInterval; + @ApiModelProperty("数据源检测时间间隔类型") + private String dsCheckIntervalType; } diff --git a/backend/src/main/java/io/dataease/dto/TaskInstance.java b/backend/src/main/java/io/dataease/dto/TaskInstance.java new file mode 100644 index 0000000000..a17aa68c1f --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/TaskInstance.java @@ -0,0 +1,22 @@ +package io.dataease.dto; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class TaskInstance implements Serializable{ + + private String taskId; + + private Long executeTime; + + private Long finishTime; + + private String status; + + private String info; + + private String qrtzInstance; + +} diff --git a/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.java b/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.java new file mode 100644 index 0000000000..858dcc781d --- /dev/null +++ b/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.java @@ -0,0 +1,17 @@ +package io.dataease.ext; + +import io.dataease.dto.TaskInstance; + +import java.util.List; + +public interface ExtTaskInstanceMapper { + + int runningCount(String taskId); + + void resetRunnings(String taskId); + + void update(TaskInstance taskInstance); + + List select(); + +} diff --git a/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.xml b/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.xml new file mode 100644 index 0000000000..d099bea160 --- /dev/null +++ b/backend/src/main/java/io/dataease/ext/ExtTaskInstanceMapper.xml @@ -0,0 +1,28 @@ + + + + + + + + update task_instance set status = 'ERROR', info = 'System Interrupt Error' where task_id = #{taskId} and 'RUNNING' + + + + update task_instance + set `task_id` = #{taskId,jdbcType=VARCHAR}, + `execute_time` = #{executeTime,jdbcType=BIGINT}, + `finish_time` = #{finishTime,jdbcType=BIGINT}, + status = #{status,jdbcType=VARCHAR}, + info = #{info,jdbcType=VARCHAR}, + qrtz_instance = #{qrtzInstance,jdbcType=VARCHAR} + where task_id = #{taskId,jdbcType=VARCHAR} + + + + + diff --git a/backend/src/main/java/io/dataease/job/sechedule/Schedular.java b/backend/src/main/java/io/dataease/job/sechedule/Schedular.java index 6a1693e6ff..c6575cc218 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/Schedular.java +++ b/backend/src/main/java/io/dataease/job/sechedule/Schedular.java @@ -22,9 +22,9 @@ public class Schedular { dataSetTableService.updateDatasetTableStatus(); } - @QuartzScheduled(cron = "0 0/30 * * * ?") + @QuartzScheduled(cron = "0 0/3 * * * ?") public void updateDatasourceStatus() { - datasourceService.updateDatasourceStatus(); + datasourceService.checkDatasourceJob(); } @QuartzScheduled(cron = "0 0/30 * * * ?") diff --git a/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/DsTaskHandler.java b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/DsTaskHandler.java new file mode 100644 index 0000000000..06da9e8228 --- /dev/null +++ b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/DsTaskHandler.java @@ -0,0 +1,98 @@ +package io.dataease.job.sechedule.strategy.impl; + +import com.google.gson.Gson; +import io.dataease.commons.utils.CommonBeanFactory; +import io.dataease.commons.utils.LogUtil; +import io.dataease.dto.TaskInstance; +import io.dataease.ext.ExtTaskInstanceMapper; +import io.dataease.job.sechedule.ScheduleManager; +import io.dataease.job.sechedule.strategy.TaskHandler; +import io.dataease.plugins.common.entity.GlobalTaskEntity; + +import io.dataease.service.datasource.DatasourceService; +import org.quartz.*; + +import org.springframework.stereotype.Service; + +import java.util.Date; + + +@Service("dsTaskHandler") +public class DsTaskHandler extends TaskHandler implements Job { + + private static final String RUNNING = "RUNNING"; + private static final String SUCCESS = "SUCCESS"; + private static final String ERROR = "ERROR"; + + @Override + protected JobDataMap jobDataMap(GlobalTaskEntity taskEntity) { + JobDataMap jobDataMap = new JobDataMap(); + jobDataMap.put("taskEntity", taskEntity); + return jobDataMap; + } + + @Override + public void resetRunningInstance(Long taskId) { + + } + + @Override + protected Boolean taskIsRunning(Long taskId) { + return null; + } + + @Override + public void addTask(ScheduleManager scheduleManager, GlobalTaskEntity taskEntity) throws Exception { + JobKey jobKey = new JobKey(taskEntity.getJobKey()); + TriggerKey triggerKey = new TriggerKey(taskEntity.getJobKey()); + Date start = new Date(taskEntity.getStartTime()); + Date end = null; + Class executor = this.getClass(); + scheduleManager.addOrUpdateCronJob(jobKey, triggerKey, executor, taskEntity.getCron(), start, end, jobDataMap(taskEntity)); + } + + @Override + public void removeTask(ScheduleManager scheduleManager, GlobalTaskEntity taskEntity) { + JobKey jobKey = new JobKey(taskEntity.getJobKey()); + TriggerKey triggerKey = new TriggerKey(taskEntity.getJobKey()); + scheduleManager.removeJob(jobKey, triggerKey); + } + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + JobDataMap jobDataMap = context.getJobDetail().getJobDataMap(); + GlobalTaskEntity taskEntity = (GlobalTaskEntity) jobDataMap.get("taskEntity"); + taskEntity.getJobKey(); + + if (isRunning(taskEntity.getJobKey())) { + LogUtil.info("Skip synchronization task: {} ,due to task status is {}", taskEntity.getJobKey(), "running"); + return; + } + + LogUtil.info("start check datasource status..."); + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setTaskId("Datasource_check_status"); + taskInstance.setExecuteTime(System.currentTimeMillis()); + taskInstance.setFinishTime(null); + taskInstance.setStatus(RUNNING); + taskInstance.setQrtzInstance(context.getFireInstanceId()); + ExtTaskInstanceMapper extTaskInstanceMapper = CommonBeanFactory.getBean(ExtTaskInstanceMapper.class); + extTaskInstanceMapper.update(taskInstance); + + DatasourceService datasourceService = CommonBeanFactory.getBean(DatasourceService.class); + datasourceService.updateDatasourceStatus(); + + taskInstance.setFinishTime(System.currentTimeMillis()); + taskInstance.setStatus(SUCCESS); + extTaskInstanceMapper.update(taskInstance); + LogUtil.info("end check datasource status."); + + } + + + private Boolean isRunning(String taskId) { + ExtTaskInstanceMapper extTaskInstanceMapper = CommonBeanFactory.getBean(ExtTaskInstanceMapper.class); + return extTaskInstanceMapper.runningCount(taskId) > 0; + } + +} diff --git a/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java index 32cac1edc0..6b7becc484 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java +++ b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java @@ -5,6 +5,7 @@ import io.dataease.auth.entity.TokenInfo; import io.dataease.auth.service.AuthUserService; import io.dataease.auth.service.impl.AuthUserServiceImpl; import io.dataease.auth.util.JWTUtils; +import io.dataease.commons.exception.DEException; import io.dataease.dto.PermissionProxy; import io.dataease.ext.ExtTaskMapper; import io.dataease.commons.utils.CommonBeanFactory; @@ -16,10 +17,16 @@ import io.dataease.job.sechedule.strategy.TaskHandler; import io.dataease.plugins.common.entity.GlobalTaskEntity; import io.dataease.plugins.common.entity.GlobalTaskInstance; import io.dataease.plugins.config.SpringContextUtil; +import io.dataease.plugins.xpack.dingtalk.dto.entity.DingtalkMsgResult; +import io.dataease.plugins.xpack.dingtalk.service.DingtalkXpackService; import io.dataease.plugins.xpack.email.dto.request.XpackEmailTaskRequest; import io.dataease.plugins.xpack.email.dto.request.XpackPixelEntity; import io.dataease.plugins.xpack.email.dto.response.XpackEmailTemplateDTO; import io.dataease.plugins.xpack.email.service.EmailXpackService; +import io.dataease.plugins.xpack.lark.dto.entity.LarkMsgResult; +import io.dataease.plugins.xpack.lark.service.LarkXpackService; +import io.dataease.plugins.xpack.wecom.dto.entity.WecomMsgResult; +import io.dataease.plugins.xpack.wecom.service.WecomXpackService; import io.dataease.service.chart.ViewExportExcel; import io.dataease.service.system.EmailService; import org.apache.commons.lang3.ObjectUtils; @@ -30,6 +37,7 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -176,9 +184,54 @@ public class EmailTaskHandler extends TaskHandler implements Job { proxy.setUserId(user.getUserId()); files = viewExportExcel.export(panelId, viewIdList, proxy); } - - emailService.sendWithImageAndFiles(recipients, emailTemplateDTO.getTitle(), contentStr, bytes, files); + List channels = null; + String recisetting = emailTemplateDTO.getRecisetting(); + if (StringUtils.isBlank(recisetting)) { + channels = new ArrayList<>(); + channels.add("email"); + } else { + channels = Arrays.stream(recisetting.split(",")).collect(Collectors.toList()); + } + + List reciLists = Arrays.stream(recipients.split(",")).collect(Collectors.toList()); + for (int i = 0; i < channels.size(); i++) { + String channel = channels.get(i); + switch (channel) { + case "email" : + emailService.sendWithImageAndFiles(recipients, emailTemplateDTO.getTitle(), contentStr, bytes, files); + break; + case "wecom" : + if (SpringContextUtil.getBean(AuthUserService.class).supportWecom()) { + WecomXpackService wecomXpackService = SpringContextUtil.getBean(WecomXpackService.class); + WecomMsgResult wecomMsgResult = wecomXpackService.pushOaMsg(reciLists, emailTemplateDTO.getTitle(), contentStr, bytes, files); + if (wecomMsgResult.getErrcode() != 0) { + DEException.throwException(wecomMsgResult.getErrmsg()); + } + } + break; + case "dingtalk" : + if (SpringContextUtil.getBean(AuthUserService.class).supportDingtalk()) { + DingtalkXpackService dingtalkXpackService = SpringContextUtil.getBean(DingtalkXpackService.class); + DingtalkMsgResult dingtalkMsgResult = dingtalkXpackService.pushOaMsg(reciLists, emailTemplateDTO.getTitle(), contentStr, bytes, files); + if (dingtalkMsgResult.getErrcode() != 0) { + DEException.throwException(dingtalkMsgResult.getErrmsg()); + } + } + break; + case "lark" : + if (SpringContextUtil.getBean(AuthUserService.class).supportLark()) { + LarkXpackService larkXpackService = SpringContextUtil.getBean(LarkXpackService.class); + LarkMsgResult larkMsgResult = larkXpackService.pushOaMsg(reciLists, emailTemplateDTO.getTitle(), contentStr, bytes, files); + if (larkMsgResult.getCode() != 0) { + DEException.throwException(larkMsgResult.getMsg()); + } + } + break; + default: + break; + } + } success(taskInstance); } catch (Exception e) { error(taskInstance, e); @@ -186,6 +239,7 @@ public class EmailTaskHandler extends TaskHandler implements Job { } } + private XpackPixelEntity buildPixel(XpackEmailTemplateDTO emailTemplateDTO) { XpackPixelEntity pixelEntity = new XpackPixelEntity(); String pixelStr = emailTemplateDTO.getPixel(); diff --git a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java index d9c360d1a8..bc53c1b879 100644 --- a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java +++ b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java @@ -22,8 +22,10 @@ public class DataSourceInitStartListener implements ApplicationListener sqlVariables = new Gson().fromJson(table.getSqlVariableDetails(), new TypeToken>() {}.getType()); + List sqlVariables = new Gson().fromJson(table.getSqlVariableDetails(), new TypeToken>() { + }.getType()); for (String parameter : Optional.ofNullable(request.getParameters()).orElse(new ArrayList<>())) { if (sqlVariables.stream().map(SqlVariableDetails::getVariableName).collect(Collectors.toList()).contains(parameter)) { hasParameters = true; @@ -816,7 +822,7 @@ public class ChartViewService { || StringUtils.containsIgnoreCase(view.getType(), "scatter") || StringUtils.containsIgnoreCase(view.getType(), "mix") ) { - assistFields = getAssistFields(dynamicAssistFields); + assistFields = getAssistFields(dynamicAssistFields, yAxis); } // 如果是插件视图 走插件内部的逻辑 @@ -866,9 +872,6 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQL(dataTableInfoDTO.getTable(), xAxis, yAxis, fieldCustomFilter, rowPermissionsTree, extFilterList, ds, view)); } - if (CollectionUtils.isNotEmpty(assistFields)) { - datasourceAssistRequest.setQuery(qp.getSQLSummary(dataTableInfoDTO.getTable(), assistFields, null, null, null, view, ds)); - } } else if (StringUtils.equalsIgnoreCase(table.getType(), DatasetType.SQL.name())) { String sql = dataTableInfoDTO.isBase64Encryption() ? new String(java.util.Base64.getDecoder().decode(dataTableInfoDTO.getSql())) : dataTableInfoDTO.getSql(); sql = handleVariable(sql, requestList, qp, table); @@ -883,9 +886,6 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQLAsTmp(sql, xAxis, yAxis, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); } - if (CollectionUtils.isNotEmpty(assistFields)) { - datasourceAssistRequest.setQuery(qp.getSQLSummaryAsTmp(sql, assistFields, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); - } } else if (StringUtils.equalsIgnoreCase(table.getType(), DatasetType.CUSTOM.name())) { DataTableInfoDTO dt = gson.fromJson(table.getInfo(), DataTableInfoDTO.class); List list = dataSetTableUnionService.listByTableId(dt.getList().get(0).getTableId()); @@ -901,9 +901,6 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQLAsTmp(sql, xAxis, yAxis, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); } - if (CollectionUtils.isNotEmpty(assistFields)) { - datasourceAssistRequest.setQuery(qp.getSQLSummaryAsTmp(sql, assistFields, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); - } } else if (StringUtils.equalsIgnoreCase(table.getType(), DatasetType.UNION.name())) { DataTableInfoDTO dt = gson.fromJson(table.getInfo(), DataTableInfoDTO.class); Map sqlMap = dataSetTableService.getUnionSQLDatasource(dt, ds); @@ -920,12 +917,11 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQLAsTmp(sql, xAxis, yAxis, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); } - if (CollectionUtils.isNotEmpty(assistFields)) { - datasourceAssistRequest.setQuery(qp.getSQLSummaryAsTmp(sql, assistFields, fieldCustomFilter, rowPermissionsTree, extFilterList, view)); - } } data = datasourceProvider.getData(datasourceRequest); if (CollectionUtils.isNotEmpty(assistFields)) { + datasourceAssistRequest.setQuery(assistSQL(datasourceRequest.getQuery(), assistFields)); + logger.info(datasourceAssistRequest.getQuery()); assistData = datasourceProvider.getData(datasourceAssistRequest); } } else if (table.getMode() == 1) {// 抽取 @@ -946,7 +942,8 @@ public class ChartViewService { datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, fieldCustomFilter, rowPermissionsTree, extFilterList, ds, view)); } if (CollectionUtils.isNotEmpty(assistFields)) { - datasourceAssistRequest.setQuery(qp.getSQLSummary(tableName, assistFields, fieldCustomFilter, rowPermissionsTree, extFilterList, view, ds)); + datasourceAssistRequest.setQuery(assistSQL(datasourceRequest.getQuery(), assistFields)); + logger.info(datasourceAssistRequest.getQuery()); assistData = datasourceProvider.getData(datasourceAssistRequest); } // 仪表板有参数不使用缓存 @@ -1143,6 +1140,19 @@ public class ChartViewService { return res; } + public String assistSQL(String sql, List assistFields) { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < assistFields.size(); i++) { + ChartViewFieldDTO dto = assistFields.get(i); + if (i == (assistFields.size() - 1)) { + stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + ")"); + } else { + stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + "),"); + } + } + return "SELECT " + stringBuilder + " FROM (" + sql + ") tmp"; + } + public ChartViewDTO uniteViewResult(String sql, Map chartData, Map tabelData, ChartViewDTO view, Boolean isDrill, List drillFilters, List dynamicAssistFields, List assistData) { Map map = new HashMap<>(); @@ -1563,7 +1573,8 @@ public class ChartViewService { private String handleVariable(String sql, ChartExtRequest requestList, QueryProvider qp, DataSetTableDTO table) throws Exception { - List sqlVariables = new Gson().fromJson(table.getSqlVariableDetails(), new TypeToken>() {}.getType()); + List sqlVariables = new Gson().fromJson(table.getSqlVariableDetails(), new TypeToken>() { + }.getType()); if (requestList != null && CollectionUtils.isNotEmpty(requestList.getFilter())) { for (ChartExtFilterRequest chartExtFilterRequest : requestList.getFilter()) { @@ -1577,7 +1588,7 @@ public class ChartViewService { for (String parameter : chartExtFilterRequest.getParameters()) { List parameters = sqlVariables.stream().filter(item -> item.getVariableName().equalsIgnoreCase(parameter)).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(parameters)) { - String filter = qp.transFilter(chartExtFilterRequest,parameters.get(0)); + String filter = qp.transFilter(chartExtFilterRequest, parameters.get(0)); sql = sql.replace("${" + parameter + "}", filter); } @@ -1658,10 +1669,14 @@ public class ChartViewService { String senior = view.getSenior(); JSONObject jsonObject = JSONObject.parseObject(senior); JSONArray assistLine = jsonObject.getJSONArray("assistLine"); + List list = new ArrayList<>(); + if (ObjectUtils.isEmpty(assistLine) || StringUtils.isBlank(assistLine.toJSONString())) { + return list; + } List assistLines = gson.fromJson(assistLine.toJSONString(), new TypeToken>() { }.getType()); - List list = new ArrayList<>(); + for (ChartSeniorAssistDTO dto : assistLines) { if (StringUtils.equalsIgnoreCase(dto.getField(), "0")) { continue; @@ -1683,13 +1698,28 @@ public class ChartViewService { return list; } - private List getAssistFields(List list) { + private List getAssistFields(List list, List yAxis) { List res = new ArrayList<>(); for (ChartSeniorAssistDTO dto : list) { DatasetTableField curField = dto.getCurField(); + ChartViewFieldDTO yField = null; + String alias = ""; + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO field = yAxis.get(i); + if (StringUtils.equalsIgnoreCase(field.getId(), curField.getId())) { + yField = field; + alias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + break; + } + } + if (ObjectUtils.isEmpty(yField)) { + continue; + } + ChartViewFieldDTO chartViewFieldDTO = new ChartViewFieldDTO(); BeanUtils.copyBean(chartViewFieldDTO, curField); chartViewFieldDTO.setSummary(dto.getSummary()); + chartViewFieldDTO.setOriginName(alias);// yAxis的字段别名,就是查找的字段名 res.add(chartViewFieldDTO); } return res; @@ -1701,8 +1731,10 @@ public class ChartViewService { } String[] strings = assistData.get(0); for (int i = 0; i < dynamicAssistFields.size(); i++) { - ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFields.get(i); - chartSeniorAssistDTO.setValue(strings[i]); + if (i < strings.length) { + ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFields.get(i); + chartSeniorAssistDTO.setValue(strings[i]); + } } } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java index fe058b6184..28ff78c646 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java @@ -46,7 +46,7 @@ public class DataSetGroupService { @Resource private SysAuthService sysAuthService; -// @DeCleaner(value = DePermissionType.DATASET, key = "pid") + @DeCleaner(value = DePermissionType.DATASET, key = "pid") public DataSetGroupDTO save(DatasetGroup datasetGroup) throws Exception { checkName(datasetGroup); if (StringUtils.isEmpty(datasetGroup.getId())) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index ebc68f3e43..b0503f299c 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -153,6 +153,8 @@ public class DataSetTableService { @DeCleaner(value = DePermissionType.DATASET, key = "sceneId") public void batchInsert(List datasetTable) throws Exception { + // 保存之前校验table名称 + checkNames(datasetTable); for (DataSetTableRequest table : datasetTable) { save(table); // 清理权限缓存 @@ -382,6 +384,13 @@ public class DataSetTableService { } } + public List getDatasetNameFromGroup(String sceneId) { + DatasetTableExample example = new DatasetTableExample(); + example.createCriteria().andSceneIdEqualTo(sceneId); + List datasetTables = datasetTableMapper.selectByExample(example); + return datasetTables.stream().map(DatasetTable::getName).collect(Collectors.toList()); + } + public List list(DataSetTableRequest dataSetTableRequest) { dataSetTableRequest.setUserId(String.valueOf(AuthUtils.getUser().getUserId())); dataSetTableRequest.setTypeFilter(dataSetTableRequest.getTypeFilter()); @@ -936,7 +945,7 @@ public class DataSetTableService { return new ArrayList<>(); } - if(!Arrays.asList("DATE", "TEXT", "NUM").contains(type)){ + if (!Arrays.asList("DATE", "TEXT", "NUM").contains(type)) { return new ArrayList<>(); } ChartViewExample chartViewExample = new ChartViewExample(); @@ -965,24 +974,28 @@ public class DataSetTableService { } } } - switch (type){ + switch (type) { case "DATE": - sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("DATETIME")).collect(Collectors.toList()); + sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("DATETIME")).collect(Collectors.toList()); sqlVariableDetails.forEach(item -> { - if(item.getType().size()> 1){ + if (item.getType().size() > 1) { item.setAlias(item.getVariableName() + "[" + item.getType().get(1) + "]"); - }else { + } else { item.setAlias(item.getVariableName()); } }); break; case "TEXT": - sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("TEXT")).collect(Collectors.toList()); - sqlVariableDetails.forEach(item -> {item.setAlias(item.getVariableName());}); + sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("TEXT")).collect(Collectors.toList()); + sqlVariableDetails.forEach(item -> { + item.setAlias(item.getVariableName()); + }); break; case "NUM": - sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("LONG") || item.getType().get(0).contains("DOUBLE")).collect(Collectors.toList()); - sqlVariableDetails.forEach(item -> {item.setAlias(item.getVariableName());}); + sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("LONG") || item.getType().get(0).contains("DOUBLE")).collect(Collectors.toList()); + sqlVariableDetails.forEach(item -> { + item.setAlias(item.getVariableName()); + }); break; } return sqlVariableDetails; @@ -1072,7 +1085,7 @@ public class DataSetTableService { return handleWith(plainSelect, select); } - private String handleWith(PlainSelect plainSelect, Select select)throws Exception{ + private String handleWith(PlainSelect plainSelect, Select select) throws Exception { StringBuilder builder = new StringBuilder(); if (CollectionUtils.isNotEmpty(select.getWithItemsList())) { builder.append("WITH"); @@ -1089,6 +1102,45 @@ public class DataSetTableService { builder.append(" " + plainSelect); return builder.toString(); } + + public Map getDBPreview(DataSetTableRequest dataSetTableRequest) throws Exception { + Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); + if (ds == null) { + throw new Exception(Translator.get("i18n_invalid_ds")); + } + Provider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + DataTableInfoDTO dataTableInfo = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + String sql = "SELECT * FROM " + dataTableInfo.getTable(); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + String sqlAsTable = qp.createSQLPreview(sql, null); + datasourceRequest.setQuery(sqlAsTable); + Map result = datasourceProvider.fetchResultAndField(datasourceRequest); + List data = result.get("dataList"); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + List> jsonArray = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.length; i++) { + map.put(fieldArray[i], ele[i]); + } + return map; + }).collect(Collectors.toList()); + } + + Map map = new HashMap<>(); + map.put("fields", fields); + map.put("data", jsonArray); + + return map; + } + public Map getSQLPreview(DataSetTableRequest dataSetTableRequest) throws Exception { Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); if (ds == null) { @@ -2090,6 +2142,31 @@ public class DataSetTableService { } } + private void checkNames(List datasetTable) { + if (CollectionUtils.isEmpty(datasetTable)) { + return; + } + Set nameSet = new HashSet<>(); + for (DataSetTableRequest table : datasetTable) { + nameSet.add(table.getName()); + } + if (nameSet.size() != datasetTable.size()) { + throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group")); + } + DatasetTableExample datasetTableExample = new DatasetTableExample(); + DatasetTableExample.Criteria criteria = datasetTableExample.createCriteria(); + if (StringUtils.isNotEmpty(datasetTable.get(0).getSceneId())) { + criteria.andSceneIdEqualTo(datasetTable.get(0).getSceneId()); + } + if (CollectionUtils.isNotEmpty(nameSet)) { + criteria.andNameIn(new ArrayList<>(nameSet)); + } + List list = datasetTableMapper.selectByExample(datasetTableExample); + if (list.size() > 0) { + throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group")); + } + } + public DataSetDetail getDatasetDetail(String id) { DataSetDetail dataSetDetail = new DataSetDetail(); DatasetTable table = datasetTableMapper.selectByPrimaryKey(id); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 6cb1581db0..e9fc915b44 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -790,6 +790,7 @@ public class ExtractDataService { Thread.sleep(1000); } if (jobStatus.getStatusDescription().equals("Finished")) { + LogUtil.info(datasetTable.getId()+ ": " + jobStatus.getLoggingString()); return; } else { DataEaseException.throwException(jobStatus.getLoggingString()); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index 41af53e43d..26c94e30a7 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -9,7 +9,10 @@ import io.dataease.auth.annotation.DeCleaner; import io.dataease.commons.constants.RedisConstants; import io.dataease.commons.utils.BeanUtils; import io.dataease.controller.sys.response.BasicInfo; +import io.dataease.dto.TaskInstance; import io.dataease.ext.ExtDataSourceMapper; +import io.dataease.ext.ExtTaskInstanceMapper; +import io.dataease.ext.UtilMapper; import io.dataease.ext.query.GridExample; import io.dataease.commons.constants.DePermissionType; import io.dataease.commons.constants.SysAuthConstants; @@ -31,17 +34,20 @@ import io.dataease.i18n.Translator; import io.dataease.plugins.common.base.domain.*; import io.dataease.plugins.common.base.mapper.DatasetTableMapper; import io.dataease.plugins.common.base.mapper.DatasourceMapper; +import io.dataease.plugins.common.base.mapper.QrtzSchedulerStateMapper; import io.dataease.plugins.common.constants.DatasetType; import io.dataease.plugins.common.constants.DatasourceCalculationMode; import io.dataease.plugins.common.constants.DatasourceTypes; import io.dataease.plugins.common.dto.datasource.DataSourceType; import io.dataease.plugins.common.dto.datasource.TableDesc; +import io.dataease.plugins.common.entity.GlobalTaskEntity; import io.dataease.plugins.common.request.datasource.DatasourceRequest; import io.dataease.plugins.config.SpringContextUtil; import io.dataease.plugins.datasource.entity.JdbcConfiguration; import io.dataease.plugins.datasource.provider.Provider; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.ApiProvider; +import io.dataease.service.ScheduleService; import io.dataease.service.dataset.DataSetGroupService; import io.dataease.service.message.DeMsgutil; import io.dataease.service.sys.SysAuthService; @@ -49,6 +55,7 @@ import io.dataease.service.system.SystemParameterService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -77,6 +84,14 @@ public class DatasourceService { private Environment env; @Resource private SystemParameterService systemParameterService; + @Autowired + private ScheduleService scheduleService; + @Resource + private QrtzSchedulerStateMapper qrtzSchedulerStateMapper; + @Resource + private UtilMapper utilMapper; + @Resource + private ExtTaskInstanceMapper extTaskInstanceMapper; public Collection types() { Collection types = new ArrayList<>(); @@ -432,6 +447,24 @@ public class DatasourceService { } } + public void checkDatasourceJob() { + List qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); + List activeQrtzInstances = qrtzSchedulerStates.stream() + .filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()) + .map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); + + + List taskInstances = extTaskInstanceMapper.select(); + taskInstances.forEach(taskInstance -> { + if (StringUtils.isNotEmpty(taskInstance.getQrtzInstance()) && !activeQrtzInstances.contains(taskInstance.getQrtzInstance().substring(0, taskInstance.getQrtzInstance().length() - 13))) { + TaskInstance update = new TaskInstance(); + update.setTaskId("Datasource_check_status"); + extTaskInstanceMapper.update(update); + } + }); + } + public void updateDatasourceStatus() { List datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample()); datasources.forEach(datasource -> checkAndUpdateDatasourceStatus(datasource, true)); @@ -496,4 +529,57 @@ public class DatasourceService { DeMsgutil.sendMsg(userId, typeId, content, gson.toJson(param)); }); } + + public void updateDatasourceStatusJob(BasicInfo basicInfo, List parameters) { + String type = ""; + Integer interval = 30; + + boolean changeDsCheckTime = false; + basicInfo.getDsCheckInterval(); + basicInfo.getDsCheckIntervalType(); + for (SystemParameter parameter : parameters) { + if (parameter.getParamKey().equalsIgnoreCase("basic.dsCheckInterval") && !parameter.getParamValue().equalsIgnoreCase(basicInfo.getDsCheckInterval())) { + changeDsCheckTime = true; + interval = Integer.valueOf(parameter.getParamValue()); + } + if (parameter.getParamKey().equalsIgnoreCase("basic.dsCheckIntervalType") && !parameter.getParamValue().equalsIgnoreCase(basicInfo.getDsCheckInterval())) { + changeDsCheckTime = true; + type = parameter.getParamValue(); + } + } + if(!changeDsCheckTime){ + return; + } + addJob(type, interval); + } + + private void addJob(String type, Integer interval) { + String cron = ""; + switch (type){ + case "hour": + cron = "0 0 0/hour * * ? *".replace("hour", interval.toString()); + break; + default: + cron = "0 0/minute * * * ? *".replace("minute", interval.toString()); + } + + GlobalTaskEntity globalTask = new GlobalTaskEntity(); + globalTask.setCron(cron); + globalTask.setCreateTime(System.currentTimeMillis()); + globalTask.setJobKey("Datasource_check_status"); + globalTask.setTaskName("Datasource check status"); + globalTask.setTaskType("dsTaskHandler"); + globalTask.setStartTime(System.currentTimeMillis()); + try { + scheduleService.addSchedule(globalTask); + }catch (Exception e){ + e.printStackTrace(); + } + } + + public void initDsCheckJob(){ + BasicInfo basicInfo = systemParameterService.basicInfo(); + addJob(basicInfo.getDsCheckIntervalType(), Integer.valueOf(basicInfo.getDsCheckInterval())); + } + } diff --git a/backend/src/main/java/io/dataease/service/system/SystemParameterService.java b/backend/src/main/java/io/dataease/service/system/SystemParameterService.java index 015858320e..0b70c4ccd1 100644 --- a/backend/src/main/java/io/dataease/service/system/SystemParameterService.java +++ b/backend/src/main/java/io/dataease/service/system/SystemParameterService.java @@ -8,7 +8,6 @@ import io.dataease.controller.sys.response.BasicInfo; import io.dataease.dto.SystemParameterDTO; import io.dataease.exception.DataEaseException; import io.dataease.plugins.common.base.domain.FileMetadata; -import io.dataease.plugins.common.base.domain.SysParamAssist; import io.dataease.plugins.common.base.domain.SystemParameter; import io.dataease.plugins.common.base.domain.SystemParameterExample; import io.dataease.plugins.common.base.mapper.SystemParameterMapper; @@ -17,10 +16,11 @@ import io.dataease.plugins.xpack.cas.dto.CasSaveResult; import io.dataease.plugins.xpack.cas.service.CasXpackService; import io.dataease.plugins.xpack.display.service.DisplayXpackService; import io.dataease.service.FileService; +import io.dataease.service.datasource.DatasourceService; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; @@ -29,7 +29,6 @@ import javax.annotation.Resource; import javax.imageio.ImageIO; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.util.*; import java.util.concurrent.atomic.AtomicReference; @@ -47,6 +46,9 @@ public class SystemParameterService { private ExtSystemParameterMapper extSystemParameterMapper; @Resource private FileService fileService; + @Resource + @Lazy + private DatasourceService datasourceService; public String searchEmail() { return extSystemParameterMapper.email(); @@ -86,7 +88,12 @@ public class SystemParameterService { if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.TEMPLATE_ACCESS_KEY.getValue())) { result.setTemplateAccessKey(param.getParamValue()); } - + if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DS_CHECK_INTERVAL.getValue())) { + result.setDsCheckInterval(param.getParamValue()); + } + if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.DS_CHECK_INTERVAL_TYPE.getValue())) { + result.setDsCheckIntervalType(param.getParamValue()); + } } } return result; @@ -109,6 +116,7 @@ public class SystemParameterService { @Transactional public CasSaveResult editBasic(List parameters) { CasSaveResult casSaveResult = afterSwitchDefaultLogin(parameters); + BasicInfo basicInfo = basicInfo(); for (int i = 0; i < parameters.size(); i++) { SystemParameter parameter = parameters.get(i); SystemParameterExample example = new SystemParameterExample(); @@ -121,9 +129,11 @@ public class SystemParameterService { } example.clear(); } + datasourceService.updateDatasourceStatusJob(basicInfo, parameters); return casSaveResult; } + @Transactional public void resetCas() { Map beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((CasXpackService.class)); @@ -188,7 +198,6 @@ public class SystemParameterService { } - public String getVersion() { return System.getenv("MS_VERSION"); } @@ -302,10 +311,11 @@ public class SystemParameterService { } } - public BasicInfo templateMarketInfo(){ + + public BasicInfo templateMarketInfo() { BasicInfo basicInfo = new BasicInfo(); List result = this.getParamList("basic.template"); - if(CollectionUtils.isNotEmpty(result)){ + if (CollectionUtils.isNotEmpty(result)) { result.stream().forEach(param -> { if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.TEMPLATE_MARKET_ULR.getValue())) { basicInfo.setTemplateMarketUlr(param.getParamValue()); @@ -315,7 +325,7 @@ public class SystemParameterService { } }); } - if(StringUtils.isEmpty(basicInfo.getTemplateMarketUlr())|| StringUtils.isEmpty(basicInfo.getTemplateAccessKey())){ + if (StringUtils.isEmpty(basicInfo.getTemplateMarketUlr()) || StringUtils.isEmpty(basicInfo.getTemplateAccessKey())) { DataEaseException.throwException("Please check market setting info"); } return basicInfo; diff --git a/backend/src/main/resources/db/migration/V40__1.15.sql b/backend/src/main/resources/db/migration/V40__1.15.sql index a64c69a2ff..0399731308 100644 --- a/backend/src/main/resources/db/migration/V40__1.15.sql +++ b/backend/src/main/resources/db/migration/V40__1.15.sql @@ -9,6 +9,20 @@ CREATE TABLE `sys_external_token` ( UPDATE `sys_menu` set `component` = 'system/datasource/DsForm' where `component` = 'system/datasource/form'; +INSERT INTO `system_parameter`(`param_key`, `param_value`, `type`, `sort`) VALUES ('basic.dsCheckInterval', 30, 'text', 1); +INSERT INTO `system_parameter`(`param_key`, `param_value`, `type`, `sort`) VALUES ('basic.dsCheckIntervalType', 'minute', 'text', 1); + +CREATE TABLE `task_instance` ( + `task_id` VARCHAR(128) NOT NULL COMMENT '任务ID', + `execute_time` bigint(13) DEFAULT NULL COMMENT '执行时间', + `finish_time` bigint(13) DEFAULT NULL COMMENT '完成时间', + `status` VARCHAR(128) DEFAULT NULL COMMENT '状态', + `info` longtext COMMENT '执行信息', + `qrtz_instance` VARCHAR(128) DEFAULT NULL COMMENT '任务实例ID', + PRIMARY KEY (`task_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci; + +INSERT INTO `task_instance` (`task_id`) VALUES ('Datasource_check_status'); INSERT INTO `sys_menu` (`menu_id`, `pid`, `sub_count`, `type`, `title`, `name`, `component`, `menu_sort`, `icon`, `path`, `i_frame`, `cache`, `hidden`, `permission`, `create_by`, `update_by`, `create_time`, `update_time`) VALUES (41, 1, 1, 1, '应用管理', 'system-template-app', 'panel/templateApp/index', 13, 'display-setting', 'panel/templateApp/index', 0, 0, 0, 'template:read', NULL, NULL, NULL, 1620444227389); @@ -40,3 +54,13 @@ INSERT INTO `sys_menu` VALUES (800, 0, 0, 1, '数据集表单', 'dataset-form', INSERT INTO `sys_msg_channel` VALUES (3, 'webmsg.channel_wecom_msg', 'sendWecom'); INSERT INTO `sys_msg_channel` VALUES (4, 'webmsg.channel_dingtalk_msg', 'sendDingtalk'); INSERT INTO `sys_msg_channel` VALUES (5, 'webmsg.channel_lark_msg', 'sendLark'); + +UPDATE `dataset_table_function` SET `func` = 'CONCAT(s1,s2,...)' WHERE `id` = 29; +UPDATE `dataset_table_function` SET `func` = 'CONCAT(s1,s2,...)' WHERE `id` = 78; + + + +ALTER TABLE `sys_task_email` + ADD COLUMN `recisetting` varchar(255) NULL COMMENT '消息渠道' AFTER `view_ids`, + ADD COLUMN `conditions` longtext NULL COMMENT '仪表板条件' AFTER `recisetting`; + diff --git a/frontend/src/assets/xianshiqi-2.png b/frontend/src/assets/xianshiqi-2.png new file mode 100644 index 0000000000..16b1942161 Binary files /dev/null and b/frontend/src/assets/xianshiqi-2.png differ diff --git a/frontend/src/assets/xianshiqi.png b/frontend/src/assets/xianshiqi.png new file mode 100644 index 0000000000..e2fb798a77 Binary files /dev/null and b/frontend/src/assets/xianshiqi.png differ diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index ded7a90df9..2f0e7bf2dd 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -794,6 +794,7 @@ export default { no_more_than: 'Size no more than', request_timeout: 'Request timeout', message_retention_time: 'Message retention time', + ds_check_time: 'Data source detection interval', test_mail_recipient: 'Used only as a test mail recipient', to_enable_tsl: 'If the SMTP port is 587, you usually need to enable TSL', to_enable_ssl: 'If the SMTP port is 465, you usually need to enable SSL', @@ -1015,7 +1016,8 @@ export default { table_item_fontsize: 'Table font size', table_header_bg: 'Header Background', table_item_bg: 'Table Background', - table_item_font_color: 'Font Color', + table_header_font_color: 'Header Font', + table_item_font_color: 'Table Font', stripe: 'Zebra pattern', start_angle: 'Start Angle', end_angle: 'End Angle', @@ -1328,7 +1330,10 @@ export default { chart_group: 'Sub Type', chart_bar_group: 'Bar Group', field_dynamic: 'Dynamic', - aggregation: 'Aggregation' + aggregation: 'Aggregation', + filter_between: 'Between', + field_not_empty: 'Field can not be empty', + summary_not_empty: 'Summary can not be empty' }, dataset: { parse_filed: 'Parse Field', @@ -2362,6 +2367,11 @@ export default { panel: 'Panel', content: 'Content', recipients: 'Recipients', + recisetting: 'channel', + email: 'Email', + wecom: 'Wecom', + dingtalk: 'Dingtalk', + lark: 'Lark', pixel: 'Pixel', default: 'Default', custom: 'Custom', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index f0cd210a09..7c6f059a92 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -794,6 +794,7 @@ export default { no_more_than: '大小不超過', request_timeout: '請求超時時間', message_retention_time: '消息保留時間', + ds_check_time: '数据源檢測時間間隔', test_mail_recipient: '僅用來作為測試郵件收件人', to_enable_tsl: '如果SMTP埠是587,通常需要啟用TSL', to_enable_ssl: '如果SMTP埠是465,通常需要啟用SSL', @@ -1015,7 +1016,8 @@ export default { table_item_fontsize: '錶格字體大小', table_header_bg: '錶頭背景', table_item_bg: '錶格背景', - table_item_font_color: '字體顔色', + table_header_font_color: '表頭字體', + table_item_font_color: '表格字體', stripe: '斑馬紋', start_angle: '起始角度', end_angle: '結束角度', @@ -1328,7 +1330,10 @@ export default { chart_group: '子類別', chart_bar_group: '分組柱狀圖', field_dynamic: '動態值', - aggregation: '聚合方式' + aggregation: '聚合方式', + filter_between: '介於', + field_not_empty: '字段不能為空', + summary_not_empty: '聚合方式不能為空' }, dataset: { parse_filed: '解析字段', @@ -2359,10 +2364,15 @@ export default { }, emailtask: { - title: '郵件主題', + title: '報告主題', panel: '儀錶闆', - content: '郵件正文', + content: '報告正文', recipients: '收件人', + recisetting: '接受設置', + email: '郵件', + wecom: '企業微信', + dingtalk: '釘釘', + lark: '飛書', pixel: '分辨率', default: '默認', custom: '自定義', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 744a5db8d0..616a57c026 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -847,6 +847,7 @@ export default { no_more_than: '大小不超过', request_timeout: '请求超时时间', message_retention_time: '消息保留时间', + ds_check_time: '数据源检测时间间隔', test_mail_recipient: '仅用来作为测试邮件收件人', to_enable_tsl: '如果SMTP端口是587,通常需要启用TSL', to_enable_ssl: '如果SMTP端口是465,通常需要启用SSL', @@ -1068,7 +1069,8 @@ export default { table_item_fontsize: '表格字体大小', table_header_bg: '表头背景', table_item_bg: '表格背景', - table_item_font_color: '字体颜色', + table_header_font_color: '表头字体', + table_item_font_color: '表格字体', stripe: '斑马纹', start_angle: '起始角度', end_angle: '结束角度', @@ -1381,7 +1383,10 @@ export default { chart_group: '子类别', chart_bar_group: '分组柱状图', field_dynamic: '动态值', - aggregation: '聚合方式' + aggregation: '聚合方式', + filter_between: '介于', + field_not_empty: '字段不能为空', + summary_not_empty: '聚合方式不能为空' }, dataset: { parse_filed: '解析字段', @@ -2413,10 +2418,15 @@ export default { }, emailtask: { - title: '邮件主题', + title: '报告主题', panel: '仪表板', - content: '邮件正文', + content: '报告正文', recipients: '收件人', + recisetting: '接受设置', + email: '邮件', + wecom: '企业微信', + dingtalk: '钉钉', + lark: '飞书', pixel: '分辨率', default: '默认', custom: '自定义', diff --git a/frontend/src/views/chart/chart/chart.js b/frontend/src/views/chart/chart/chart.js index 91de281b9b..65eea85321 100644 --- a/frontend/src/views/chart/chart/chart.js +++ b/frontend/src/views/chart/chart/chart.js @@ -18,6 +18,7 @@ export const DEFAULT_COLOR_CASE = { alpha: 100, tableHeaderBgColor: '#6D9A49', tableItemBgColor: '#FFFFFF', + tableHeaderFontColor: '#000000', tableFontColor: '#000000', tableStripe: true, dimensionColor: '#000000', diff --git a/frontend/src/views/chart/chart/common/common.js b/frontend/src/views/chart/chart/common/common.js index f8b97be386..ccb98c70d6 100644 --- a/frontend/src/views/chart/chart/common/common.js +++ b/frontend/src/views/chart/chart/common/common.js @@ -231,7 +231,12 @@ export function seniorCfg(chart_option, chart) { if (customStyle.yAxis) { yAxis = JSON.parse(JSON.stringify(customStyle.yAxis)) } - senior.assistLine.forEach(ele => { + + const fixedLines = senior.assistLine.filter(ele => ele.field === '0') + const dynamicLines = chart.data.dynamicAssistLines + const lines = fixedLines.concat(dynamicLines) + + lines.forEach(ele => { if (chart.type.includes('horizontal')) { chart_option.series[0].markLine.data.push({ symbol: 'none', diff --git a/frontend/src/views/chart/chart/common/common_antv.js b/frontend/src/views/chart/chart/common/common_antv.js index c2945b25af..9c736d0d8e 100644 --- a/frontend/src/views/chart/chart/common/common_antv.js +++ b/frontend/src/views/chart/chart/common/common_antv.js @@ -658,7 +658,12 @@ export function getAnalyse(chart) { const a = JSON.parse(JSON.stringify(customStyle.yAxis)) yAxisPosition = transAxisPosition(chart, a) } - senior.assistLine.forEach(ele => { + + const fixedLines = senior.assistLine.filter(ele => ele.field === '0') + const dynamicLines = chart.data.dynamicAssistLines + const lines = fixedLines.concat(dynamicLines) + + lines.forEach(ele => { const content = ele.name + ' : ' + parseFloat(ele.value) assistLine.push({ type: 'line', diff --git a/frontend/src/views/chart/chart/common/common_table.js b/frontend/src/views/chart/chart/common/common_table.js index 32c26e808a..4a2e5bf44e 100644 --- a/frontend/src/views/chart/chart/common/common_table.js +++ b/frontend/src/views/chart/chart/common/common_table.js @@ -23,17 +23,17 @@ export function getCustomTheme(chart) { verticalBorderColor: borderColor }, text: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, bolderText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, measureText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign } @@ -45,17 +45,17 @@ export function getCustomTheme(chart) { verticalBorderColor: borderColor }, text: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, bolderText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, measureText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign } @@ -67,17 +67,17 @@ export function getCustomTheme(chart) { verticalBorderColor: borderColor }, text: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, bolderText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign }, measureText: { - fill: DEFAULT_COLOR_CASE.tableFontColor, + fill: DEFAULT_COLOR_CASE.tableHeaderFontColor, fontSize: DEFAULT_SIZE.tableTitleFontSize, textAlign: headerAlign } @@ -121,23 +121,23 @@ export function getCustomTheme(chart) { theme.cornerCell.cell.backgroundColor = h_c theme.cornerCell.cell.horizontalBorderColor = b_c theme.cornerCell.cell.verticalBorderColor = b_c - theme.cornerCell.bolderText.fill = c.tableFontColor - theme.cornerCell.text.fill = c.tableFontColor - theme.cornerCell.measureText.fill = c.tableFontColor + theme.cornerCell.bolderText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.cornerCell.text.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.cornerCell.measureText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor theme.rowCell.cell.backgroundColor = h_c theme.rowCell.cell.horizontalBorderColor = b_c theme.rowCell.cell.verticalBorderColor = b_c - theme.rowCell.bolderText.fill = c.tableFontColor - theme.rowCell.text.fill = c.tableFontColor - theme.rowCell.measureText.fill = c.tableFontColor + theme.rowCell.bolderText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.rowCell.text.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.rowCell.measureText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor theme.colCell.cell.backgroundColor = h_c theme.colCell.cell.horizontalBorderColor = b_c theme.colCell.cell.verticalBorderColor = b_c - theme.colCell.bolderText.fill = c.tableFontColor - theme.colCell.text.fill = c.tableFontColor - theme.colCell.measureText.fill = c.tableFontColor + theme.colCell.bolderText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.colCell.text.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor + theme.colCell.measureText.fill = c.tableHeaderFontColor ? c.tableHeaderFontColor : c.tableFontColor theme.dataCell.cell.backgroundColor = i_c theme.dataCell.cell.horizontalBorderColor = b_c diff --git a/frontend/src/views/chart/chart/util.js b/frontend/src/views/chart/chart/util.js index b44f3fa95a..e595f0e5a9 100644 --- a/frontend/src/views/chart/chart/util.js +++ b/frontend/src/views/chart/chart/util.js @@ -45,6 +45,7 @@ export const TYPE_CONFIGS = [ 'color-selector': [ 'tableHeaderBgColor', 'tableItemBgColor', + 'tableHeaderFontColor', 'tableFontColor', 'tableBorderColor', 'alpha' @@ -88,6 +89,7 @@ export const TYPE_CONFIGS = [ 'color-selector': [ 'tableHeaderBgColor', 'tableItemBgColor', + 'tableHeaderFontColor', 'tableFontColor', 'tableBorderColor', 'alpha' @@ -134,6 +136,7 @@ export const TYPE_CONFIGS = [ 'color-selector': [ 'tableHeaderBgColor', 'tableItemBgColor', + 'tableHeaderFontColor', 'tableFontColor', 'tableBorderColor', 'alpha' @@ -1463,6 +1466,7 @@ export const TYPE_CONFIGS = [ 'color-selector': [ 'tableHeaderBgColor', 'tableItemBgColor', + 'tableHeaderFontColor', 'tableFontColor', 'alpha' ], @@ -1499,6 +1503,7 @@ export const TYPE_CONFIGS = [ 'color-selector': [ 'tableHeaderBgColor', 'tableItemBgColor', + 'tableHeaderFontColor', 'tableFontColor', 'alpha' ], diff --git a/frontend/src/views/chart/components/normal/LabelNormal.vue b/frontend/src/views/chart/components/normal/LabelNormal.vue index 155175ce78..707f6ca950 100644 --- a/frontend/src/views/chart/components/normal/LabelNormal.vue +++ b/frontend/src/views/chart/components/normal/LabelNormal.vue @@ -259,6 +259,10 @@ export default { resultFormat() { if (!this.chart.data) return const value = this.chart.data.series[0].data[0] + if (value === null || value === undefined) { + this.result = '-' + return + } let yAxis = [] try { yAxis = JSON.parse(this.chart.yaxis) diff --git a/frontend/src/views/chart/components/senior/AssistLine.vue b/frontend/src/views/chart/components/senior/AssistLine.vue index da7b7fdcfd..e21665a93f 100644 --- a/frontend/src/views/chart/components/senior/AssistLine.vue +++ b/frontend/src/views/chart/components/senior/AssistLine.vue @@ -16,10 +16,14 @@ {{ $t('chart.field_fixed') }} + {{ $t('chart.field_dynamic') }} - + {{ item.value }} + + {{ item.curField.name + '(' + $t('chart.' + item.summary) + ')' }} + @@ -34,7 +38,7 @@ width="70%" class="dialog-css" > - +