diff --git a/backend/pom.xml b/backend/pom.xml index 8ed59a2fa0..2ec6c7ff5e 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -207,7 +207,7 @@ io.dataease dataease-plugin-interface - 1.6 + 1.7 cn.hutool diff --git a/backend/src/main/java/io/dataease/auth/util/JWTUtils.java b/backend/src/main/java/io/dataease/auth/util/JWTUtils.java index 9e27662bf1..28072efd05 100644 --- a/backend/src/main/java/io/dataease/auth/util/JWTUtils.java +++ b/backend/src/main/java/io/dataease/auth/util/JWTUtils.java @@ -17,16 +17,13 @@ import org.springframework.core.env.Environment; import java.util.Date; - public class JWTUtils { - // token过期时间1min (过期会自动刷新续命 目的是避免一直都是同一个token ) private static final long EXPIRE_TIME = 1 * 60 * 1000; // 登录间隔时间10min 超过这个时间强制重新登录 private static long Login_Interval; - /** * 校验token是否正确 * @@ -82,7 +79,8 @@ public class JWTUtils { public static boolean loginExpire(String token) { if (Login_Interval == 0) { // 默认超时时间是8h - int minute = CommonBeanFactory.getBean(Environment.class).getProperty("dataease.login_timeout", Integer.class, 8 * 60); + Long minute = CommonBeanFactory.getBean(Environment.class).getProperty("dataease.login_timeout", Long.class, + 8 * 60L); // 分钟换算成毫秒 Login_Interval = minute * 1000 * 60; } @@ -128,19 +126,19 @@ public class JWTUtils { public static String signLink(String resourceId, Long userId, String secret) { Algorithm algorithm = Algorithm.HMAC256(secret); - if(userId == null){ + if (userId == null) { return JWT.create().withClaim("resourceId", resourceId).sign(algorithm); - }else { + } else { return JWT.create().withClaim("resourceId", resourceId).withClaim("userId", userId).sign(algorithm); } } - public static boolean verifyLink(String token, String resourceId, Long userId, String secret) { + public static boolean verifyLink(String token, String resourceId, Long userId, String secret) { Algorithm algorithm = Algorithm.HMAC256(secret); JWTVerifier verifier; - if(userId == null){ + if (userId == null) { verifier = JWT.require(algorithm).withClaim("resourceId", resourceId).build(); - }else { + } else { verifier = JWT.require(algorithm).withClaim("resourceId", resourceId).withClaim("userId", userId).build(); } diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.java b/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.java new file mode 100644 index 0000000000..efec50afcf --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.java @@ -0,0 +1,9 @@ +package io.dataease.base.mapper.ext; + +public interface ExtTaskMapper { + + int runningCount(Long taskId); + + void resetRunnings(Long taskId); + +} diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.xml new file mode 100644 index 0000000000..213d5ae488 --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtTaskMapper.xml @@ -0,0 +1,13 @@ + + + + + + + + update sys_task_instance set status = -1, info = 'System Interrupt Error' where task_id = #{taskId} and status = 0 + + + diff --git a/backend/src/main/java/io/dataease/commons/constants/ColumnPermissionConstants.java b/backend/src/main/java/io/dataease/commons/constants/ColumnPermissionConstants.java new file mode 100644 index 0000000000..52f1a00796 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/ColumnPermissionConstants.java @@ -0,0 +1,10 @@ +package io.dataease.commons.constants; + +public class ColumnPermissionConstants { + + public final static String Prohibit = "Prohibit"; + public final static String Desensitization = "Desensitization"; + public final static String Desensitization_desc = "******"; + + +} diff --git a/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolExecutor.java b/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolExecutor.java new file mode 100644 index 0000000000..d55d9e8a65 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolExecutor.java @@ -0,0 +1,113 @@ +package io.dataease.commons.pool; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import io.dataease.commons.utils.LogUtil; + +public class PriorityThreadPoolExecutor extends ThreadPoolExecutor { + + public static AtomicInteger globaInteger = new AtomicInteger(1); + + private ThreadLocal local = new ThreadLocal() { + @Override + protected Integer initialValue() { + return 1; + } + }; + + public PriorityThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, getWorkQueue()); + } + + public PriorityThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + ThreadFactory threadFactory) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, getWorkQueue(), threadFactory); + } + + public PriorityThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, getWorkQueue(), handler); + } + + public PriorityThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + ThreadFactory threadFactory, RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, getWorkQueue(), threadFactory, handler); + } + + protected static PriorityBlockingQueue getWorkQueue() { + return new PriorityBlockingQueue(); + } + + @Override + public void execute(Runnable command) { + int andIncrement = globaInteger.getAndIncrement(); + Integer theadInteger = local.get(); + try { + if (theadInteger == 0) { + this.execute(command, 0); + } else { + this.execute(command, andIncrement); + } + + } finally { + local.set(1); + } + } + + public void execute(Runnable command, int priority) { + super.execute(new PriorityRunnable(command, priority)); + } + + public Future submit(Callable task, int priority) { + local.set(priority); + return super.submit(task); + } + + protected static class PriorityRunnable> + implements Runnable, Comparable> { + private final static AtomicLong seq = new AtomicLong(); + private final long seqNum; + Runnable run; + private int priority; + + public PriorityRunnable(Runnable run, int priority) { + seqNum = seq.getAndIncrement(); + this.run = run; + this.priority = priority; + } + + public int getPriority() { + return priority; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public Runnable getRun() { + return run; + } + + @Override + public void run() { + LogUtil.info("number " + priority + " is starting..."); + this.run.run(); + } + + @Override + public int compareTo(PriorityRunnable other) { + int res = 0; + if (this.priority == other.priority) { + if (other.run != this.run) {// ASC + res = (seqNum < other.seqNum ? -1 : 1); + } + } else {// DESC + res = this.priority > other.priority ? 1 : -1; + } + return res; + } + } + +} diff --git a/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolProperties.java b/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolProperties.java new file mode 100644 index 0000000000..efb807ba86 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/pool/PriorityThreadPoolProperties.java @@ -0,0 +1,17 @@ +package io.dataease.commons.pool; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import lombok.Data; + +@ConfigurationProperties(prefix = "detask", ignoreInvalidFields = true) +@Data +@Component +public class PriorityThreadPoolProperties { + + private int corePoolSize = 2; + private int maximumPoolSize = 100; + private int keepAliveTime = 60; + +} diff --git a/backend/src/main/java/io/dataease/config/AsyncConfig.java b/backend/src/main/java/io/dataease/config/AsyncConfig.java index 22a88ef995..8a8786c294 100644 --- a/backend/src/main/java/io/dataease/config/AsyncConfig.java +++ b/backend/src/main/java/io/dataease/config/AsyncConfig.java @@ -1,15 +1,25 @@ package io.dataease.config; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Resource; + import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import io.dataease.commons.pool.PriorityThreadPoolExecutor; +import io.dataease.commons.pool.PriorityThreadPoolProperties; + @EnableAsync(proxyTargetClass = true) @Configuration public class AsyncConfig { + @Resource + private PriorityThreadPoolProperties priorityThreadPoolProperties; + @Bean public AsyncTaskExecutor taskExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); @@ -18,4 +28,18 @@ public class AsyncConfig { executor.setMaxPoolSize(10); return executor; } + + @Bean + public PriorityThreadPoolExecutor priorityExecutor() { + int corePoolSize = priorityThreadPoolProperties.getCorePoolSize(); + + int maximumPoolSize = priorityThreadPoolProperties.getMaximumPoolSize(); + + int keepAliveTime = priorityThreadPoolProperties.getKeepAliveTime(); + + PriorityThreadPoolExecutor executor = new PriorityThreadPoolExecutor(corePoolSize, maximumPoolSize, + keepAliveTime, TimeUnit.SECONDS); + return executor; + } + } diff --git a/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java b/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java index 3c42edf228..efa12809b8 100644 --- a/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java +++ b/backend/src/main/java/io/dataease/dto/datasource/DBTableDTO.java @@ -15,6 +15,8 @@ public class DBTableDTO { private String datasourceId; @ApiModelProperty("数据源名称") private String name; + @ApiModelProperty("表注释") + private String remark; @ApiModelProperty("启用检测") private boolean enableCheck; @ApiModelProperty("数据集路径") diff --git a/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java b/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java new file mode 100644 index 0000000000..e53066e9d7 --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/datasource/TableDesc.java @@ -0,0 +1,15 @@ +package io.dataease.dto.datasource; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class TableDesc { + @ApiModelProperty("表名称") + private String name; + @ApiModelProperty("表备注") + private String remark; +} diff --git a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java index c93e59e11c..3a41798e93 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java +++ b/backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java @@ -28,7 +28,7 @@ public class ScheduleManager { * @throws SchedulerException */ public void addSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class cls, int repeatIntervalTime, - JobDataMap jobDataMap) throws SchedulerException { + JobDataMap jobDataMap) throws SchedulerException { JobBuilder jobBuilder = JobBuilder.newJob(cls).withIdentity(jobKey); @@ -46,7 +46,8 @@ public class ScheduleManager { scheduler.scheduleJob(jd, trigger); } - public void addSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class cls, int repeatIntervalTime) throws SchedulerException { + public void addSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class cls, int repeatIntervalTime) + throws SchedulerException { addSimpleJob(jobKey, triggerKey, cls, repeatIntervalTime); } @@ -59,7 +60,8 @@ public class ScheduleManager { * @param cron * @param jobDataMap */ - public void addCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, Date endTime, JobDataMap jobDataMap) { + public void addCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, + Date endTime, JobDataMap jobDataMap) { try { LogUtil.info("addCronJob: " + triggerKey.getName() + "," + triggerKey.getGroup()); @@ -99,7 +101,8 @@ public class ScheduleManager { } } - public void addCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, Date endTime) { + public void addCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, + Date endTime) { addCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null); } @@ -140,7 +143,8 @@ public class ScheduleManager { * @param cron * @throws SchedulerException */ - public void modifyCronJobTime(TriggerKey triggerKey, String cron, Date startTime, Date endTime) throws SchedulerException { + public void modifyCronJobTime(TriggerKey triggerKey, String cron, Date startTime, Date endTime) + throws SchedulerException { LogUtil.info("modifyCronJobTime: " + triggerKey.getName() + "," + triggerKey.getGroup()); @@ -151,7 +155,6 @@ public class ScheduleManager { return; } - /** 方式一 :调用 rescheduleJob 开始 */ TriggerBuilder triggerBuilder = TriggerBuilder.newTrigger();// 触发器 @@ -279,7 +282,6 @@ public class ScheduleManager { } } - public static void startJobs(Scheduler sched) { try { sched.start(); @@ -289,7 +291,6 @@ public class ScheduleManager { } } - public void shutdownJobs(Scheduler sched) { try { if (!sched.isShutdown()) { @@ -312,7 +313,7 @@ public class ScheduleManager { * @throws SchedulerException */ public void addOrUpdateSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class clz, - int intervalTime, JobDataMap jobDataMap) throws SchedulerException { + int intervalTime, JobDataMap jobDataMap) throws SchedulerException { if (scheduler.checkExists(triggerKey)) { modifySimpleJobTime(triggerKey, intervalTime); @@ -323,7 +324,7 @@ public class ScheduleManager { } public void addOrUpdateSingleJob(JobKey jobKey, TriggerKey triggerKey, Class clz, - Date date, JobDataMap jobDataMap) throws SchedulerException { + Date date, JobDataMap jobDataMap) throws SchedulerException { if (scheduler.checkExists(triggerKey)) { modifySingleJobTime(triggerKey, date); } else { @@ -333,15 +334,15 @@ public class ScheduleManager { } public void addOrUpdateSingleJob(JobKey jobKey, TriggerKey triggerKey, Class clz, - Date date) throws SchedulerException { + Date date) throws SchedulerException { addOrUpdateSingleJob(jobKey, triggerKey, clz, date, null); } - public void addOrUpdateSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class clz, int intervalTime) throws SchedulerException { + public void addOrUpdateSimpleJob(JobKey jobKey, TriggerKey triggerKey, Class clz, int intervalTime) + throws SchedulerException { addOrUpdateSimpleJob(jobKey, triggerKey, clz, intervalTime, null); } - /** * 添加或修改 cronJob * @@ -352,7 +353,8 @@ public class ScheduleManager { * @param jobDataMap * @throws SchedulerException */ - public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, Date endTime, JobDataMap jobDataMap) throws SchedulerException { + public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, + Date endTime, JobDataMap jobDataMap) throws SchedulerException { LogUtil.info("AddOrUpdateCronJob: " + jobKey.getName() + "," + triggerKey.getGroup()); @@ -363,7 +365,8 @@ public class ScheduleManager { } } - public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, Date endTime) throws SchedulerException { + public void addOrUpdateCronJob(JobKey jobKey, TriggerKey triggerKey, Class jobClass, String cron, Date startTime, + Date endTime) throws SchedulerException { addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null); } @@ -398,7 +401,8 @@ public class ScheduleManager { if (!CronExpression.isValidExpression(cron)) { DataEaseException.throwException("cron :" + cron + " error"); } - return TriggerBuilder.newTrigger().withIdentity("Calculate Date").withSchedule(CronScheduleBuilder.cronSchedule(cron)).build(); + return TriggerBuilder.newTrigger().withIdentity("Calculate Date") + .withSchedule(CronScheduleBuilder.cronSchedule(cron)).build(); } diff --git a/backend/src/main/java/io/dataease/job/sechedule/strategy/TaskHandler.java b/backend/src/main/java/io/dataease/job/sechedule/strategy/TaskHandler.java index c685930a17..87dff10bdf 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/strategy/TaskHandler.java +++ b/backend/src/main/java/io/dataease/job/sechedule/strategy/TaskHandler.java @@ -13,9 +13,6 @@ import java.util.Date; public abstract class TaskHandler implements InitializingBean { - private static final String[] week = {"SUNDAY", "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY"}; - - public void addTask(ScheduleManager scheduleManager, GlobalTaskEntity taskEntity) throws Exception { // 1。首先看看是否过期 Long endTime = taskEntity.getEndTime(); @@ -30,12 +27,11 @@ public abstract class TaskHandler implements InitializingBean { if (ObjectUtils.isNotEmpty(taskEntity.getEndTime())) { new Date(taskEntity.getEndTime()); } - Class executor = this.getClass(); + Class executor = this.getClass(); String cron = cron(taskEntity); scheduleManager.addOrUpdateCronJob(jobKey, triggerKey, executor, cron, start, end, jobDataMap(taskEntity)); } - protected abstract JobDataMap jobDataMap(GlobalTaskEntity taskEntity); private String cron(GlobalTaskEntity taskEntity) { @@ -54,36 +50,34 @@ public abstract class TaskHandler implements InitializingBean { instance.setTime(date); if (taskEntity.getRateType() == 0) { - return - instance.get(Calendar.SECOND) + " " + - instance.get(Calendar.MINUTE) + " " + - instance.get(Calendar.HOUR_OF_DAY) + " * * ?"; + return instance.get(Calendar.SECOND) + " " + + instance.get(Calendar.MINUTE) + " " + + instance.get(Calendar.HOUR_OF_DAY) + " * * ?"; } if (taskEntity.getRateType() == 1) { - return - instance.get(Calendar.SECOND) + " " + - instance.get(Calendar.MINUTE) + " " + - instance.get(Calendar.HOUR_OF_DAY) + " ? * " + - getDayOfWeek(instance); + return instance.get(Calendar.SECOND) + " " + + instance.get(Calendar.MINUTE) + " " + + instance.get(Calendar.HOUR_OF_DAY) + " ? * " + + getDayOfWeek(instance); } if (taskEntity.getRateType() == 2) { - return - instance.get(Calendar.SECOND) + " " + - instance.get(Calendar.MINUTE) + " " + - instance.get(Calendar.HOUR_OF_DAY) + " " + - instance.get(Calendar.DATE) + " * ?"; + return instance.get(Calendar.SECOND) + " " + + instance.get(Calendar.MINUTE) + " " + + instance.get(Calendar.HOUR_OF_DAY) + " " + + instance.get(Calendar.DATE) + " * ?"; } return null; } + public abstract void resetRunningInstance(Long taskId); + private String getDayOfWeek(Calendar instance) { int index = instance.get(Calendar.DAY_OF_WEEK); - index = (index + 1) % 7; + index = (index + 1) % 7; return String.valueOf(index); } - public void removeTask(ScheduleManager scheduleManager, GlobalTaskEntity taskEntity) { JobKey jobKey = new JobKey(taskEntity.getTaskId().toString()); TriggerKey triggerKey = new TriggerKey(taskEntity.getTaskId().toString()); @@ -95,14 +89,16 @@ public abstract class TaskHandler implements InitializingBean { scheduleManager.fireNow(jobKey); } - - //判断任务是否过期 + // 判断任务是否过期 public Boolean taskExpire(Long endTime) { - if (ObjectUtils.isEmpty(endTime)) return false; + if (ObjectUtils.isEmpty(endTime)) + return false; Long now = System.currentTimeMillis(); return now > endTime; } + protected abstract Boolean taskIsRunning(Long taskId); + @Override public void afterPropertiesSet() throws Exception { String beanName = null; diff --git a/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java index d84ba736de..12ba9352a2 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java +++ b/backend/src/main/java/io/dataease/job/sechedule/strategy/impl/EmailTaskHandler.java @@ -5,6 +5,7 @@ import io.dataease.auth.entity.TokenInfo; import io.dataease.auth.service.AuthUserService; import io.dataease.auth.service.impl.AuthUserServiceImpl; import io.dataease.auth.util.JWTUtils; +import io.dataease.base.mapper.ext.ExtTaskMapper; import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.LogUtil; import io.dataease.commons.utils.ServletUtils; @@ -20,6 +21,7 @@ import io.dataease.service.system.EmailService; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.quartz.*; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -46,6 +48,16 @@ public class EmailTaskHandler extends TaskHandler implements Job { return jobDataMap; } + public EmailTaskHandler proxy() { + return CommonBeanFactory.getBean(EmailTaskHandler.class); + } + + @Override + protected Boolean taskIsRunning(Long taskId) { + ExtTaskMapper extTaskMapper = CommonBeanFactory.getBean(ExtTaskMapper.class); + return extTaskMapper.runningCount(taskId) > 0; + } + @Override public void execute(JobExecutionContext context) throws JobExecutionException { // 插件没有加载 空转 @@ -54,11 +66,16 @@ public class EmailTaskHandler extends TaskHandler implements Job { JobDataMap jobDataMap = context.getJobDetail().getJobDataMap(); GlobalTaskEntity taskEntity = (GlobalTaskEntity) jobDataMap.get("taskEntity"); + ScheduleManager scheduleManager = SpringContextUtil.getBean(ScheduleManager.class); if (taskExpire(taskEntity.getEndTime())) { - ScheduleManager scheduleManager = SpringContextUtil.getBean(ScheduleManager.class); removeTask(scheduleManager, taskEntity); return; } + if (taskIsRunning(taskEntity.getTaskId())) { + LogUtil.info("Skip synchronization task: {} ,due to task status is {}", + taskEntity.getTaskId(), "running"); + return; + } GlobalTaskInstance taskInstance = buildInstance(taskEntity); Long instanceId = saveInstance(taskInstance); @@ -67,10 +84,15 @@ public class EmailTaskHandler extends TaskHandler implements Job { XpackEmailTemplateDTO emailTemplate = (XpackEmailTemplateDTO) jobDataMap.get("emailTemplate"); SysUserEntity creator = (SysUserEntity) jobDataMap.get("creator"); LogUtil.info("start execute send panel report task..."); - sendReport(taskInstance, emailTemplate, creator); + proxy().sendReport(taskInstance, emailTemplate, creator); } + @Override + public void resetRunningInstance(Long taskId) { + ExtTaskMapper extTaskMapper = CommonBeanFactory.getBean(ExtTaskMapper.class); + extTaskMapper.resetRunnings(taskId); + } public Long saveInstance(GlobalTaskInstance taskInstance) { EmailXpackService emailXpackService = SpringContextUtil.getBean(EmailXpackService.class); @@ -99,11 +121,12 @@ public class EmailTaskHandler extends TaskHandler implements Job { emailXpackService.saveInstance(taskInstance); } - + @Async("priorityExecutor") public void sendReport(GlobalTaskInstance taskInstance, XpackEmailTemplateDTO emailTemplateDTO, - SysUserEntity user) { + SysUserEntity user) { EmailXpackService emailXpackService = SpringContextUtil.getBean(EmailXpackService.class); try { + String panelId = emailTemplateDTO.getPanelId(); String url = panelUrl(panelId); String token = tokenByUser(user); @@ -116,11 +139,15 @@ public class EmailTaskHandler extends TaskHandler implements Job { String recipients = emailTemplateDTO.getRecipients(); byte[] content = emailTemplateDTO.getContent(); EmailService emailService = SpringContextUtil.getBean(EmailService.class); + String contentStr = ""; if (ObjectUtils.isNotEmpty(content)) { contentStr = new String(content, "UTF-8"); } - emailService.sendWithImage(recipients, emailTemplateDTO.getTitle(), contentStr, bytes); + emailService.sendWithImage(recipients, emailTemplateDTO.getTitle(), + contentStr, bytes); + + Thread.sleep(10000); success(taskInstance); } catch (Exception e) { error(taskInstance, e); diff --git a/backend/src/main/java/io/dataease/listener/GlobalTaskStartListener.java b/backend/src/main/java/io/dataease/listener/GlobalTaskStartListener.java index c7361ee93b..c8de41274a 100644 --- a/backend/src/main/java/io/dataease/listener/GlobalTaskStartListener.java +++ b/backend/src/main/java/io/dataease/listener/GlobalTaskStartListener.java @@ -32,6 +32,7 @@ public class GlobalTaskStartListener implements ApplicationListener { TaskHandler taskHandler = TaskStrategyFactory.getInvokeStrategy(task.getTaskType()); try { + taskHandler.resetRunningInstance(task.getTaskId()); taskHandler.addTask(scheduleManager, task); } catch (Exception e) { e.printStackTrace(); diff --git a/backend/src/main/java/io/dataease/plugins/server/ColumnPermissionsController.java b/backend/src/main/java/io/dataease/plugins/server/ColumnPermissionsController.java new file mode 100644 index 0000000000..78d3e30215 --- /dev/null +++ b/backend/src/main/java/io/dataease/plugins/server/ColumnPermissionsController.java @@ -0,0 +1,97 @@ +package io.dataease.plugins.server; + +import com.github.pagehelper.Page; +import com.github.pagehelper.PageHelper; +import io.dataease.commons.utils.PageUtils; +import io.dataease.commons.utils.Pager; +import io.dataease.i18n.Translator; +import io.dataease.plugins.common.entity.XpackConditionEntity; +import io.dataease.plugins.common.entity.XpackGridRequest; +import io.dataease.plugins.config.SpringContextUtil; +import io.dataease.plugins.xpack.auth.dto.request.DataSetColumnPermissionsDTO; +import io.dataease.plugins.xpack.auth.dto.request.DataSetRowPermissionsDTO; +import io.dataease.plugins.xpack.auth.dto.request.DatasetColumnPermissions; +import io.dataease.plugins.xpack.auth.dto.request.DatasetRowPermissions; +import io.dataease.plugins.xpack.auth.service.ColumnPermissionService; +import io.dataease.plugins.xpack.auth.service.RowPermissionService; +import io.swagger.annotations.ApiOperation; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.*; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping("plugin/dataset/columnPermissions") +public class ColumnPermissionsController { + + + @ApiOperation("保存") + @PostMapping("save") + public void save(@RequestBody DatasetColumnPermissions datasetColumnPermissions) throws Exception { + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + DataSetColumnPermissionsDTO request = new DataSetColumnPermissionsDTO(); + request.setAuthTargetType(datasetColumnPermissions.getAuthTargetType()); + request.setAuthTargetId(datasetColumnPermissions.getAuthTargetId()); + request.setDatasetId(datasetColumnPermissions.getDatasetId()); + List columnPermissionsDTOS = columnPermissionService.searchPermissions(request); + if(StringUtils.isEmpty(datasetColumnPermissions.getId())){ + if(!CollectionUtils.isEmpty(columnPermissionsDTOS)){ + throw new Exception(Translator.get("i18n_rp_exist")); + } + }else { + if(!CollectionUtils.isEmpty(columnPermissionsDTOS) && columnPermissionsDTOS.size() > 1){ + throw new Exception(Translator.get("i18n_rp_exist")); + } + if(columnPermissionsDTOS.size() == 1 && !columnPermissionsDTOS.get(0).getId().equalsIgnoreCase(datasetColumnPermissions.getId())){ + throw new Exception(Translator.get("i18n_rp_exist")); + } + } + columnPermissionService.save(datasetColumnPermissions); + } + + @ApiOperation("查询") + @PostMapping("/list") + public List searchPermissions(@RequestBody DataSetColumnPermissionsDTO request) { + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + return columnPermissionService.searchPermissions(request); + } + + @ApiOperation("删除") + @PostMapping("/delete/{id}") + public void delete(@PathVariable String id) { + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + columnPermissionService.delete(id); + } + + @ApiOperation("分页查询") + @PostMapping("/pageList/{datasetId}/{goPage}/{pageSize}") + public Pager> rowPermissions(@PathVariable String datasetId, @PathVariable int goPage, @PathVariable int pageSize, @RequestBody XpackGridRequest request) { + Page page = PageHelper.startPage(goPage, pageSize, true); + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + List conditionEntities = request.getConditions() == null ? new ArrayList<>() : request.getConditions(); + XpackConditionEntity entity = new XpackConditionEntity(); + entity.setField("dataset_column_permissions.dataset_id"); + entity.setOperator("eq"); + entity.setValue(datasetId); + conditionEntities.add(entity); + request.setConditions(conditionEntities); + return PageUtils.setPageInfo(page, columnPermissionService.queryPermissions(request)); + } + + @ApiOperation("有权限的对象") + @PostMapping("/authObjs") + public List authObjs(@RequestBody DataSetColumnPermissionsDTO request) { + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + return (List) columnPermissionService.authObjs(request); + } + + @ApiOperation("详情") + @PostMapping("/permissionInfo") + public DataSetColumnPermissionsDTO permissionInfo(@RequestBody DataSetColumnPermissionsDTO request) { + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + return columnPermissionService.permissionInfo(request); + } + +} diff --git a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java index 9ecd645704..fd8eb56a74 100644 --- a/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java +++ b/backend/src/main/java/io/dataease/plugins/server/XEmailTaskServer.java @@ -3,6 +3,7 @@ package io.dataease.plugins.server; import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; import io.dataease.commons.exception.DEException; +import io.dataease.commons.pool.PriorityThreadPoolExecutor; import io.dataease.commons.utils.*; import io.dataease.plugins.common.entity.GlobalTaskEntity; import io.dataease.plugins.common.entity.GlobalTaskInstance; @@ -23,6 +24,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.concurrent.Future; + +import javax.annotation.Resource; @Api(tags = "xpack:定时报告") @RequestMapping("/plugin/task") @@ -32,6 +36,9 @@ public class XEmailTaskServer { @Autowired private ScheduleService scheduleService; + @Resource + private PriorityThreadPoolExecutor priorityExecutor; + @PostMapping("/queryTasks/{goPage}/{pageSize}") public Pager> queryTask(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody XpackGridRequest request) { @@ -85,7 +92,19 @@ public class XEmailTaskServer { String token = ServletUtils.getToken(); String fileId = null; try { - fileId = emailXpackService.print(url, token, buildPixel(request.getPixel())); + Future future = priorityExecutor.submit(() -> { + try { + return emailXpackService.print(url, token, buildPixel(request.getPixel())); + } catch (Exception e) { + LogUtil.error(e.getMessage(), e); + DEException.throwException("预览失败,请联系管理员"); + } + return null; + }, 0); + Object object = future.get(); + if (ObjectUtils.isNotEmpty(object)) { + fileId = object.toString(); + } } catch (Exception e) { LogUtil.error(e.getMessage(), e); DEException.throwException("预览失败,请联系管理员"); diff --git a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java index a09e5576cc..d99d7a80b0 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java @@ -1,5 +1,6 @@ package io.dataease.provider.datasource; +import io.dataease.dto.datasource.TableDesc; import io.dataease.dto.datasource.TableFiled; import io.dataease.controller.request.datasource.DatasourceRequest; @@ -12,7 +13,7 @@ public abstract class DatasourceProvider { abstract public List getData(DatasourceRequest datasourceRequest) throws Exception; - abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; + abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { getData(datasourceRequest); diff --git a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java index 8943c498c5..3b1e9c7c61 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java @@ -9,6 +9,7 @@ import io.dataease.controller.request.datasource.es.Requst; import io.dataease.controller.request.datasource.es.RequstWithCursor; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.dto.datasource.EsConfiguration; +import io.dataease.dto.datasource.TableDesc; import io.dataease.dto.datasource.TableFiled; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; @@ -37,11 +38,11 @@ public class EsProvider extends DatasourceProvider { /** * 这里使用声明式缓存不是很妥当 * 改为chartViewService中使用编程式缓存 - @Cacheable( - value = JdbcConstants.JDBC_PROVIDER_KEY, - key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", - condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" - ) + * + * @Cacheable( value = JdbcConstants.JDBC_PROVIDER_KEY, + * key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", + * condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" + * ) */ @Override public List getData(DatasourceRequest dsr) throws Exception { @@ -49,7 +50,7 @@ public class EsProvider extends DatasourceProvider { try { EsConfiguration esConfiguration = new Gson().fromJson(dsr.getDatasource().getConfiguration(), EsConfiguration.class); HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); @@ -58,24 +59,24 @@ public class EsProvider extends DatasourceProvider { requst.setQuery(dsr.getQuery()); requst.setFetch_size(dsr.getFetchSize()); String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + esConfiguration.getUri() + "?format=json" : esConfiguration.getUrl() + "/" + esConfiguration.getUri() + "?format=json"; - String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); list.addAll(fetchResult(esReponse)); - if(dsr.isPageable()){ - Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size(); + if (dsr.isPageable()) { + Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize() : list.size(); list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize); } - if(!dsr.isPreviewData()){ - while (StringUtils.isNotEmpty(esReponse.getCursor())) { - RequstWithCursor requstWithCursor = new RequstWithCursor(); - requstWithCursor.setQuery(dsr.getQuery()); - requstWithCursor.setFetch_size(dsr.getFetchSize()); - requstWithCursor.setCursor(esReponse.getCursor()); - response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig); - esReponse = new Gson().fromJson(response, EsReponse.class); - list.addAll(fetchResult(esReponse)); - } + if (!dsr.isPreviewData()) { + while (StringUtils.isNotEmpty(esReponse.getCursor())) { + RequstWithCursor requstWithCursor = new RequstWithCursor(); + requstWithCursor.setQuery(dsr.getQuery()); + requstWithCursor.setFetch_size(dsr.getFetchSize()); + requstWithCursor.setCursor(esReponse.getCursor()); + response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig); + esReponse = new Gson().fromJson(response, EsReponse.class); + list.addAll(fetchResult(esReponse)); + } } } catch (Exception e) { e.printStackTrace(); @@ -110,7 +111,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResult(EsReponse esReponse) throws Exception { List list = new LinkedList<>(); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } list.addAll(esReponse.getRows()); @@ -132,7 +133,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResultField(String response) throws Exception { List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } @@ -150,7 +151,7 @@ public class EsProvider extends DatasourceProvider { private List fetchResultField4Sql(String response) throws Exception { List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } @@ -183,31 +184,35 @@ public class EsProvider extends DatasourceProvider { } @Override - public List getTables(DatasourceRequest datasourceRequest) throws Exception { - List tables = new ArrayList<>(); + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tables = new ArrayList<>(); try { String response = exexQuery(datasourceRequest, "show tables", "?format=json"); tables = fetchTables(response); - tables = tables.stream().filter(table -> !table.startsWith(".")).collect(Collectors.toList()); + tables = tables.stream().filter(table -> !table.getName().startsWith(".")).collect(Collectors.toList()); } catch (Exception e) { DataEaseException.throwException(e); } return tables; } - private List fetchTables(String response) throws Exception { - List tables = new ArrayList<>(); + private List fetchTables(String response) throws Exception { + List tables = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); - if(esReponse.getError() != null){ + if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } for (String[] row : esReponse.getRows()) { - if(row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")){ - tables.add(row[0]); - } - if(row.length == 2 && row[1].contains("TABLE")){ - tables.add(row[0]); + if (row.length == 3 && row[1].contains("TABLE") && row[2].equalsIgnoreCase("INDEX")) { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName(row[0]); + tables.add(tableDesc); + } + if (row.length == 2 && row[1].contains("TABLE")) { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName(row[0]); + tables.add(tableDesc); } } return tables; @@ -219,36 +224,35 @@ public class EsProvider extends DatasourceProvider { } - @Override public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); String response = exexGetQuery(datasourceRequest); - if(JSONObject.parseObject(response).getJSONObject("error") != null){ + if (JSONObject.parseObject(response).getJSONObject("error") != null) { throw new Exception(JSONObject.parseObject(response).getJSONObject("error").getString("reason")); } - String version = JSONObject.parseObject(response).getJSONObject("version").getString("number"); + String version = JSONObject.parseObject(response).getJSONObject("version").getString("number"); String[] versionList = version.split("\\."); - if(Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3){ + if (Integer.valueOf(versionList[0]) < 7 && Integer.valueOf(versionList[1]) < 3) { throw new Exception(Translator.get("i18n_es_limit")); } - if(Integer.valueOf(versionList[0]) == 6 ) { + if (Integer.valueOf(versionList[0]) == 6) { esConfiguration.setUri("_xpack/sql"); } - if(Integer.valueOf(versionList[0]) == 7 ) { + if (Integer.valueOf(versionList[0]) == 7) { esConfiguration.setUri("_sql"); } datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration)); getTables(datasourceRequest); } - private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){ + private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); - uri = esConfiguration.getUri()+uri; + uri = esConfiguration.getUri() + uri; HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); @@ -258,20 +262,20 @@ public class EsProvider extends DatasourceProvider { requst.setQuery(sql); requst.setFetch_size(datasourceRequest.getFetchSize()); String url = esConfiguration.getUrl().endsWith("/") ? esConfiguration.getUrl() + uri : esConfiguration.getUrl() + "/" + uri; - String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); return response; } - private String exexGetQuery(DatasourceRequest datasourceRequest){ + private String exexGetQuery(DatasourceRequest datasourceRequest) { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); HttpClientConfig httpClientConfig = new HttpClientConfig(); - if(StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())){ + if (StringUtils.isNotEmpty(esConfiguration.getEsUsername()) && StringUtils.isNotEmpty(esConfiguration.getEsPassword())) { String auth = esConfiguration.getEsUsername() + ":" + esConfiguration.getEsPassword(); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); } - String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig); + String response = HttpClientUtil.get(esConfiguration.getUrl(), httpClientConfig); return response; } diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 48a2af205f..5e558dd700 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -302,12 +302,12 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public List getTables(DatasourceRequest datasourceRequest) throws Exception { - List tables = new ArrayList<>(); + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tables = new ArrayList<>(); String queryStr = getTablesSql(datasourceRequest); try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) { while (resultSet.next()) { - tables.add(resultSet.getString(1)); + tables.add(getTableDesc(datasourceRequest, resultSet)); } } catch (Exception e) { DataEaseException.throwException(e); @@ -317,7 +317,7 @@ public class JdbcProvider extends DatasourceProvider { if (queryView != null) { try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryView)) { while (resultSet.next()) { - tables.add(resultSet.getString(1)); + tables.add(getTableDesc(datasourceRequest, resultSet)); } } catch (Exception e) { DataEaseException.throwException(e); @@ -327,6 +327,19 @@ public class JdbcProvider extends DatasourceProvider { return tables; } + private TableDesc getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException { + TableDesc tableDesc = new TableDesc(); + DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); + if (datasourceType == DatasourceTypes.oracle) { + tableDesc.setRemark(resultSet.getString(3)); + } + if (datasourceType == DatasourceTypes.mysql) { + tableDesc.setRemark(resultSet.getString(2)); + } + tableDesc.setName(resultSet.getString(1)); + return tableDesc; + } + @Override public List getSchema(DatasourceRequest datasourceRequest) throws Exception { List schemas = new ArrayList<>(); @@ -583,6 +596,8 @@ public class JdbcProvider extends DatasourceProvider { switch (datasourceType) { case mysql: case mariadb: + JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); + return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase()); case de_doris: case ds_doris: case hive: @@ -600,7 +615,7 @@ public class JdbcProvider extends DatasourceProvider { if (StringUtils.isEmpty(oracleConfiguration.getSchema())) { throw new Exception(Translator.get("i18n_schema_is_empty")); } - return "select table_name, owner from all_tables where owner='" + oracleConfiguration.getSchema() + "'"; + return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'TABLE'"; case pg: PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class); if (StringUtils.isEmpty(pgConfiguration.getSchema())) { @@ -649,7 +664,7 @@ public class JdbcProvider extends DatasourceProvider { if (StringUtils.isEmpty(oracleConfiguration.getSchema())) { throw new Exception(Translator.get("i18n_schema_is_empty")); } - return "select VIEW_NAME from all_views where owner='" + oracleConfiguration.getSchema() + "'"; + return "select table_name, owner, comments from all_tab_comments where owner='" + oracleConfiguration.getSchema() + "' AND table_type = 'VIEW'"; case pg: PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class); if (StringUtils.isEmpty(pgConfiguration.getSchema())) { diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 291a17c709..f0a7679475 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -6,6 +6,7 @@ import io.dataease.base.domain.*; import io.dataease.base.mapper.ChartViewMapper; import io.dataease.base.mapper.ext.ExtChartGroupMapper; import io.dataease.base.mapper.ext.ExtChartViewMapper; +import io.dataease.commons.constants.ColumnPermissionConstants; import io.dataease.commons.constants.CommonConstants; import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.utils.AuthUtils; @@ -27,6 +28,7 @@ import io.dataease.provider.query.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableUnionService; +import io.dataease.service.dataset.PermissionService; import io.dataease.service.datasource.DatasourceService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -41,6 +43,8 @@ import java.util.*; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc; + /** * @Author gin * @Date 2021/3/1 12:34 下午 @@ -61,6 +65,8 @@ public class ChartViewService { private ExtChartGroupMapper extChartGroupMapper; @Resource private DataSetTableUnionService dataSetTableUnionService; + @Resource + private PermissionService permissionService; //默认使用非公平 private ReentrantLock lock = new ReentrantLock(); @@ -187,6 +193,25 @@ public class ChartViewService { return calcData(view, request, request.isCache()); } + private void checkPermissions(List chartViewFieldDTOS, List fields, List desensitizationList, Boolean alowDesensitization) throws Exception{ + String filedName = ""; + for (ChartViewFieldBaseDTO chartViewFieldDTO : chartViewFieldDTOS) { + if(alowDesensitization){ + if (!fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) { + filedName = filedName + chartViewFieldDTO.getName() + " ,"; + } + }else { + if (desensitizationList.contains(chartViewFieldDTO.getDataeaseName()) || !fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()).contains(chartViewFieldDTO.getDataeaseName())) { + filedName = filedName + chartViewFieldDTO.getName() + " ,"; + } + } + } + filedName = filedName.endsWith(",") ? filedName.substring(0, filedName.length() - 1) : filedName; + if(StringUtils.isNotEmpty(filedName)){ + throw new Exception("以下字段没有权限: " + filedName); + } + } + public ChartViewDTO calcData(ChartViewDTO view, ChartExtRequest requestList, boolean cache) throws Exception { if (ObjectUtils.isEmpty(view)) { throw new RuntimeException(Translator.get("i18n_chart_delete")); @@ -208,39 +233,52 @@ public class ChartViewService { }.getType()); List drill = new Gson().fromJson(view.getDrillFields(), new TypeToken>() { }.getType()); - // 获取对应数据集行权限 + + DatasetTableField datasetTableFieldObj = DatasetTableField.builder().tableId(view.getTableId()).checked(Boolean.TRUE).build(); List fields = dataSetTableFieldsService.list(datasetTableFieldObj); DatasetTable datasetTable = dataSetTableService.get(view.getTableId()); - List permissionFields = dataSetTableService.getCustomFilters(fields, datasetTable, requestList.getUser()); + + //列权限 + List desensitizationList = new ArrayList<>(); + fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable, null); + checkPermissions(fieldCustomFilter, fields, desensitizationList, false); + //行权限 + List permissionFields = permissionService.getCustomFilters(fields, datasetTable, requestList.getUser()); fieldCustomFilter.addAll(permissionFields); for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) { ele.setField(dataSetTableFieldsService.get(ele.getId())); } - if (StringUtils.equalsIgnoreCase("text", view.getType()) - || StringUtils.equalsIgnoreCase("gauge", view.getType()) - || StringUtils.equalsIgnoreCase("liquid", view.getType())) { - xAxis = new ArrayList<>(); - if (CollectionUtils.isEmpty(yAxis)) { - ChartViewDTO dto = new ChartViewDTO(); - BeanUtils.copyBean(dto, view); - return dto; - } - } else if (StringUtils.equalsIgnoreCase("table-info", view.getType())) { - yAxis = new ArrayList<>(); - if (CollectionUtils.isEmpty(xAxis)) { - ChartViewDTO dto = new ChartViewDTO(); - BeanUtils.copyBean(dto, view); - return dto; - } - } else { - if (CollectionUtils.isEmpty(xAxis) && CollectionUtils.isEmpty(yAxis)) { - ChartViewDTO dto = new ChartViewDTO(); - BeanUtils.copyBean(dto, view); - return dto; - } + if (CollectionUtils.isEmpty(xAxis) && CollectionUtils.isEmpty(yAxis)) { + return emptyChartViewDTO(view); + } + + switch (view.getType()){ + case "text": + case "gauge": + case "liquid": + xAxis = new ArrayList<>(); + checkPermissions(yAxis, fields, desensitizationList, false); + if (CollectionUtils.isEmpty(yAxis)) { + return emptyChartViewDTO(view); + } + break; + case "table-info": + yAxis = new ArrayList<>(); + checkPermissions(xAxis, fields, desensitizationList, true); + if (CollectionUtils.isEmpty(xAxis)) { + return emptyChartViewDTO(view); + } + break; + case "table-normal": + checkPermissions(xAxis, fields, desensitizationList, true); + checkPermissions(yAxis, fields, desensitizationList, true); + break; + default: + checkPermissions(xAxis, fields, desensitizationList, false); + checkPermissions(yAxis, fields, desensitizationList, false); } // 过滤来自仪表板的条件 @@ -344,7 +382,7 @@ public class ChartViewService { if (ObjectUtils.isEmpty(ds)) { throw new RuntimeException(Translator.get("i18n_datasource_delete")); } - if(StringUtils.isNotEmpty(ds.getStatus()) && ds.getStatus().equalsIgnoreCase("Error")){ + if (StringUtils.isNotEmpty(ds.getStatus()) && ds.getStatus().equalsIgnoreCase("Error")) { throw new Exception(Translator.get("i18n_invalid_ds")); } DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); @@ -569,7 +607,7 @@ public class ChartViewService { } // table组件,明细表,也用于导出数据 - Map mapTableNormal = transTableNormal(xAxis, yAxis, view, data, extStack); + Map mapTableNormal = transTableNormal(xAxis, yAxis, view, data, extStack, desensitizationList); map.putAll(mapChart); map.putAll(mapTableNormal); @@ -587,6 +625,12 @@ public class ChartViewService { return dto; } + private ChartViewDTO emptyChartViewDTO(ChartViewDTO view) { + ChartViewDTO dto = new ChartViewDTO(); + BeanUtils.copyBean(dto, view); + return dto; + } + private boolean checkCalcType(String dateStyle, String calcType) { switch (dateStyle) { case "y": @@ -1510,7 +1554,7 @@ public class ChartViewService { } // 表格 - private Map transTableNormal(List xAxis, List yAxis, ChartViewWithBLOBs view, List data, List extStack) { + private Map transTableNormal(List xAxis, List yAxis, ChartViewWithBLOBs view, List data, List extStack, List desensitizationList) { Map map = new TreeMap<>(); List fields = new ArrayList<>(); List> tableRow = new ArrayList<>(); @@ -1526,9 +1570,14 @@ public class ChartViewService { data.forEach(ele -> { Map d = new HashMap<>(); for (int i = 0; i < fields.size(); i++) { + if(CollectionUtils.isNotEmpty(desensitizationList) && desensitizationList.contains(fields.get(i).getDataeaseName())){ + d.put(fields.get(i).getDataeaseName(), ColumnPermissionConstants.Desensitization_desc); + continue; + } + ChartViewFieldDTO chartViewFieldDTO = fields.get(i); if (chartViewFieldDTO.getDeType() == 0 || chartViewFieldDTO.getDeType() == 1) { - d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); + d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); } else if (chartViewFieldDTO.getDeType() == 2 || chartViewFieldDTO.getDeType() == 3) { d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? null : new BigDecimal(ele[i]).setScale(2, RoundingMode.HALF_UP)); } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java index 9b5c480810..4744c14637 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java @@ -91,6 +91,10 @@ public class DataSetTableFieldsService { return datasetTableFieldMapper.selectByExample(datasetTableFieldExample); } + public DatasetTableField selectByPrimaryKey(String id) { + return datasetTableFieldMapper.selectByPrimaryKey(id); + } + public List getListByIdsEach(List ids) { List list = new ArrayList<>(); if (CollectionUtils.isNotEmpty(ids)) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index b6eace9983..bac36183c6 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1,21 +1,12 @@ package io.dataease.service.dataset; - -import com.alibaba.fastjson.JSONObject; import com.google.gson.Gson; -import io.dataease.auth.api.dto.CurrentRoleDto; -import io.dataease.auth.api.dto.CurrentUserDto; -import io.dataease.auth.entity.SysUserEntity; -import io.dataease.auth.service.AuthUserService; import io.dataease.base.domain.*; import io.dataease.base.mapper.*; import io.dataease.base.mapper.ext.ExtDataSetGroupMapper; import io.dataease.base.mapper.ext.ExtDataSetTableMapper; import io.dataease.base.mapper.ext.UtilMapper; -import io.dataease.commons.constants.DatasourceTypes; -import io.dataease.commons.constants.JobStatus; -import io.dataease.commons.constants.ScheduleType; -import io.dataease.commons.constants.TaskStatus; +import io.dataease.commons.constants.*; import io.dataease.commons.exception.DEException; import io.dataease.commons.utils.*; import io.dataease.controller.request.dataset.DataSetGroupRequest; @@ -23,7 +14,6 @@ import io.dataease.controller.request.dataset.DataSetTableRequest; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.controller.response.DataSetDetail; -import io.dataease.dto.chart.ChartCustomFilterItemDTO; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.dto.dataset.*; import io.dataease.dto.dataset.union.UnionDTO; @@ -32,12 +22,7 @@ import io.dataease.dto.dataset.union.UnionParamDTO; import io.dataease.dto.datasource.TableFiled; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; -import io.dataease.plugins.config.SpringContextUtil; import io.dataease.plugins.loader.ClassloaderResponsity; -import io.dataease.plugins.xpack.auth.dto.request.DataSetRowPermissionsDTO; -import io.dataease.plugins.xpack.auth.dto.request.DatasetRowPermissions; -import io.dataease.plugins.xpack.auth.dto.response.XpackSysAuthDetailDTO; -import io.dataease.plugins.xpack.auth.service.RowPermissionService; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.datasource.JdbcProvider; @@ -72,7 +57,7 @@ import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; -; +;import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc; /** * @Author gin @@ -108,7 +93,7 @@ public class DataSetTableService { @Resource private DatasetTableFieldMapper datasetTableFieldMapper; @Resource - private AuthUserService authUserService; + private PermissionService permissionService; private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; @@ -448,103 +433,6 @@ public class DataSetTableService { return map; } - private List rowPermissions(String datasetId, Long userId) { - List datasetRowPermissions = new ArrayList<>(); - Map beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((RowPermissionService.class)); - if (beansOfType.keySet().size() == 0) { - return new ArrayList<>(); - } - RowPermissionService rowPermissionService = SpringContextUtil.getBean(RowPermissionService.class); - CurrentUserDto user = AuthUtils.getUser(); - List roleIds = new ArrayList<>(); - Long deptId = null; - - if (user == null && userId == null) { - return datasetRowPermissions; - } - - if (user != null && userId != null) { - return datasetRowPermissions; - } - - if (user != null) { - if (user.getIsAdmin()) { - return datasetRowPermissions; - } - userId = user.getUserId(); - deptId = user.getDeptId(); - roleIds = user.getRoles().stream().map(CurrentRoleDto::getId).collect(Collectors.toList()); - } - - if (userId != null) { - SysUserEntity userEntity = authUserService.getUserById(userId); - if (userEntity.getIsAdmin()) { - return datasetRowPermissions; - } - deptId = userEntity.getDeptId(); - roleIds = authUserService.roles(userId).stream().map(r -> Long.valueOf(r)).collect(Collectors.toList()); - } - - - DataSetRowPermissionsDTO dataSetRowPermissionsDTO = new DataSetRowPermissionsDTO(); - dataSetRowPermissionsDTO.setDatasetId(datasetId); - dataSetRowPermissionsDTO.setAuthTargetIds(Collections.singletonList(userId)); - dataSetRowPermissionsDTO.setAuthTargetType("user"); - datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); - dataSetRowPermissionsDTO.setAuthTargetIds(roleIds); - dataSetRowPermissionsDTO.setAuthTargetType("role"); - datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); - dataSetRowPermissionsDTO.setAuthTargetIds(Collections.singletonList(deptId)); - dataSetRowPermissionsDTO.setAuthTargetType("dept"); - datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); - return datasetRowPermissions; - } - - private DatasetTableField getFieldById(List fields, String fieldId) { - DatasetTableField field = null; - for (DatasetTableField datasetTableField : fields) { - if (fieldId.equalsIgnoreCase(datasetTableField.getId())) { - field = datasetTableField; - } - } - return field; - } - - public List getCustomFilters(List fields, DatasetTable datasetTable, Long user) { - List customFilter = new ArrayList<>(); - for (DatasetRowPermissions datasetRowPermissions : rowPermissions(datasetTable.getId(), user)) { - ChartFieldCustomFilterDTO dto = new ChartFieldCustomFilterDTO(); - if (StringUtils.isEmpty(datasetRowPermissions.getDatasetFieldId())) { - continue; - } - DatasetTableField field = getFieldById(fields, datasetRowPermissions.getDatasetFieldId()); - if (field == null) { - continue; - } - dto.setField(field); - dto.setId(field.getId()); - dto.setFilterType(datasetRowPermissions.getFilterType()); - if (datasetRowPermissions.getFilterType().equalsIgnoreCase("logic")) { - if (StringUtils.isEmpty(datasetRowPermissions.getFilter())) { - continue; - } - List lists = JSONObject.parseArray(datasetRowPermissions.getFilter(), ChartCustomFilterItemDTO.class); - lists.forEach(chartCustomFilterDTO -> { - chartCustomFilterDTO.setFieldId(field.getId()); - }); - dto.setFilter(lists); - dto.setLogic(datasetRowPermissions.getLogic()); - customFilter.add(dto); - } else { - if (StringUtils.isEmpty(datasetRowPermissions.getEnumCheckField())) { - continue; - } - dto.setEnumCheckField(Arrays.asList(datasetRowPermissions.getEnumCheckField().split(",").clone())); - customFilter.add(dto); - } - } - return customFilter; - } public Map getPreviewData(DataSetTableRequest dataSetTableRequest, Integer page, Integer pageSize, List extFields) throws Exception { Map map = new HashMap<>(); @@ -560,7 +448,11 @@ public class DataSetTableService { return map; } DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(dataSetTableRequest.getId()); - List customFilter = getCustomFilters(fields, datasetTable, null); + //列权限 + List desensitizationList = new ArrayList<>(); + fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable, null); + //行权限 + List customFilter = permissionService.getCustomFilters(fields, datasetTable, null); String[] fieldArray = fields.stream().map(DatasetTableField::getDataeaseName).toArray(String[]::new); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); @@ -873,7 +765,11 @@ public class DataSetTableService { jsonArray = data.stream().map(ele -> { Map tmpMap = new HashMap<>(); for (int i = 0; i < ele.length; i++) { - tmpMap.put(fieldArray[i], ele[i]); + if (desensitizationList.contains(fieldArray[i])) { + tmpMap.put(fieldArray[i], Desensitization_desc); + } else { + tmpMap.put(fieldArray[i], ele[i]); + } } return tmpMap; }).collect(Collectors.toList()); diff --git a/backend/src/main/java/io/dataease/service/dataset/PermissionService.java b/backend/src/main/java/io/dataease/service/dataset/PermissionService.java new file mode 100644 index 0000000000..f82898b949 --- /dev/null +++ b/backend/src/main/java/io/dataease/service/dataset/PermissionService.java @@ -0,0 +1,202 @@ +package io.dataease.service.dataset; + +import com.alibaba.fastjson.JSONObject; +import io.dataease.auth.api.dto.CurrentRoleDto; +import io.dataease.auth.api.dto.CurrentUserDto; +import io.dataease.auth.entity.SysUserEntity; +import io.dataease.auth.service.AuthUserService; +import io.dataease.base.domain.DatasetTable; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.commons.constants.ColumnPermissionConstants; +import io.dataease.commons.utils.AuthUtils; +import io.dataease.dto.chart.ChartCustomFilterItemDTO; +import io.dataease.dto.chart.ChartFieldCustomFilterDTO; +import io.dataease.plugins.config.SpringContextUtil; +import io.dataease.plugins.xpack.auth.dto.request.*; +import io.dataease.plugins.xpack.auth.service.ColumnPermissionService; +import io.dataease.plugins.xpack.auth.service.RowPermissionService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class PermissionService { + @Resource + private AuthUserService authUserService; + + public List getCustomFilters(List fields, DatasetTable datasetTable, Long user) { + List customFilter = new ArrayList<>(); + for (DatasetRowPermissions datasetRowPermissions : rowPermissions(datasetTable.getId(), user)) { + ChartFieldCustomFilterDTO dto = new ChartFieldCustomFilterDTO(); + if (StringUtils.isEmpty(datasetRowPermissions.getDatasetFieldId())) { + continue; + } + DatasetTableField field = getFieldById(fields, datasetRowPermissions.getDatasetFieldId()); + if (field == null) { + continue; + } + dto.setField(field); + dto.setId(field.getId()); + dto.setFilterType(datasetRowPermissions.getFilterType()); + if (datasetRowPermissions.getFilterType().equalsIgnoreCase("logic")) { + if (StringUtils.isEmpty(datasetRowPermissions.getFilter())) { + continue; + } + List lists = JSONObject.parseArray(datasetRowPermissions.getFilter(), ChartCustomFilterItemDTO.class); + lists.forEach(chartCustomFilterDTO -> { + chartCustomFilterDTO.setFieldId(field.getId()); + }); + dto.setFilter(lists); + dto.setLogic(datasetRowPermissions.getLogic()); + customFilter.add(dto); + } else { + if (StringUtils.isEmpty(datasetRowPermissions.getEnumCheckField())) { + continue; + } + dto.setEnumCheckField(Arrays.asList(datasetRowPermissions.getEnumCheckField().split(",").clone())); + customFilter.add(dto); + } + } + return customFilter; + } + + public List filterColumnPermissons(List fields, ListdesensitizationList, DatasetTable datasetTable, Long user){ + List result = new ArrayList<>(); + List allColumnPermissionItems = new ArrayList<>(); + for (DataSetColumnPermissionsDTO dataSetColumnPermissionsDTO : columnPermissions(datasetTable.getId(), user)) { + ColumnPermissions columnPermissions = JSONObject.parseObject(dataSetColumnPermissionsDTO.getPermissions(), ColumnPermissions.class); + if(!columnPermissions.getEnable()){continue;} + allColumnPermissionItems.addAll(columnPermissions.getColumns().stream().filter(columnPermissionItem -> columnPermissionItem.getSelected()).collect(Collectors.toList())); + } + fields.forEach(field ->{ + List permissions = allColumnPermissionItems.stream().filter(columnPermissionItem -> columnPermissionItem.getId().equalsIgnoreCase(field.getId())).map(ColumnPermissionItem::getOpt).collect(Collectors.toList()); + if(CollectionUtils.isEmpty(permissions)){ + result.add(field); + }else { + if(!permissions.contains(ColumnPermissionConstants.Prohibit)){ + desensitizationList.add(field.getDataeaseName()); + result.add(field); + } + } + }); + return result; + } + + + private List rowPermissions(String datasetId, Long userId) { + List datasetRowPermissions = new ArrayList<>(); + Map beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((RowPermissionService.class)); + if (beansOfType.keySet().size() == 0) { + return new ArrayList<>(); + } + RowPermissionService rowPermissionService = SpringContextUtil.getBean(RowPermissionService.class); + CurrentUserDto user = AuthUtils.getUser(); + List roleIds = new ArrayList<>(); + Long deptId = null; + + if (user == null && userId == null) { + return datasetRowPermissions; + } + + if (user != null && userId != null) { + return datasetRowPermissions; + } + + if (user != null) { + if (user.getIsAdmin()) { + return datasetRowPermissions; + } + userId = user.getUserId(); + deptId = user.getDeptId(); + roleIds = user.getRoles().stream().map(CurrentRoleDto::getId).collect(Collectors.toList()); + } + + if (userId != null) { + SysUserEntity userEntity = authUserService.getUserById(userId); + if (userEntity.getIsAdmin()) { + return datasetRowPermissions; + } + deptId = userEntity.getDeptId(); + roleIds = authUserService.roles(userId).stream().map(r -> Long.valueOf(r)).collect(Collectors.toList()); + } + + + DataSetRowPermissionsDTO dataSetRowPermissionsDTO = new DataSetRowPermissionsDTO(); + dataSetRowPermissionsDTO.setDatasetId(datasetId); + dataSetRowPermissionsDTO.setAuthTargetIds(Collections.singletonList(userId)); + dataSetRowPermissionsDTO.setAuthTargetType("user"); + datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); + dataSetRowPermissionsDTO.setAuthTargetIds(roleIds); + dataSetRowPermissionsDTO.setAuthTargetType("role"); + datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); + dataSetRowPermissionsDTO.setAuthTargetIds(Collections.singletonList(deptId)); + dataSetRowPermissionsDTO.setAuthTargetType("dept"); + datasetRowPermissions.addAll(rowPermissionService.searchRowPermissions(dataSetRowPermissionsDTO)); + return datasetRowPermissions; + } + + private List columnPermissions(String datasetId, Long userId) { + List datasetColumnPermissions = new ArrayList<>(); + Map beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((ColumnPermissionService.class)); + if (beansOfType.keySet().size() == 0) { + return new ArrayList<>(); + } + ColumnPermissionService columnPermissionService = SpringContextUtil.getBean(ColumnPermissionService.class); + CurrentUserDto user = AuthUtils.getUser(); + List roleIds = new ArrayList<>(); + Long deptId = null; + + if (user == null && userId == null) { + return datasetColumnPermissions; + } + + if (user != null && userId != null) { + return datasetColumnPermissions; + } + + if (user != null) { + if (user.getIsAdmin()) { + return datasetColumnPermissions; + } + userId = user.getUserId(); + deptId = user.getDeptId(); + roleIds = user.getRoles().stream().map(CurrentRoleDto::getId).collect(Collectors.toList()); + } + + if (userId != null) { + SysUserEntity userEntity = authUserService.getUserById(userId); + if (userEntity.getIsAdmin()) { + return datasetColumnPermissions; + } + deptId = userEntity.getDeptId(); + roleIds = authUserService.roles(userId).stream().map(r -> Long.valueOf(r)).collect(Collectors.toList()); + } + + DataSetColumnPermissionsDTO dataSetColumnPermissionsDTO = new DataSetColumnPermissionsDTO(); + dataSetColumnPermissionsDTO.setDatasetId(datasetId); + dataSetColumnPermissionsDTO.setAuthTargetIds(Collections.singletonList(userId)); + dataSetColumnPermissionsDTO.setAuthTargetType("user"); + datasetColumnPermissions.addAll(columnPermissionService.searchPermissions(dataSetColumnPermissionsDTO)); + dataSetColumnPermissionsDTO.setAuthTargetIds(roleIds); + dataSetColumnPermissionsDTO.setAuthTargetType("role"); + datasetColumnPermissions.addAll(columnPermissionService.searchPermissions(dataSetColumnPermissionsDTO)); + dataSetColumnPermissionsDTO.setAuthTargetIds(Collections.singletonList(deptId)); + dataSetColumnPermissionsDTO.setAuthTargetType("dept"); + datasetColumnPermissions.addAll(columnPermissionService.searchPermissions(dataSetColumnPermissionsDTO)); + return datasetColumnPermissions; + } + + private DatasetTableField getFieldById(List fields, String fieldId) { + DatasetTableField field = null; + for (DatasetTableField datasetTableField : fields) { + if (fieldId.equalsIgnoreCase(datasetTableField.getId())) { + field = datasetTableField; + } + } + return field; + } +} diff --git a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java index 99fae46150..16090f0b04 100644 --- a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java +++ b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java @@ -4,20 +4,18 @@ import com.google.gson.Gson; import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.Datasource; +import io.dataease.commons.constants.ColumnPermissionConstants; import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.dto.chart.ChartFieldCustomFilterDTO; import io.dataease.i18n.Translator; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.ProviderFactory; import io.dataease.controller.request.datasource.DatasourceRequest; +import io.dataease.service.dataset.*; import io.dataease.service.datasource.DatasourceService; import io.dataease.dto.dataset.DataSetTableUnionDTO; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.provider.query.QueryProvider; -import io.dataease.service.dataset.DataSetFieldService; -import io.dataease.service.dataset.DataSetTableFieldsService; -import io.dataease.service.dataset.DataSetTableService; -import io.dataease.service.dataset.DataSetTableUnionService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -40,32 +38,38 @@ public class DirectFieldService implements DataSetFieldService { private DatasourceService datasourceService; @Resource private DataSetTableUnionService dataSetTableUnionService; + @Resource + private PermissionService permissionService; @Override - public List fieldValues(String fieldId, Long userId) throws Exception{ - List list = dataSetTableFieldsService.getListByIds(new ArrayList() {{ - add(fieldId); - }}); - if (CollectionUtils.isEmpty(list)) return null; + public List fieldValues(String fieldId, Long userId) throws Exception { + DatasetTableField field = dataSetTableFieldsService.selectByPrimaryKey(fieldId); + if (field == null || StringUtils.isEmpty(field.getTableId())) return null; - DatasetTableField field = list.get(0); - String tableId = field.getTableId(); - if (StringUtils.isEmpty(tableId)) return null; - - DatasetTable datasetTable = dataSetTableService.get(tableId); + DatasetTable datasetTable = dataSetTableService.get(field.getTableId()); if (ObjectUtils.isEmpty(datasetTable) || StringUtils.isEmpty(datasetTable.getName())) return null; - String tableName; - DatasetTableField datasetTableField = DatasetTableField.builder().tableId(tableId).checked(Boolean.TRUE).build(); + DatasetTableField datasetTableField = DatasetTableField.builder().tableId(field.getTableId()).checked(Boolean.TRUE).build(); List fields = dataSetTableFieldsService.list(datasetTableField); - List customFilter = dataSetTableService.getCustomFilters(fields, datasetTable, userId); + + //列权限 + List desensitizationList = new ArrayList<>(); + fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable, userId); + + if (CollectionUtils.isNotEmpty(desensitizationList) && desensitizationList.contains(field.getDataeaseName())) { + List results = new ArrayList<>(); + results.add(ColumnPermissionConstants.Desensitization_desc); + return results; + } + //行权限 + List customFilter = permissionService.getCustomFilters(fields, datasetTable, userId); DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceProvider datasourceProvider = null; if (datasetTable.getMode() == 0) {// 直连 if (StringUtils.isEmpty(datasetTable.getDataSourceId())) return null; Datasource ds = datasourceService.get(datasetTable.getDataSourceId()); - if(StringUtils.isNotEmpty(ds.getStatus()) && ds.getStatus().equalsIgnoreCase("Error")){ + if (StringUtils.isNotEmpty(ds.getStatus()) && ds.getStatus().equalsIgnoreCase("Error")) { throw new Exception(Translator.get("i18n_invalid_ds")); } datasourceProvider = ProviderFactory.getProvider(ds.getType()); @@ -94,7 +98,7 @@ public class DirectFieldService implements DataSetFieldService { datasourceProvider = ProviderFactory.getProvider(ds.getType()); datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - tableName = "ds_" + datasetTable.getId().replaceAll("-", "_"); + String tableName = "ds_" + datasetTable.getId().replaceAll("-", "_"); datasourceRequest.setTable(tableName); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery(qp.createQuerySQL(tableName, Collections.singletonList(field), true, null, customFilter)); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index e900d106bd..c709074ebf 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -193,22 +193,23 @@ public class DatasourceService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); datasourceProvider.checkStatus(datasourceRequest); - List tables = datasourceProvider.getTables(datasourceRequest); + List tables = datasourceProvider.getTables(datasourceRequest); // 获取当前数据源下的db类型数据集 DatasetTableExample datasetTableExample = new DatasetTableExample(); datasetTableExample.createCriteria().andTypeEqualTo("db").andDataSourceIdEqualTo(datasource.getId()); List datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample); List list = new ArrayList<>(); - for (String name : tables) { + for (TableDesc tableDesc : tables) { DBTableDTO dbTableDTO = new DBTableDTO(); dbTableDTO.setDatasourceId(datasource.getId()); - dbTableDTO.setName(name); + dbTableDTO.setName(tableDesc.getName()); + dbTableDTO.setRemark(tableDesc.getRemark()); dbTableDTO.setEnableCheck(true); dbTableDTO.setDatasetPath(null); for (DatasetTable datasetTable : datasetTables) { DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); - if (StringUtils.equals(name, dataTableInfoDTO.getTable())) { + if (StringUtils.equals(tableDesc.getName(), dataTableInfoDTO.getTable())) { dbTableDTO.setEnableCheck(false); List parents = dataSetGroupService.getParents(datasetTable.getSceneId()); StringBuilder stringBuilder = new StringBuilder(); diff --git a/backend/src/main/resources/db/migration/V31__1.7.sql b/backend/src/main/resources/db/migration/V31__1.7.sql index 1828fc902c..e244d139aa 100644 --- a/backend/src/main/resources/db/migration/V31__1.7.sql +++ b/backend/src/main/resources/db/migration/V31__1.7.sql @@ -1,2 +1,13 @@ +CREATE TABLE `dataset_column_permissions` ( + `id` varchar(64) NOT NULL COMMENT 'File ID', + `auth_target_type` varchar(255) DEFAULT NULL COMMENT '权限类型:组织/角色/用户', + `auth_target_id` bigint(20) DEFAULT NULL COMMENT '权限对象ID', + `dataset_id` varchar(64) DEFAULT NULL COMMENT '数据集ID', + `permissions` longtext DEFAULT NULL COMMENT '权限', + `update_time` bigint(13) NULL DEFAULT NULL, + PRIMARY KEY (`id`) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci; + INSERT INTO `sys_menu` (`menu_id`, `pid`, `sub_count`, `type`, `title`, `name`, `component`, `menu_sort`, `icon`, `path`, `i_frame`, `cache`, `hidden`, `permission`, `create_by`, `update_by`, `create_time`, `update_time`) VALUES (61, 0, 0, 1, '首页', 'wizard', 'wizard/index', 0, '', '/wizard', b'1', b'0', b'0', NULL, NULL, NULL, NULL, 1614915491036); INSERT INTO `system_parameter` (`param_key`, `param_value`, `type`, `sort`) VALUES ('ui.openHomePage', 'true', 'boolean', 13); + diff --git a/backend/src/main/resources/generatorConfig.xml b/backend/src/main/resources/generatorConfig.xml index 5de60adbf5..e90877a423 100644 --- a/backend/src/main/resources/generatorConfig.xml +++ b/backend/src/main/resources/generatorConfig.xml @@ -60,6 +60,6 @@ - +
diff --git a/frontend/src/components/DeDrag/index.vue b/frontend/src/components/DeDrag/index.vue index 0c339f1b8f..8832a2c813 100644 --- a/frontend/src/components/DeDrag/index.vue +++ b/frontend/src/components/DeDrag/index.vue @@ -535,7 +535,7 @@ export default { return this.$store.state.curComponent }, curGap() { - return (this.canvasStyleData.panel.gap === 'yes' && this.element.auxiliaryMatrix && this.element.type !== 'custom') ? this.componentGap : 0 + return (this.canvasStyleData.panel.gap === 'yes' && this.element.auxiliaryMatrix) ? this.componentGap : 0 }, ...mapState([ 'editor', diff --git a/frontend/src/components/canvas/utils/utils.js b/frontend/src/components/canvas/utils/utils.js index f0ef0184bd..932e596c4b 100644 --- a/frontend/src/components/canvas/utils/utils.js +++ b/frontend/src/components/canvas/utils/utils.js @@ -74,6 +74,9 @@ export function panelInit(componentDatas) { } } } + if (item.type === 'custom') { + item.options.manualModify = false + } if (item.filters && item.filters.length > 0) { item.filters = [] } diff --git a/frontend/src/components/dataease/DeOutWidget.vue b/frontend/src/components/dataease/DeOutWidget.vue index 42ef2809cd..aefbd1878b 100644 --- a/frontend/src/components/dataease/DeOutWidget.vue +++ b/frontend/src/components/dataease/DeOutWidget.vue @@ -161,7 +161,7 @@ export default {