diff --git a/backend/pom.xml b/backend/pom.xml
index de5cb967ad..ed76e48968 100644
--- a/backend/pom.xml
+++ b/backend/pom.xml
@@ -438,48 +438,48 @@
true
-
+
-
- org.apache.maven.plugins
- maven-antrun-plugin
-
-
-
- main-class-placement
- generate-resources
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
+
-
-
-
- run
-
-
-
-
-
+
+
+
+
+
+
+
+
+
diff --git a/backend/src/main/java/io/dataease/base/domain/DeEngine.java b/backend/src/main/java/io/dataease/base/domain/DeEngine.java
new file mode 100644
index 0000000000..0849151d75
--- /dev/null
+++ b/backend/src/main/java/io/dataease/base/domain/DeEngine.java
@@ -0,0 +1,27 @@
+package io.dataease.base.domain;
+
+import java.io.Serializable;
+import lombok.Data;
+
+@Data
+public class DeEngine implements Serializable {
+ private String id;
+
+ private String name;
+
+ private String desc;
+
+ private String type;
+
+ private Long createTime;
+
+ private Long updateTime;
+
+ private String createBy;
+
+ private String status;
+
+ private String configuration;
+
+ private static final long serialVersionUID = 1L;
+}
\ No newline at end of file
diff --git a/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java b/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java
new file mode 100644
index 0000000000..243c047ddc
--- /dev/null
+++ b/backend/src/main/java/io/dataease/base/domain/DeEngineExample.java
@@ -0,0 +1,740 @@
+package io.dataease.base.domain;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DeEngineExample {
+ protected String orderByClause;
+
+ protected boolean distinct;
+
+ protected List oredCriteria;
+
+ public DeEngineExample() {
+ oredCriteria = new ArrayList();
+ }
+
+ public void setOrderByClause(String orderByClause) {
+ this.orderByClause = orderByClause;
+ }
+
+ public String getOrderByClause() {
+ return orderByClause;
+ }
+
+ public void setDistinct(boolean distinct) {
+ this.distinct = distinct;
+ }
+
+ public boolean isDistinct() {
+ return distinct;
+ }
+
+ public List getOredCriteria() {
+ return oredCriteria;
+ }
+
+ public void or(Criteria criteria) {
+ oredCriteria.add(criteria);
+ }
+
+ public Criteria or() {
+ Criteria criteria = createCriteriaInternal();
+ oredCriteria.add(criteria);
+ return criteria;
+ }
+
+ public Criteria createCriteria() {
+ Criteria criteria = createCriteriaInternal();
+ if (oredCriteria.size() == 0) {
+ oredCriteria.add(criteria);
+ }
+ return criteria;
+ }
+
+ protected Criteria createCriteriaInternal() {
+ Criteria criteria = new Criteria();
+ return criteria;
+ }
+
+ public void clear() {
+ oredCriteria.clear();
+ orderByClause = null;
+ distinct = false;
+ }
+
+ protected abstract static class GeneratedCriteria {
+ protected List criteria;
+
+ protected GeneratedCriteria() {
+ super();
+ criteria = new ArrayList();
+ }
+
+ public boolean isValid() {
+ return criteria.size() > 0;
+ }
+
+ public List getAllCriteria() {
+ return criteria;
+ }
+
+ public List getCriteria() {
+ return criteria;
+ }
+
+ protected void addCriterion(String condition) {
+ if (condition == null) {
+ throw new RuntimeException("Value for condition cannot be null");
+ }
+ criteria.add(new Criterion(condition));
+ }
+
+ protected void addCriterion(String condition, Object value, String property) {
+ if (value == null) {
+ throw new RuntimeException("Value for " + property + " cannot be null");
+ }
+ criteria.add(new Criterion(condition, value));
+ }
+
+ protected void addCriterion(String condition, Object value1, Object value2, String property) {
+ if (value1 == null || value2 == null) {
+ throw new RuntimeException("Between values for " + property + " cannot be null");
+ }
+ criteria.add(new Criterion(condition, value1, value2));
+ }
+
+ public Criteria andIdIsNull() {
+ addCriterion("id is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdIsNotNull() {
+ addCriterion("id is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdEqualTo(String value) {
+ addCriterion("id =", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdNotEqualTo(String value) {
+ addCriterion("id <>", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdGreaterThan(String value) {
+ addCriterion("id >", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdGreaterThanOrEqualTo(String value) {
+ addCriterion("id >=", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdLessThan(String value) {
+ addCriterion("id <", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdLessThanOrEqualTo(String value) {
+ addCriterion("id <=", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdLike(String value) {
+ addCriterion("id like", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdNotLike(String value) {
+ addCriterion("id not like", value, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdIn(List values) {
+ addCriterion("id in", values, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdNotIn(List values) {
+ addCriterion("id not in", values, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdBetween(String value1, String value2) {
+ addCriterion("id between", value1, value2, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andIdNotBetween(String value1, String value2) {
+ addCriterion("id not between", value1, value2, "id");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameIsNull() {
+ addCriterion("`name` is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameIsNotNull() {
+ addCriterion("`name` is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameEqualTo(String value) {
+ addCriterion("`name` =", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameNotEqualTo(String value) {
+ addCriterion("`name` <>", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameGreaterThan(String value) {
+ addCriterion("`name` >", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameGreaterThanOrEqualTo(String value) {
+ addCriterion("`name` >=", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameLessThan(String value) {
+ addCriterion("`name` <", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameLessThanOrEqualTo(String value) {
+ addCriterion("`name` <=", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameLike(String value) {
+ addCriterion("`name` like", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameNotLike(String value) {
+ addCriterion("`name` not like", value, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameIn(List values) {
+ addCriterion("`name` in", values, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameNotIn(List values) {
+ addCriterion("`name` not in", values, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameBetween(String value1, String value2) {
+ addCriterion("`name` between", value1, value2, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andNameNotBetween(String value1, String value2) {
+ addCriterion("`name` not between", value1, value2, "name");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescIsNull() {
+ addCriterion("`desc` is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescIsNotNull() {
+ addCriterion("`desc` is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescEqualTo(String value) {
+ addCriterion("`desc` =", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescNotEqualTo(String value) {
+ addCriterion("`desc` <>", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescGreaterThan(String value) {
+ addCriterion("`desc` >", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescGreaterThanOrEqualTo(String value) {
+ addCriterion("`desc` >=", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescLessThan(String value) {
+ addCriterion("`desc` <", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescLessThanOrEqualTo(String value) {
+ addCriterion("`desc` <=", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescLike(String value) {
+ addCriterion("`desc` like", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescNotLike(String value) {
+ addCriterion("`desc` not like", value, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescIn(List values) {
+ addCriterion("`desc` in", values, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescNotIn(List values) {
+ addCriterion("`desc` not in", values, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescBetween(String value1, String value2) {
+ addCriterion("`desc` between", value1, value2, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andDescNotBetween(String value1, String value2) {
+ addCriterion("`desc` not between", value1, value2, "desc");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeIsNull() {
+ addCriterion("`type` is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeIsNotNull() {
+ addCriterion("`type` is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeEqualTo(String value) {
+ addCriterion("`type` =", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeNotEqualTo(String value) {
+ addCriterion("`type` <>", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeGreaterThan(String value) {
+ addCriterion("`type` >", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeGreaterThanOrEqualTo(String value) {
+ addCriterion("`type` >=", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeLessThan(String value) {
+ addCriterion("`type` <", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeLessThanOrEqualTo(String value) {
+ addCriterion("`type` <=", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeLike(String value) {
+ addCriterion("`type` like", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeNotLike(String value) {
+ addCriterion("`type` not like", value, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeIn(List values) {
+ addCriterion("`type` in", values, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeNotIn(List values) {
+ addCriterion("`type` not in", values, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeBetween(String value1, String value2) {
+ addCriterion("`type` between", value1, value2, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andTypeNotBetween(String value1, String value2) {
+ addCriterion("`type` not between", value1, value2, "type");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeIsNull() {
+ addCriterion("create_time is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeIsNotNull() {
+ addCriterion("create_time is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeEqualTo(Long value) {
+ addCriterion("create_time =", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeNotEqualTo(Long value) {
+ addCriterion("create_time <>", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeGreaterThan(Long value) {
+ addCriterion("create_time >", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeGreaterThanOrEqualTo(Long value) {
+ addCriterion("create_time >=", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeLessThan(Long value) {
+ addCriterion("create_time <", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeLessThanOrEqualTo(Long value) {
+ addCriterion("create_time <=", value, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeIn(List values) {
+ addCriterion("create_time in", values, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeNotIn(List values) {
+ addCriterion("create_time not in", values, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeBetween(Long value1, Long value2) {
+ addCriterion("create_time between", value1, value2, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateTimeNotBetween(Long value1, Long value2) {
+ addCriterion("create_time not between", value1, value2, "createTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeIsNull() {
+ addCriterion("update_time is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeIsNotNull() {
+ addCriterion("update_time is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeEqualTo(Long value) {
+ addCriterion("update_time =", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeNotEqualTo(Long value) {
+ addCriterion("update_time <>", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeGreaterThan(Long value) {
+ addCriterion("update_time >", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeGreaterThanOrEqualTo(Long value) {
+ addCriterion("update_time >=", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeLessThan(Long value) {
+ addCriterion("update_time <", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeLessThanOrEqualTo(Long value) {
+ addCriterion("update_time <=", value, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeIn(List values) {
+ addCriterion("update_time in", values, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeNotIn(List values) {
+ addCriterion("update_time not in", values, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeBetween(Long value1, Long value2) {
+ addCriterion("update_time between", value1, value2, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andUpdateTimeNotBetween(Long value1, Long value2) {
+ addCriterion("update_time not between", value1, value2, "updateTime");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByIsNull() {
+ addCriterion("create_by is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByIsNotNull() {
+ addCriterion("create_by is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByEqualTo(String value) {
+ addCriterion("create_by =", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByNotEqualTo(String value) {
+ addCriterion("create_by <>", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByGreaterThan(String value) {
+ addCriterion("create_by >", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByGreaterThanOrEqualTo(String value) {
+ addCriterion("create_by >=", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByLessThan(String value) {
+ addCriterion("create_by <", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByLessThanOrEqualTo(String value) {
+ addCriterion("create_by <=", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByLike(String value) {
+ addCriterion("create_by like", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByNotLike(String value) {
+ addCriterion("create_by not like", value, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByIn(List values) {
+ addCriterion("create_by in", values, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByNotIn(List values) {
+ addCriterion("create_by not in", values, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByBetween(String value1, String value2) {
+ addCriterion("create_by between", value1, value2, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andCreateByNotBetween(String value1, String value2) {
+ addCriterion("create_by not between", value1, value2, "createBy");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusIsNull() {
+ addCriterion("`status` is null");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusIsNotNull() {
+ addCriterion("`status` is not null");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusEqualTo(String value) {
+ addCriterion("`status` =", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusNotEqualTo(String value) {
+ addCriterion("`status` <>", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusGreaterThan(String value) {
+ addCriterion("`status` >", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusGreaterThanOrEqualTo(String value) {
+ addCriterion("`status` >=", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusLessThan(String value) {
+ addCriterion("`status` <", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusLessThanOrEqualTo(String value) {
+ addCriterion("`status` <=", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusLike(String value) {
+ addCriterion("`status` like", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusNotLike(String value) {
+ addCriterion("`status` not like", value, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusIn(List values) {
+ addCriterion("`status` in", values, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusNotIn(List values) {
+ addCriterion("`status` not in", values, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusBetween(String value1, String value2) {
+ addCriterion("`status` between", value1, value2, "status");
+ return (Criteria) this;
+ }
+
+ public Criteria andStatusNotBetween(String value1, String value2) {
+ addCriterion("`status` not between", value1, value2, "status");
+ return (Criteria) this;
+ }
+ }
+
+ public static class Criteria extends GeneratedCriteria {
+
+ protected Criteria() {
+ super();
+ }
+ }
+
+ public static class Criterion {
+ private String condition;
+
+ private Object value;
+
+ private Object secondValue;
+
+ private boolean noValue;
+
+ private boolean singleValue;
+
+ private boolean betweenValue;
+
+ private boolean listValue;
+
+ private String typeHandler;
+
+ public String getCondition() {
+ return condition;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public Object getSecondValue() {
+ return secondValue;
+ }
+
+ public boolean isNoValue() {
+ return noValue;
+ }
+
+ public boolean isSingleValue() {
+ return singleValue;
+ }
+
+ public boolean isBetweenValue() {
+ return betweenValue;
+ }
+
+ public boolean isListValue() {
+ return listValue;
+ }
+
+ public String getTypeHandler() {
+ return typeHandler;
+ }
+
+ protected Criterion(String condition) {
+ super();
+ this.condition = condition;
+ this.typeHandler = null;
+ this.noValue = true;
+ }
+
+ protected Criterion(String condition, Object value, String typeHandler) {
+ super();
+ this.condition = condition;
+ this.value = value;
+ this.typeHandler = typeHandler;
+ if (value instanceof List) {
+ this.listValue = true;
+ } else {
+ this.singleValue = true;
+ }
+ }
+
+ protected Criterion(String condition, Object value) {
+ this(condition, value, null);
+ }
+
+ protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
+ super();
+ this.condition = condition;
+ this.value = value;
+ this.secondValue = secondValue;
+ this.typeHandler = typeHandler;
+ this.betweenValue = true;
+ }
+
+ protected Criterion(String condition, Object value, Object secondValue) {
+ this(condition, value, secondValue, null);
+ }
+ }
+}
\ No newline at end of file
diff --git a/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java
new file mode 100644
index 0000000000..2bcf37b094
--- /dev/null
+++ b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.java
@@ -0,0 +1,36 @@
+package io.dataease.base.mapper;
+
+import io.dataease.base.domain.DeEngine;
+import io.dataease.base.domain.DeEngineExample;
+import java.util.List;
+import org.apache.ibatis.annotations.Param;
+
+public interface DeEngineMapper {
+ long countByExample(DeEngineExample example);
+
+ int deleteByExample(DeEngineExample example);
+
+ int deleteByPrimaryKey(String id);
+
+ int insert(DeEngine record);
+
+ int insertSelective(DeEngine record);
+
+ List selectByExampleWithBLOBs(DeEngineExample example);
+
+ List selectByExample(DeEngineExample example);
+
+ DeEngine selectByPrimaryKey(String id);
+
+ int updateByExampleSelective(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
+
+ int updateByExampleWithBLOBs(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
+
+ int updateByExample(@Param("record") DeEngine record, @Param("example") DeEngineExample example);
+
+ int updateByPrimaryKeySelective(DeEngine record);
+
+ int updateByPrimaryKeyWithBLOBs(DeEngine record);
+
+ int updateByPrimaryKey(DeEngine record);
+}
\ No newline at end of file
diff --git a/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml
new file mode 100644
index 0000000000..dd37c69f8a
--- /dev/null
+++ b/backend/src/main/java/io/dataease/base/mapper/DeEngineMapper.xml
@@ -0,0 +1,323 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ and ${criterion.condition}
+
+
+ and ${criterion.condition} #{criterion.value}
+
+
+ and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+
+
+ and ${criterion.condition}
+
+ #{listItem}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ and ${criterion.condition}
+
+
+ and ${criterion.condition} #{criterion.value}
+
+
+ and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
+
+
+ and ${criterion.condition}
+
+ #{listItem}
+
+
+
+
+
+
+
+
+
+
+ id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`
+
+
+ configuration
+
+
+
+
+
+ delete from de_engine
+ where id = #{id,jdbcType=VARCHAR}
+
+
+ delete from de_engine
+
+
+
+
+
+ insert into de_engine (id, `name`, `desc`,
+ `type`, create_time, update_time,
+ create_by, `status`, configuration
+ )
+ values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR},
+ #{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT},
+ #{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR}
+ )
+
+
+ insert into de_engine
+
+
+ id,
+
+
+ `name`,
+
+
+ `desc`,
+
+
+ `type`,
+
+
+ create_time,
+
+
+ update_time,
+
+
+ create_by,
+
+
+ `status`,
+
+
+ configuration,
+
+
+
+
+ #{id,jdbcType=VARCHAR},
+
+
+ #{name,jdbcType=VARCHAR},
+
+
+ #{desc,jdbcType=VARCHAR},
+
+
+ #{type,jdbcType=VARCHAR},
+
+
+ #{createTime,jdbcType=BIGINT},
+
+
+ #{updateTime,jdbcType=BIGINT},
+
+
+ #{createBy,jdbcType=VARCHAR},
+
+
+ #{status,jdbcType=VARCHAR},
+
+
+ #{configuration,jdbcType=LONGVARCHAR},
+
+
+
+
+
+ update de_engine
+
+
+ id = #{record.id,jdbcType=VARCHAR},
+
+
+ `name` = #{record.name,jdbcType=VARCHAR},
+
+
+ `desc` = #{record.desc,jdbcType=VARCHAR},
+
+
+ `type` = #{record.type,jdbcType=VARCHAR},
+
+
+ create_time = #{record.createTime,jdbcType=BIGINT},
+
+
+ update_time = #{record.updateTime,jdbcType=BIGINT},
+
+
+ create_by = #{record.createBy,jdbcType=VARCHAR},
+
+
+ `status` = #{record.status,jdbcType=VARCHAR},
+
+
+ configuration = #{record.configuration,jdbcType=LONGVARCHAR},
+
+
+
+
+
+
+
+ update de_engine
+ set id = #{record.id,jdbcType=VARCHAR},
+ `name` = #{record.name,jdbcType=VARCHAR},
+ `desc` = #{record.desc,jdbcType=VARCHAR},
+ `type` = #{record.type,jdbcType=VARCHAR},
+ create_time = #{record.createTime,jdbcType=BIGINT},
+ update_time = #{record.updateTime,jdbcType=BIGINT},
+ create_by = #{record.createBy,jdbcType=VARCHAR},
+ `status` = #{record.status,jdbcType=VARCHAR},
+ configuration = #{record.configuration,jdbcType=LONGVARCHAR}
+
+
+
+
+
+ update de_engine
+ set id = #{record.id,jdbcType=VARCHAR},
+ `name` = #{record.name,jdbcType=VARCHAR},
+ `desc` = #{record.desc,jdbcType=VARCHAR},
+ `type` = #{record.type,jdbcType=VARCHAR},
+ create_time = #{record.createTime,jdbcType=BIGINT},
+ update_time = #{record.updateTime,jdbcType=BIGINT},
+ create_by = #{record.createBy,jdbcType=VARCHAR},
+ `status` = #{record.status,jdbcType=VARCHAR}
+
+
+
+
+
+ update de_engine
+
+
+ `name` = #{name,jdbcType=VARCHAR},
+
+
+ `desc` = #{desc,jdbcType=VARCHAR},
+
+
+ `type` = #{type,jdbcType=VARCHAR},
+
+
+ create_time = #{createTime,jdbcType=BIGINT},
+
+
+ update_time = #{updateTime,jdbcType=BIGINT},
+
+
+ create_by = #{createBy,jdbcType=VARCHAR},
+
+
+ `status` = #{status,jdbcType=VARCHAR},
+
+
+ configuration = #{configuration,jdbcType=LONGVARCHAR},
+
+
+ where id = #{id,jdbcType=VARCHAR}
+
+
+ update de_engine
+ set `name` = #{name,jdbcType=VARCHAR},
+ `desc` = #{desc,jdbcType=VARCHAR},
+ `type` = #{type,jdbcType=VARCHAR},
+ create_time = #{createTime,jdbcType=BIGINT},
+ update_time = #{updateTime,jdbcType=BIGINT},
+ create_by = #{createBy,jdbcType=VARCHAR},
+ `status` = #{status,jdbcType=VARCHAR},
+ configuration = #{configuration,jdbcType=LONGVARCHAR}
+ where id = #{id,jdbcType=VARCHAR}
+
+
+ update de_engine
+ set `name` = #{name,jdbcType=VARCHAR},
+ `desc` = #{desc,jdbcType=VARCHAR},
+ `type` = #{type,jdbcType=VARCHAR},
+ create_time = #{createTime,jdbcType=BIGINT},
+ update_time = #{updateTime,jdbcType=BIGINT},
+ create_by = #{createBy,jdbcType=VARCHAR},
+ `status` = #{status,jdbcType=VARCHAR}
+ where id = #{id,jdbcType=VARCHAR}
+
+
\ No newline at end of file
diff --git a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
index 1250a90a08..a28cbba4a4 100644
--- a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
+++ b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java
@@ -8,14 +8,15 @@ public enum DatasourceTypes {
ds_doris("ds_doris", "ds_doris", "com.mysql.jdbc.Driver", "`", "`", "'", "'"),
pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
- de_doris("de_doris", "de_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
mongo("mongo", "mongodb", "com.mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""),
ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""),
es("es", "es", "", "\"", "\"", "\"", "\""),
redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
- api("api", "api", "", "\"", "\"", "\"", "\"");
+ api("api", "api", "", "\"", "\"", "\"", "\""),
+ engine_doris("engine_doris", "engine_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
+ engine_mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'");
private String feature;
diff --git a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java b/backend/src/main/java/io/dataease/commons/utils/TableUtils.java
similarity index 54%
rename from backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java
rename to backend/src/main/java/io/dataease/commons/utils/TableUtils.java
index 154b5aadb4..b2bdfafb2b 100644
--- a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java
+++ b/backend/src/main/java/io/dataease/commons/utils/TableUtils.java
@@ -1,28 +1,28 @@
package io.dataease.commons.utils;
-public class DorisTableUtils {
+public class TableUtils {
- public static String dorisName(String datasetId) {
+ public static String tableName(String datasetId) {
return "ds_" + datasetId.replace("-", "_");
}
- public static String dorisTmpName(String dorisName) {
+ public static String tmpName(String dorisName) {
return "tmp_" + dorisName;
}
- public static String dorisDeleteName(String dorisName) {
+ public static String deleteName(String dorisName) {
return "delete_" + dorisName;
}
- public static String dorisAddName(String dorisName) {
+ public static String addName(String dorisName) {
return "add_" + dorisName;
}
- public static String dorisFieldName(String dorisName) {
+ public static String fieldName(String dorisName) {
return "f_" + Md5Utils.md5(dorisName);
}
- public static String dorisFieldNameShort(String dorisName) {
+ public static String fieldNameShort(String dorisName) {
return "f_" + Md5Utils.md5(dorisName).substring(8, 24);
}
diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java
index 0792dd0f46..77979c0308 100644
--- a/backend/src/main/java/io/dataease/config/CommonConfig.java
+++ b/backend/src/main/java/io/dataease/config/CommonConfig.java
@@ -23,28 +23,6 @@ public class CommonConfig {
private Environment env; // 保存了配置文件的信息
private static String root_path = "/opt/dataease/data/kettle/";
- @Bean(name = "DorisDatasource")
- @ConditionalOnMissingBean
- public Datasource configuration() {
- JSONObject jsonObject = new JSONObject();
- jsonObject.put("dataSourceType", "jdbc");
- jsonObject.put("dataBase", env.getProperty("doris.db", "doris"));
- jsonObject.put("username", env.getProperty("doris.user", "root"));
- jsonObject.put("password", env.getProperty("doris.password", "dataease"));
- jsonObject.put("host", env.getProperty("doris.host", "doris"));
- jsonObject.put("port", env.getProperty("doris.port", "9030"));
- jsonObject.put("httpPort", env.getProperty("doris.httpPort", "8030"));
-
- Datasource datasource = new Datasource();
- datasource.setId("doris");
- datasource.setName("doris");
- datasource.setDesc("doris");
- datasource.setType("de_doris");
- datasource.setConfiguration(jsonObject.toJSONString());
- return datasource;
- }
-
-
@Bean
@ConditionalOnMissingBean
public KettleFileRepository kettleFileRepository() throws Exception {
diff --git a/backend/src/main/java/io/dataease/controller/engine/EngineController.java b/backend/src/main/java/io/dataease/controller/engine/EngineController.java
new file mode 100644
index 0000000000..c032534158
--- /dev/null
+++ b/backend/src/main/java/io/dataease/controller/engine/EngineController.java
@@ -0,0 +1,45 @@
+package io.dataease.controller.engine;
+
+import io.dataease.base.domain.DeEngine;
+import io.dataease.controller.ResultHolder;
+import io.dataease.dto.DatasourceDTO;
+import io.dataease.service.engine.EngineService;
+import org.springframework.web.bind.annotation.*;
+import springfox.documentation.annotations.ApiIgnore;
+
+import javax.annotation.Resource;
+
+@ApiIgnore
+@RequestMapping("engine")
+@RestController
+public class EngineController {
+
+ @Resource
+ private EngineService engineService;
+
+ @ApiIgnore
+ @GetMapping("/mode")
+ public String runMode() throws Exception{
+ return engineService.mode();
+ }
+
+ @ApiIgnore
+ @GetMapping("/info")
+ public DeEngine info() throws Exception{
+ return engineService.info();
+ }
+
+ @ApiIgnore
+ @PostMapping("/validate")
+ public ResultHolder validate(@RequestBody DatasourceDTO datasource) throws Exception {
+ return engineService.validate(datasource);
+ }
+
+
+ @ApiIgnore
+ @PostMapping("/save")
+ public ResultHolder save(@RequestBody DeEngine engine) throws Exception {
+ return engineService.save(engine);
+ }
+
+}
diff --git a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
index 453972e4dd..8918c488b8 100644
--- a/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
+++ b/backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java
@@ -20,5 +20,8 @@ public class DataSourceInitStartListener implements ApplicationListener datasetTableFields);
+
+ public abstract String insertSql(String name, List dataList, int page, int pageNumber);
+}
diff --git a/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java
new file mode 100644
index 0000000000..ada0dd0294
--- /dev/null
+++ b/backend/src/main/java/io/dataease/provider/DDLProviderImpl.java
@@ -0,0 +1,50 @@
+package io.dataease.provider;
+
+import io.dataease.base.domain.DatasetTableField;
+import io.dataease.commons.utils.Md5Utils;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class DDLProviderImpl extends DDLProvider {
+ @Override
+ public String createView(String name, String viewSQL) {
+ return null;
+ }
+
+ @Override
+ public String dropTable(String name) {
+ return null;
+ }
+
+ @Override
+ public String dropView(String name) {
+ return null;
+ }
+
+ @Override
+ public String replaceTable(String name) {
+ return null;
+ }
+
+ @Override
+ public String createTableSql(String name, List datasetTableFields) {
+ return null;
+ }
+
+ @Override
+ public String insertSql(String name, List dataList, int page, int pageNumber) {
+ String insertSql = "INSERT INTO TABLE_NAME VALUES ".replace("TABLE_NAME", name);
+ StringBuffer values = new StringBuffer();
+
+ Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
+ for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
+ values.append("(").append(Md5Utils.md5(String.join(",", Arrays.asList(strings))))
+ .append("," ).append(String.join(",", Arrays.asList(strings)))
+ .append("),");
+ }
+ return insertSql + values.substring(0, values.length() - 1);
+ }
+
+
+}
diff --git a/backend/src/main/java/io/dataease/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/provider/ProviderFactory.java
index b901fbb02b..3dc13a582b 100644
--- a/backend/src/main/java/io/dataease/provider/ProviderFactory.java
+++ b/backend/src/main/java/io/dataease/provider/ProviderFactory.java
@@ -2,8 +2,6 @@ package io.dataease.provider;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.provider.datasource.DatasourceProvider;
-import io.dataease.provider.query.DDLProvider;
-import io.dataease.provider.query.QueryProvider;
import io.dataease.provider.query.api.ApiProvider;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@@ -40,8 +38,6 @@ public class ProviderFactory implements ApplicationContextAware {
case mariadb:
case ds_doris:
return context.getBean("mysqlQuery", QueryProvider.class);
- case de_doris:
- return context.getBean("dorisQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
@@ -62,6 +58,10 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", ApiProvider.class);
+ case engine_doris:
+ return context.getBean("dorisEngineQuery", QueryProvider.class);
+ case engine_mysql:
+ return context.getBean("mysqlEngineQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}
@@ -70,10 +70,12 @@ public class ProviderFactory implements ApplicationContextAware {
public static DDLProvider getDDLProvider(String type) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
- case de_doris:
- return context.getBean("dorisDDL", DDLProvider.class);
+ case engine_doris:
+ return context.getBean("dorisEngineDDL", DDLProvider.class);
+ case engine_mysql:
+ return context.getBean("mysqlEngineDDL", DDLProvider.class);
default:
- return context.getBean("dorisDDL", DDLProvider.class);
+ return context.getBean("dorisEngineDDL", DDLProvider.class);
}
}
diff --git a/backend/src/main/java/io/dataease/provider/query/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java
similarity index 99%
rename from backend/src/main/java/io/dataease/provider/query/QueryProvider.java
rename to backend/src/main/java/io/dataease/provider/QueryProvider.java
index 8716317f0b..c7eca21213 100644
--- a/backend/src/main/java/io/dataease/provider/query/QueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java
@@ -1,4 +1,4 @@
-package io.dataease.provider.query;
+package io.dataease.provider;
import com.google.gson.Gson;
import io.dataease.base.domain.ChartViewWithBLOBs;
diff --git a/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java b/backend/src/main/java/io/dataease/provider/QueryProviderImpl.java
similarity index 99%
rename from backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java
rename to backend/src/main/java/io/dataease/provider/QueryProviderImpl.java
index 548713b458..7d8083e2ee 100644
--- a/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java
+++ b/backend/src/main/java/io/dataease/provider/QueryProviderImpl.java
@@ -1,4 +1,4 @@
-package io.dataease.provider.query;
+package io.dataease.provider;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
diff --git a/backend/src/main/java/io/dataease/provider/query/SQLConstants.java b/backend/src/main/java/io/dataease/provider/SQLConstants.java
similarity index 96%
rename from backend/src/main/java/io/dataease/provider/query/SQLConstants.java
rename to backend/src/main/java/io/dataease/provider/SQLConstants.java
index e7174b5211..3fbe98f655 100644
--- a/backend/src/main/java/io/dataease/provider/query/SQLConstants.java
+++ b/backend/src/main/java/io/dataease/provider/SQLConstants.java
@@ -1,4 +1,4 @@
-package io.dataease.provider.query;
+package io.dataease.provider;
import java.util.ArrayList;
import java.util.List;
diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
index 1c8cd2a827..d135676871 100644
--- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
+++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
@@ -11,7 +11,7 @@ import io.dataease.dto.datasource.*;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.provider.ProviderFactory;
-import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.QueryProvider;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@@ -226,7 +226,7 @@ public class JdbcProvider extends DatasourceProvider {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) {
case mysql:
- case de_doris:
+ case engine_doris:
case ds_doris:
case mariadb:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
@@ -425,7 +425,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
- case de_doris:
+ case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
username = mysqlConfiguration.getUsername();
@@ -529,7 +529,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
- case de_doris:
+ case engine_doris:
case ds_doris:
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class);
dataSource.setUrl(mysqlConfiguration.getJdbc());
@@ -607,7 +607,7 @@ public class JdbcProvider extends DatasourceProvider {
case mariadb:
JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class);
return String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", jdbcConfiguration.getDataBase());
- case de_doris:
+ case engine_doris:
case ds_doris:
case hive:
return "show tables";
@@ -656,7 +656,7 @@ public class JdbcProvider extends DatasourceProvider {
switch (datasourceType) {
case mysql:
case mariadb:
- case de_doris:
+ case engine_doris:
case ds_doris:
case ck:
return null;
diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java
similarity index 74%
rename from backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java
rename to backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java
index fec288dee2..2305748abd 100644
--- a/backend/src/main/java/io/dataease/provider/query/doris/DorisConstants.java
+++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisConstants.java
@@ -1,17 +1,17 @@
-package io.dataease.provider.query.doris;
+package io.dataease.provider.engine.doris;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
-import static io.dataease.commons.constants.DatasourceTypes.de_doris;
+import static io.dataease.commons.constants.DatasourceTypes.engine_doris;
/**
* @Author gin
* @Date 2021/7/8 7:22 下午
*/
public class DorisConstants extends SQLConstants {
- public static final String KEYWORD_TABLE = de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
+ public static final String KEYWORD_TABLE = engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
- public static final String KEYWORD_FIX = "%s." + de_doris.getKeywordPrefix() + "%s" + de_doris.getKeywordSuffix();
+ public static final String KEYWORD_FIX = "%s." + engine_doris.getKeywordPrefix() + "%s" + engine_doris.getKeywordSuffix();
public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
diff --git a/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java
new file mode 100644
index 0000000000..1d9965e2ca
--- /dev/null
+++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisDDLProvider.java
@@ -0,0 +1,85 @@
+package io.dataease.provider.engine.doris;
+
+import io.dataease.base.domain.DatasetTableField;
+import io.dataease.commons.utils.TableUtils;
+import io.dataease.provider.DDLProviderImpl;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @Author gin
+ * @Date 2021/5/17 4:27 下午
+ */
+@Service("dorisEngineDDL")
+public class DorisDDLProvider extends DDLProviderImpl {
+ private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
+ "Column_Fields" +
+ "UNIQUE KEY(dataease_uuid)\n" +
+ "DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
+ "PROPERTIES(\"replication_num\" = \"1\");";
+
+ @Override
+ public String createView(String name, String viewSQL) {
+ return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
+ }
+
+ @Override
+ public String dropTable(String name) {
+ return "DROP TABLE IF EXISTS " + name;
+ }
+
+ @Override
+ public String dropView(String name) {
+ return "DROP VIEW IF EXISTS " + name;
+ }
+
+ @Override
+ public String replaceTable(String name){
+ return "ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false')"
+ .replace("DORIS_TABLE", name).replace("DORIS_TMP_TABLE", TableUtils.tmpName(name));
+ }
+
+ @Override
+ public String createTableSql(String tableName, List datasetTableFields) {
+ String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
+ return creatTableSql.replace("TABLE_NAME", tableName).replace("Column_Fields", dorisTableColumnSql);
+ }
+
+ private String createDorisTableColumnSql(final List datasetTableFields) {
+ StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
+ for (DatasetTableField datasetTableField : datasetTableFields) {
+ Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
+ Integer size = datasetTableField.getSize() * 3;
+ if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
+ size = 65533;
+ }
+ switch (datasetTableField.getDeExtractType()) {
+ case 0:
+ Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
+ break;
+ case 1:
+ size = size < 50? 50 : size;
+ Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
+ break;
+ case 2:
+ Column_Fields.append("bigint").append(",`");
+ break;
+ case 3:
+ Column_Fields.append("DOUBLE").append(",`");
+ break;
+ case 4:
+ Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
+ break;
+ default:
+ Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
+ break;
+ }
+ }
+
+ Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
+ Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
+ return Column_Fields.toString();
+ }
+
+}
diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
similarity index 99%
rename from backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java
rename to backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
index 4e2a59f65d..22b807936b 100644
--- a/backend/src/main/java/io/dataease/provider/query/doris/DorisQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
@@ -1,4 +1,4 @@
-package io.dataease.provider.query.doris;
+package io.dataease.provider.engine.doris;
import io.dataease.base.domain.ChartViewWithBLOBs;
import io.dataease.base.domain.DatasetTableField;
@@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -28,13 +28,13 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
* @Date 2021/5/17 4:11 下午
*/
-@Service("dorisQuery")
+@Service("dorisEngineQuery")
public class DorisQueryProvider extends QueryProvider {
@Resource
private DatasetTableFieldMapper datasetTableFieldMapper;
diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java
new file mode 100644
index 0000000000..d2590b85f8
--- /dev/null
+++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlConstants.java
@@ -0,0 +1,49 @@
+package io.dataease.provider.engine.mysql;
+
+import io.dataease.provider.SQLConstants;
+
+import static io.dataease.commons.constants.DatasourceTypes.engine_mysql;
+
+/**
+ * @Author gin
+ * @Date 2021/7/8 7:22 下午
+ */
+public class MysqlConstants extends SQLConstants {
+ public static final String KEYWORD_TABLE = engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
+
+ public static final String KEYWORD_FIX = "%s." + engine_mysql.getKeywordPrefix() + "%s" + engine_mysql.getKeywordSuffix();
+
+ public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)";
+
+ public static final String DATE_FORMAT = "DATE_FORMAT(%s,'%s')";
+
+ public static final String FROM_UNIXTIME = "FROM_UNIXTIME(%s,'%s')";
+
+ public static final String STR_TO_DATE = "STR_TO_DATE(%s,'%s')";
+
+ public static final String CAST = "CAST(%s AS %s)";
+
+ public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%S";
+
+ public static final String DEFAULT_INT_FORMAT = "BIGINT";
+
+ public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)";
+
+ public static final String WHERE_VALUE_NULL = "(NULL,'')";
+
+ public static final String WHERE_VALUE_VALUE = "'%s'";
+
+ public static final String WHERE_NUMBER_VALUE = "%s";
+
+ public static final String AGG_COUNT = "COUNT(*)";
+
+ public static final String AGG_FIELD = "%s(%s)";
+
+ public static final String WHERE_BETWEEN = "'%s' AND '%s'";
+
+ public static final String BRACKETS = "(%s)";
+
+ public static final String ROUND = "ROUND(%s,%s)";
+
+ public static final String VARCHAR = "VARCHAR";
+}
diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
new file mode 100644
index 0000000000..61e7168d26
--- /dev/null
+++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlDDLProvider.java
@@ -0,0 +1,35 @@
+package io.dataease.provider.engine.mysql;
+
+import io.dataease.commons.utils.TableUtils;
+import io.dataease.provider.DDLProviderImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * @Author gin
+ * @Date 2021/5/17 4:27 下午
+ */
+@Service("mysqlEngineDDL")
+public class MysqlDDLProvider extends DDLProviderImpl {
+ @Override
+ public String createView(String name, String viewSQL) {
+ return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
+ }
+
+ @Override
+ public String dropTable(String name) {
+ return "DROP TABLE IF EXISTS " + name;
+ }
+
+ @Override
+ public String dropView(String name) {
+ return "DROP VIEW IF EXISTS " + name;
+ }
+
+ @Override
+ public String replaceTable(String name){
+ String replaceTableSql = "rename table FROM_TABLE to FROM_TABLE_tmp, TO_TABLE to FROM_TABLE, FROM_TABLE_tmp to TO_TABLE; "
+ .replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
+ String dropTableSql = "DROP TABLE IF EXISTS " + TableUtils.tmpName(name);
+ return replaceTableSql + ";" + dropTableSql;
+ }
+}
diff --git a/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java
new file mode 100644
index 0000000000..9909543e8e
--- /dev/null
+++ b/backend/src/main/java/io/dataease/provider/engine/mysql/MysqlQueryProvider.java
@@ -0,0 +1,1107 @@
+package io.dataease.provider.engine.mysql;
+
+import io.dataease.base.domain.ChartViewWithBLOBs;
+import io.dataease.base.domain.DatasetTableField;
+import io.dataease.base.domain.DatasetTableFieldExample;
+import io.dataease.base.domain.Datasource;
+import io.dataease.base.mapper.DatasetTableFieldMapper;
+import io.dataease.controller.request.chart.ChartExtFilterRequest;
+import io.dataease.dto.chart.ChartCustomFilterItemDTO;
+import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
+import io.dataease.dto.chart.ChartViewFieldDTO;
+import io.dataease.dto.sqlObj.SQLObj;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import org.stringtemplate.v4.ST;
+import org.stringtemplate.v4.STGroup;
+import org.stringtemplate.v4.STGroupFile;
+
+import javax.annotation.Resource;
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
+
+/**
+ * @Author gin
+ * @Date 2021/5/17 4:11 下午
+ */
+@Service("mysqlEngineQuery")
+public class MysqlQueryProvider extends QueryProvider {
+ @Resource
+ private DatasetTableFieldMapper datasetTableFieldMapper;
+
+ @Override
+ public Integer transFieldType(String field) {
+ switch (field) {
+ case "CHAR":
+ case "VARCHAR":
+ case "TEXT":
+ case "TINYTEXT":
+ case "MEDIUMTEXT":
+ case "LONGTEXT":
+ case "ENUM":
+ return 0;// 文本
+ case "DATE":
+ case "TIME":
+ case "YEAR":
+ case "DATETIME":
+ case "TIMESTAMP":
+ return 1;// 时间
+ case "INT":
+ case "SMALLINT":
+ case "MEDIUMINT":
+ case "INTEGER":
+ case "BIGINT":
+ return 2;// 整型
+ case "FLOAT":
+ case "DOUBLE":
+ case "DECIMAL":
+ return 3;// 浮点
+ case "BIT":
+ case "TINYINT":
+ return 4;// 布尔
+ default:
+ return 0;
+ }
+ }
+
+ @Override
+ public String createSQLPreview(String sql, String orderBy) {
+ return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp LIMIT 0,1000";
+ }
+
+ @Override
+ public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter) {
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List xFields = new ArrayList<>();
+ List xOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(fields)) {
+ for (int i = 0; i < fields.size(); i++) {
+ DatasetTableField f = fields.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(f.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
+ String fieldName = "";
+ // 处理横轴字段
+ if (f.getDeExtractType() == 1) {
+ if (f.getDeType() == 2 || f.getDeType() == 3) {
+ fieldName = String.format(MysqlConstants.UNIX_TIMESTAMP, originField) + "*1000";
+ } else {
+ fieldName = originField;
+ }
+ } else if (f.getDeExtractType() == 0) {
+ if (f.getDeType() == 2) {
+ fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT);
+ } else if (f.getDeType() == 3) {
+ fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ } else if (f.getDeType() == 1) {
+ fieldName = String.format(MysqlConstants.STR_TO_DATE, originField, MysqlConstants.DEFAULT_DATE_FORMAT);
+ } else {
+ fieldName = originField;
+ }
+ } else {
+ if (f.getDeType() == 1) {
+ String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000";
+ fieldName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT);
+ } else if (f.getDeType() == 2) {
+ fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT);
+ } else if (f.getDeType() == 3) {
+ fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ } else {
+ fieldName = originField;
+ }
+ }
+ xFields.add(SQLObj.builder()
+ .fieldName(fieldName)
+ .fieldAlias(fieldAlias)
+ .build());
+ }
+ }
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("previewSql");
+ st_sql.add("isGroup", isGroup);
+ if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+
+ return st_sql.render();
+ }
+
+ @Override
+ public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup, List fieldCustomFilter) {
+ return createQuerySQL("(" + sql + ")", fields, isGroup, null, fieldCustomFilter);
+ }
+
+ @Override
+ public String createQueryTableWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List fieldCustomFilter) {
+ return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
+ }
+
+ @Override
+ public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds, List fieldCustomFilter) {
+ return createQuerySQL(table, fields, isGroup, null, fieldCustomFilter) + " LIMIT 0," + limit;
+ }
+
+ @Override
+ public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup, List fieldCustomFilter) {
+ return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT 0," + limit;
+ }
+
+ @Override
+ public String createQuerySQLWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List fieldCustomFilter) {
+ return createQuerySQLAsTmp(sql, fields, isGroup, fieldCustomFilter) + " LIMIT " + (page - 1) * pageSize + "," + realSize;
+ }
+
+ @Override
+ public String getSQL(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List xFields = new ArrayList<>();
+ List xOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(xAxis)) {
+ for (int i = 0; i < xAxis.size(); i++) {
+ ChartViewFieldDTO x = xAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(x.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
+ // 处理横轴字段
+ xFields.add(getXFields(x, originField, fieldAlias));
+
+ // 处理横轴排序
+ if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
+ xOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(x.getSort())
+ .build());
+ }
+ }
+ }
+ List yFields = new ArrayList<>();
+ List yWheres = new ArrayList<>();
+ List yOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(yAxis)) {
+ for (int i = 0; i < yAxis.size(); i++) {
+ ChartViewFieldDTO y = yAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(y.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
+ // 处理纵轴字段
+ yFields.add(getYFields(y, originField, fieldAlias));
+ // 处理纵轴过滤
+ yWheres.add(getYWheres(y, originField, fieldAlias));
+ // 处理纵轴排序
+ if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
+ yOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(y.getSort())
+ .build());
+ }
+ }
+ }
+ // 处理视图中字段过滤
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ // 处理仪表板字段过滤
+ String extWheres = transExtFilterList(tableObj, extFilterRequestList);
+ // 构建sql所有参数
+ List fields = new ArrayList<>();
+ fields.addAll(xFields);
+ fields.addAll(yFields);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (extWheres != null) wheres.add(extWheres);
+ List groups = new ArrayList<>();
+ groups.addAll(xFields);
+ // 外层再次套sql
+ List orders = new ArrayList<>();
+ orders.addAll(xOrders);
+ orders.addAll(yOrders);
+ List aggWheres = new ArrayList<>();
+ aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("querySql");
+ if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
+ if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String sql = st_sql.render();
+
+ ST st = stg.getInstanceOf("querySql");
+ SQLObj tableSQL = SQLObj.builder()
+ .tableName(String.format(MysqlConstants.BRACKETS, sql))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
+ .build();
+ if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
+ if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
+ if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
+ return sqlLimit(st.render(), view);
+ }
+
+ @Override
+ public String getSQLTableInfo(String table, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List xFields = new ArrayList<>();
+ List xOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(xAxis)) {
+ for (int i = 0; i < xAxis.size(); i++) {
+ ChartViewFieldDTO x = xAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(x.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
+ // 处理横轴字段
+ xFields.add(getXFields(x, originField, fieldAlias));
+
+ // 处理横轴排序
+ if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
+ xOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(x.getSort())
+ .build());
+ }
+ }
+ }
+ // 处理视图中字段过滤
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ // 处理仪表板字段过滤
+ String extWheres = transExtFilterList(tableObj, extFilterRequestList);
+ // 构建sql所有参数
+ List fields = new ArrayList<>();
+ fields.addAll(xFields);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (extWheres != null) wheres.add(extWheres);
+ List groups = new ArrayList<>();
+ groups.addAll(xFields);
+ // 外层再次套sql
+ List orders = new ArrayList<>();
+ orders.addAll(xOrders);
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("previewSql");
+ st_sql.add("isGroup", false);
+ if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String sql = st_sql.render();
+
+ ST st = stg.getInstanceOf("previewSql");
+ st.add("isGroup", false);
+ SQLObj tableSQL = SQLObj.builder()
+ .tableName(String.format(MysqlConstants.BRACKETS, sql))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
+ .build();
+ if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
+ if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
+ return sqlLimit(st.render(), view);
+ }
+
+ @Override
+ public String getSQLAsTmpTableInfo(String sql, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+ return getSQLTableInfo("(" + sqlFix(sql) + ")", xAxis, fieldCustomFilter, extFilterRequestList, null, view);
+ }
+
+ @Override
+ public String getSQLAsTmp(String sql, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) {
+ return getSQL("(" + sql + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, null, view);
+ }
+
+ @Override
+ public String getSQLStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, Datasource ds, ChartViewWithBLOBs view) {
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List xFields = new ArrayList<>();
+ List xOrders = new ArrayList<>();
+ List xList = new ArrayList<>();
+ xList.addAll(xAxis);
+ xList.addAll(extStack);
+ if (CollectionUtils.isNotEmpty(xList)) {
+ for (int i = 0; i < xList.size(); i++) {
+ ChartViewFieldDTO x = xList.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(x.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
+ // 处理横轴字段
+ xFields.add(getXFields(x, originField, fieldAlias));
+
+ // 处理横轴排序
+ if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
+ xOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(x.getSort())
+ .build());
+ }
+ }
+ }
+ List yFields = new ArrayList<>();
+ List yWheres = new ArrayList<>();
+ List yOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(yAxis)) {
+ for (int i = 0; i < yAxis.size(); i++) {
+ ChartViewFieldDTO y = yAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(y.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
+ // 处理纵轴字段
+ yFields.add(getYFields(y, originField, fieldAlias));
+ // 处理纵轴过滤
+ yWheres.add(getYWheres(y, originField, fieldAlias));
+ // 处理纵轴排序
+ if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
+ yOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(y.getSort())
+ .build());
+ }
+ }
+ }
+ // 处理视图中字段过滤
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ // 处理仪表板字段过滤
+ String extWheres = transExtFilterList(tableObj, extFilterRequestList);
+ // 构建sql所有参数
+ List fields = new ArrayList<>();
+ fields.addAll(xFields);
+ fields.addAll(yFields);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (extWheres != null) wheres.add(extWheres);
+ List groups = new ArrayList<>();
+ groups.addAll(xFields);
+ // 外层再次套sql
+ List orders = new ArrayList<>();
+ orders.addAll(xOrders);
+ orders.addAll(yOrders);
+ List aggWheres = new ArrayList<>();
+ aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("querySql");
+ if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
+ if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String sql = st_sql.render();
+
+ ST st = stg.getInstanceOf("querySql");
+ SQLObj tableSQL = SQLObj.builder()
+ .tableName(String.format(MysqlConstants.BRACKETS, sql))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
+ .build();
+ if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
+ if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
+ if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
+ return sqlLimit(st.render(), view);
+ }
+
+ @Override
+ public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, ChartViewWithBLOBs view) {
+ return getSQLStack("(" + table + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extStack, null, view);
+ }
+
+ @Override
+ public String getSQLScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, Datasource ds, ChartViewWithBLOBs view) {
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List xFields = new ArrayList<>();
+ List xOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(xAxis)) {
+ for (int i = 0; i < xAxis.size(); i++) {
+ ChartViewFieldDTO x = xAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(x.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
+ // 处理横轴字段
+ xFields.add(getXFields(x, originField, fieldAlias));
+
+ // 处理横轴排序
+ if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) {
+ xOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(x.getSort())
+ .build());
+ }
+ }
+ }
+ List yFields = new ArrayList<>();
+ List yWheres = new ArrayList<>();
+ List yOrders = new ArrayList<>();
+ List yList = new ArrayList<>();
+ yList.addAll(yAxis);
+ yList.addAll(extBubble);
+ if (CollectionUtils.isNotEmpty(yList)) {
+ for (int i = 0; i < yList.size(); i++) {
+ ChartViewFieldDTO y = yList.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(y.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
+ // 处理纵轴字段
+ yFields.add(getYFields(y, originField, fieldAlias));
+ // 处理纵轴过滤
+ yWheres.add(getYWheres(y, originField, fieldAlias));
+ // 处理纵轴排序
+ if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
+ yOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(y.getSort())
+ .build());
+ }
+ }
+ }
+ // 处理视图中字段过滤
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ // 处理仪表板字段过滤
+ String extWheres = transExtFilterList(tableObj, extFilterRequestList);
+ // 构建sql所有参数
+ List fields = new ArrayList<>();
+ fields.addAll(xFields);
+ fields.addAll(yFields);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (extWheres != null) wheres.add(extWheres);
+ List groups = new ArrayList<>();
+ groups.addAll(xFields);
+ // 外层再次套sql
+ List orders = new ArrayList<>();
+ orders.addAll(xOrders);
+ orders.addAll(yOrders);
+ List aggWheres = new ArrayList<>();
+ aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("querySql");
+ if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields);
+ if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String sql = st_sql.render();
+
+ ST st = stg.getInstanceOf("querySql");
+ SQLObj tableSQL = SQLObj.builder()
+ .tableName(String.format(MysqlConstants.BRACKETS, sql))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
+ .build();
+ if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
+ if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
+ if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
+ return sqlLimit(st.render(), view);
+ }
+
+ @Override
+ public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, ChartViewWithBLOBs view) {
+ return getSQLScatter("(" + table + ")", xAxis, yAxis, fieldCustomFilter, extFilterRequestList, extBubble, null, view);
+ }
+
+ @Override
+ public String searchTable(String table) {
+ return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'";
+ }
+
+ @Override
+ public String getSQLSummary(String table, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) {
+ // 字段汇总 排序等
+ SQLObj tableObj = SQLObj.builder()
+ .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MysqlConstants.KEYWORD_TABLE, table))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
+ .build();
+ List yFields = new ArrayList<>();
+ List yWheres = new ArrayList<>();
+ List yOrders = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(yAxis)) {
+ for (int i = 0; i < yAxis.size(); i++) {
+ ChartViewFieldDTO y = yAxis.get(i);
+ String originField;
+ if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originField = calcFieldRegex(y.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ } else {
+ originField = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getDataeaseName());
+ }
+ String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
+ // 处理纵轴字段
+ yFields.add(getYFields(y, originField, fieldAlias));
+ // 处理纵轴过滤
+ yWheres.add(getYWheres(y, originField, fieldAlias));
+ // 处理纵轴排序
+ if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) {
+ yOrders.add(SQLObj.builder()
+ .orderField(originField)
+ .orderAlias(fieldAlias)
+ .orderDirection(y.getSort())
+ .build());
+ }
+ }
+ }
+ // 处理视图中字段过滤
+ String customWheres = transCustomFilterList(tableObj, fieldCustomFilter);
+ // 处理仪表板字段过滤
+ String extWheres = transExtFilterList(tableObj, extFilterRequestList);
+ // 构建sql所有参数
+ List fields = new ArrayList<>();
+ fields.addAll(yFields);
+ List wheres = new ArrayList<>();
+ if (customWheres != null) wheres.add(customWheres);
+ if (extWheres != null) wheres.add(extWheres);
+ List groups = new ArrayList<>();
+ // 外层再次套sql
+ List orders = new ArrayList<>();
+ orders.addAll(yOrders);
+ List aggWheres = new ArrayList<>();
+ aggWheres.addAll(yWheres.stream().filter(ObjectUtils::isNotEmpty).collect(Collectors.toList()));
+
+ STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE);
+ ST st_sql = stg.getInstanceOf("querySql");
+ if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields);
+ if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres);
+ if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj);
+ String sql = st_sql.render();
+
+ ST st = stg.getInstanceOf("querySql");
+ SQLObj tableSQL = SQLObj.builder()
+ .tableName(String.format(MysqlConstants.BRACKETS, sql))
+ .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
+ .build();
+ if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
+ if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
+ if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL);
+ return sqlLimit(st.render(), view);
+ }
+
+ @Override
+ public String getSQLSummaryAsTmp(String sql, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) {
+ return getSQLSummary("(" + sql + ")", yAxis, fieldCustomFilter, extFilterRequestList, view);
+ }
+
+ @Override
+ public String wrapSql(String sql) {
+ sql = sql.trim();
+ if (sql.lastIndexOf(";") == (sql.length() - 1)) {
+ sql = sql.substring(0, sql.length() - 1);
+ }
+ String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0";
+ return tmpSql;
+ }
+
+ @Override
+ public String createRawQuerySQL(String table, List fields, Datasource ds) {
+ String[] array = fields.stream().map(f -> {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("`").append(f.getDataeaseName()).append("` AS ").append(f.getDataeaseName());
+ return stringBuilder.toString();
+ }).toArray(String[]::new);
+ return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table);
+ }
+
+ @Override
+ public String createRawQuerySQLAsTmp(String sql, List fields) {
+ return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null);
+ }
+
+ @Override
+ public String convertTableToSql(String tableName, Datasource ds) {
+ return createSQLPreview("SELECT * FROM " + String.format(MysqlConstants.KEYWORD_TABLE, tableName), null);
+ }
+
+ private String sqlFix(String sql) {
+ if (sql.lastIndexOf(";") == (sql.length() - 1)) {
+ sql = sql.substring(0, sql.length() - 1);
+ }
+ return sql;
+ }
+
+ public String transMysqlFilterTerm(String term) {
+ switch (term) {
+ case "eq":
+ return " = ";
+ case "not_eq":
+ return " <> ";
+ case "lt":
+ return " < ";
+ case "le":
+ return " <= ";
+ case "gt":
+ return " > ";
+ case "ge":
+ return " >= ";
+ case "in":
+ return " IN ";
+ case "not in":
+ return " NOT IN ";
+ case "like":
+ return " LIKE ";
+ case "not like":
+ return " NOT LIKE ";
+ case "null":
+ return " IS NULL ";
+ case "not_null":
+ return " IS NOT NULL ";
+ case "empty":
+ return " = ";
+ case "not_empty":
+ return " <> ";
+ case "between":
+ return " BETWEEN ";
+ default:
+ return "";
+ }
+ }
+
+ public String transCustomFilterList(SQLObj tableObj, List requestList) {
+ if (CollectionUtils.isEmpty(requestList)) {
+ return null;
+ }
+ List res = new ArrayList<>();
+
+ for (ChartFieldCustomFilterDTO request : requestList) {
+ List list = new ArrayList<>();
+ DatasetTableField field = request.getField();
+
+ if (ObjectUtils.isEmpty(field)) {
+ continue;
+ }
+ String whereName = "";
+ String originName;
+ if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originName = calcFieldRegex(field.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
+ originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName());
+ } else {
+ originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName());
+ }
+
+ if (field.getDeType() == 1) {
+ if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5 || field.getDeExtractType() == 1) {
+ whereName = String.format(MysqlConstants.STR_TO_DATE, originName, MysqlConstants.DEFAULT_DATE_FORMAT);
+ }
+ if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ String cast = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000";
+ whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT);
+ }
+ } else if (field.getDeType() == 2 || field.getDeType() == 3) {
+ if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
+ whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ }
+ if (field.getDeExtractType() == 1) {
+ whereName = String.format(MysqlConstants.UNIX_TIMESTAMP, originName) + "*1000";
+ }
+ if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ whereName = originName;
+ }
+ } else if (field.getDeType() == 0) {
+ whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.VARCHAR);
+ } else {
+ whereName = originName;
+ }
+
+ if (StringUtils.equalsIgnoreCase(request.getFilterType(), "enum")) {
+ if (CollectionUtils.isNotEmpty(request.getEnumCheckField())) {
+ res.add("(" + whereName + " IN ('" + String.join("','", request.getEnumCheckField()) + "'))");
+ }
+ } else {
+ List filter = request.getFilter();
+ for (ChartCustomFilterItemDTO filterItemDTO : filter) {
+ String value = filterItemDTO.getValue();
+ String whereTerm = transMysqlFilterTerm(filterItemDTO.getTerm());
+ String whereValue = "";
+
+ if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "empty")) {
+ whereValue = "''";
+ } else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
+ whereValue = "''";
+ } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
+ whereValue = "('" + StringUtils.join(value, "','") + "')";
+ } else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
+ whereValue = "'%" + value + "%'";
+ } else {
+ // Doris field type test
+ /*if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ whereValue = String.format(DorisConstants.WHERE_NUMBER_VALUE, value);
+ } else {
+ whereValue = String.format(DorisConstants.WHERE_VALUE_VALUE, value);
+ }*/
+ whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, value);
+ }
+ list.add(SQLObj.builder()
+ .whereField(whereName)
+ .whereTermAndValue(whereTerm + whereValue)
+ .build());
+ }
+
+ List strList = new ArrayList<>();
+ list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
+ if (CollectionUtils.isNotEmpty(list)) {
+ res.add("(" + String.join(" " + getLogic(request.getLogic()) + " ", strList) + ")");
+ }
+ }
+ }
+ return CollectionUtils.isNotEmpty(res) ? "(" + String.join(" AND ", res) + ")" : null;
+ }
+
+ public String transExtFilterList(SQLObj tableObj, List requestList) {
+ if (CollectionUtils.isEmpty(requestList)) {
+ return null;
+ }
+ List list = new ArrayList<>();
+ for (ChartExtFilterRequest request : requestList) {
+ List value = request.getValue();
+ DatasetTableField field = request.getDatasetTableField();
+ if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) {
+ continue;
+ }
+ String whereName = "";
+ String whereTerm = transMysqlFilterTerm(request.getOperator());
+ String whereValue = "";
+
+ String originName;
+ if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) {
+ // 解析origin name中有关联的字段生成sql表达式
+ originName = calcFieldRegex(field.getOriginName(), tableObj);
+ } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
+ originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName());
+ } else {
+ originName = String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getDataeaseName());
+ }
+
+ if (field.getDeType() == 1) {
+ if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5 || field.getDeExtractType() == 1) {
+ whereName = String.format(MysqlConstants.STR_TO_DATE, originName, MysqlConstants.DEFAULT_DATE_FORMAT);
+ }
+ if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ String cast = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000";
+ whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT);
+ }
+ } else if (field.getDeType() == 0) {
+ whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.VARCHAR);
+ } else if (field.getDeType() == 2 || field.getDeType() == 3) {
+ if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
+ whereName = String.format(MysqlConstants.CAST, originName, MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ }
+ if (field.getDeExtractType() == 1) {
+ whereName = String.format(MysqlConstants.UNIX_TIMESTAMP, originName) + "*1000";
+ }
+ if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ whereName = originName;
+ }
+ } else {
+ whereName = originName;
+ }
+
+
+ if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) {
+ whereValue = "('" + StringUtils.join(value, "','") + "')";
+ } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) {
+ whereValue = "'%" + value.get(0) + "%'";
+ } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) {
+ if (request.getDatasetTableField().getDeType() == 1) {
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
+ String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
+ whereValue = String.format(MysqlConstants.WHERE_BETWEEN, startTime, endTime);
+ } else {
+ whereValue = String.format(MysqlConstants.WHERE_BETWEEN, value.get(0), value.get(1));
+ }
+ } else {
+ // doris field type test
+ /*if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) {
+ whereValue = String.format(DorisConstants.WHERE_NUMBER_VALUE, value.get(0));
+ } else {
+ whereValue = String.format(DorisConstants.WHERE_VALUE_VALUE, value.get(0));
+ }*/
+ whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, value.get(0));
+ }
+ list.add(SQLObj.builder()
+ .whereField(whereName)
+ .whereTermAndValue(whereTerm + whereValue)
+ .build());
+ }
+ List strList = new ArrayList<>();
+ list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
+ return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" AND ", strList) + ")" : null;
+ }
+
+ private String transDateFormat(String dateStyle, String datePattern) {
+ String split = "-";
+ if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) {
+ split = "-";
+ } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) {
+ split = "/";
+ } else {
+ split = "-";
+ }
+
+ if (StringUtils.isEmpty(dateStyle)) {
+ return "%Y-%m-%d %H:%i:%S";
+ }
+
+ switch (dateStyle) {
+ case "y":
+ return "%Y";
+ case "y_M":
+ return "%Y" + split + "%m";
+ case "y_M_d":
+ return "%Y" + split + "%m" + split + "%d";
+ case "H_m_s":
+ return "%H:%i:%S";
+ case "y_M_d_H_m":
+ return "%Y" + split + "%m" + split + "%d" + " %H:%i";
+ case "y_M_d_H_m_s":
+ return "%Y" + split + "%m" + split + "%d" + " %H:%i:%S";
+ default:
+ return "%Y-%m-%d %H:%i:%S";
+ }
+ }
+
+ private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) {
+ String fieldName = "";
+ if (x.getDeExtractType() == 1) {
+ if (x.getDeType() == 2 || x.getDeType() == 3) {
+ fieldName = String.format(MysqlConstants.UNIX_TIMESTAMP, originField) + "*1000";
+ } else if (x.getDeType() == 1) {
+ String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
+ fieldName = String.format(MysqlConstants.DATE_FORMAT, originField, format);
+ } else {
+ fieldName = originField;
+ }
+ } else {
+ if (x.getDeType() == 1) {
+ String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
+ if (x.getDeExtractType() == 0) {
+ fieldName = String.format(MysqlConstants.DATE_FORMAT, originField, format);
+ } else {
+ String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000";
+ String from_unixtime = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT);
+ fieldName = String.format(MysqlConstants.DATE_FORMAT, from_unixtime, format);
+ }
+ } else if (x.getDeType() == 0) {
+ fieldName = String.format(MysqlConstants.CAST, originField, MysqlConstants.VARCHAR);
+ } else {
+ fieldName = originField;
+ }
+ }
+ return SQLObj.builder()
+ .fieldName(fieldName)
+ .fieldAlias(fieldAlias)
+ .build();
+ }
+
+ private List getXWheres(ChartViewFieldDTO x, String originField, String fieldAlias) {
+ List list = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(x.getFilter()) && x.getFilter().size() > 0) {
+ x.getFilter().forEach(f -> {
+ String whereName = "";
+ String whereTerm = transMysqlFilterTerm(f.getTerm());
+ String whereValue = "";
+ if (x.getDeType() == 1 && x.getDeExtractType() != 1) {
+ String cast = String.format(MysqlConstants.CAST, originField, MysqlConstants.DEFAULT_INT_FORMAT) + "/1000";
+ whereName = String.format(MysqlConstants.FROM_UNIXTIME, cast, MysqlConstants.DEFAULT_DATE_FORMAT);
+ } else {
+ whereName = originField;
+ }
+ if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) {
+ whereValue = "''";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) {
+ whereValue = "''";
+ } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
+ whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
+ } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
+ whereValue = "'%" + f.getValue() + "%'";
+ } else {
+ whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, f.getValue());
+ }
+ list.add(SQLObj.builder()
+ .whereField(whereName)
+ .whereAlias(fieldAlias)
+ .whereTermAndValue(whereTerm + whereValue)
+ .build());
+ });
+ }
+ return list;
+ }
+
+ private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
+ String fieldName = "";
+ if (StringUtils.equalsIgnoreCase(y.getDataeaseName(), "*")) {
+ fieldName = MysqlConstants.AGG_COUNT;
+ } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
+ fieldName = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), originField);
+ } else {
+ if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
+ String cast = String.format(MysqlConstants.CAST, originField, y.getDeType() == 2 ? MysqlConstants.DEFAULT_INT_FORMAT : MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ String agg = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), cast);
+ String cast1 = String.format(MysqlConstants.CAST, agg, MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ fieldName = String.format(MysqlConstants.ROUND, cast1, "2");
+ } else {
+ String cast = String.format(MysqlConstants.CAST, originField, y.getDeType() == 2 ? MysqlConstants.DEFAULT_INT_FORMAT : MysqlConstants.DEFAULT_FLOAT_FORMAT);
+ fieldName = String.format(MysqlConstants.AGG_FIELD, y.getSummary(), cast);
+ }
+ }
+ return SQLObj.builder()
+ .fieldName(fieldName)
+ .fieldAlias(fieldAlias)
+ .build();
+ }
+
+ private String getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) {
+ List list = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) {
+ y.getFilter().forEach(f -> {
+ String whereTerm = transMysqlFilterTerm(f.getTerm());
+ String whereValue = "";
+ // 原始类型不是时间，在de中被转成时间的字段做处理
+ if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
+ whereValue = "";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) {
+ whereValue = "''";
+ } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) {
+ whereValue = "''";
+ } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
+ whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')";
+ } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
+ whereValue = "'%" + f.getValue() + "%'";
+ } else {
+ whereValue = String.format(MysqlConstants.WHERE_VALUE_VALUE, f.getValue());
+ }
+ list.add(SQLObj.builder()
+ .whereField(fieldAlias)
+ .whereAlias(fieldAlias)
+ .whereTermAndValue(whereTerm + whereValue)
+ .build());
+ });
+ }
+ List strList = new ArrayList<>();
+ list.forEach(ele -> strList.add(ele.getWhereField() + " " + ele.getWhereTermAndValue()));
+ return CollectionUtils.isNotEmpty(list) ? "(" + String.join(" " + getLogic(y.getLogic()) + " ", strList) + ")" : null;
+ }
+
+ private String calcFieldRegex(String originField, SQLObj tableObj) {
+ originField = originField.replaceAll("[\\t\\n\\r]]", "");
+ // 正则提取[xxx]
+ String regex = "\\[(.*?)]";
+ Pattern pattern = Pattern.compile(regex);
+ Matcher matcher = pattern.matcher(originField);
+ Set ids = new HashSet<>();
+ while (matcher.find()) {
+ String id = matcher.group(1);
+ ids.add(id);
+ }
+ if (CollectionUtils.isEmpty(ids)) {
+ return originField;
+ }
+ DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
+ datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids));
+ List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
+ for (DatasetTableField ele : calcFields) {
+ originField = originField.replaceAll("\\[" + ele.getId() + "]",
+ String.format(MysqlConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getDataeaseName()));
+ }
+ return originField;
+ }
+
+ private String sqlLimit(String sql, ChartViewWithBLOBs view) {
+ if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
+ return sql + " LIMIT 0," + view.getResultCount();
+ } else {
+ return sql;
+ }
+ }
+}
diff --git a/backend/src/main/java/io/dataease/provider/query/DDLProvider.java b/backend/src/main/java/io/dataease/provider/query/DDLProvider.java
deleted file mode 100644
index ba1e218c1e..0000000000
--- a/backend/src/main/java/io/dataease/provider/query/DDLProvider.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package io.dataease.provider.query;
-
-/**
- * @Author gin
- * @Date 2021/5/17 4:19 下午
- */
-public abstract class DDLProvider {
- public abstract String createView(String name, String viewSQL);
-
- public abstract String dropTable(String name);
-
- public abstract String dropView(String name);
-}
diff --git a/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java
index 80bfe8f3c4..8f9038662e 100644
--- a/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.api;
-import io.dataease.provider.query.QueryProviderImpl;
+import io.dataease.provider.QueryProviderImpl;
import org.springframework.stereotype.Service;
@Service("apiQuery")
diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java b/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java
index 173849d756..7f9bd5d9ec 100644
--- a/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/ck/CKConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.ck;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.ck;
diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java
index c1165b2535..8f5d32a266 100644
--- a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java
@@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java
index 307d166243..e914d91b9c 100644
--- a/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java
+++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2Constants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.db2;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.db2;
diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java
index a1593afca8..d9fd103ce0 100644
--- a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java
@@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.Db2Configuration;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("db2Query")
public class Db2QueryProvider extends QueryProvider {
diff --git a/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java b/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java
deleted file mode 100644
index cfa3f420a7..0000000000
--- a/backend/src/main/java/io/dataease/provider/query/doris/DorisDDLProvider.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package io.dataease.provider.query.doris;
-
-import io.dataease.provider.query.DDLProvider;
-import org.springframework.stereotype.Service;
-
-/**
- * @Author gin
- * @Date 2021/5/17 4:27 下午
- */
-@Service("dorisDDL")
-public class DorisDDLProvider extends DDLProvider {
- @Override
- public String createView(String name, String viewSQL) {
- return "CREATE VIEW IF NOT EXISTS " + name + " AS (" + viewSQL + ")";
- }
-
- @Override
- public String dropTable(String name) {
- return "DROP TABLE IF EXISTS " + name;
- }
-
- @Override
- public String dropView(String name) {
- return "DROP VIEW IF EXISTS " + name;
- }
-}
diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java
index 77f24131ab..95fb323686 100644
--- a/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/es/EsQueryProvider.java
@@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("esQuery")
public class EsQueryProvider extends QueryProvider {
diff --git a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java b/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java
index 62301f3bf6..a9186a68e7 100644
--- a/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/es/EsSqlLConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.es;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.es;
diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
index 4e99ae88e8..ec1a82007f 100644
--- a/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.hive;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;
diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java
index 457651b642..9e054efe3f 100644
--- a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java
@@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java
index 0d4f8127de..14a7107f96 100644
--- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.mongodb;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mongo;
diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
index e2922121c0..97f7bcf7f1 100644
--- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java
@@ -11,8 +11,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java b/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java
index 160ff281c9..33b1e3d8ea 100644
--- a/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/mysql/MySQLConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.mysql;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.mysql;
diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
index 706c0fe034..95f653fb12 100644
--- a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
@@ -10,8 +10,8 @@ import io.dataease.dto.chart.ChartCustomFilterItemDTO;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -28,7 +28,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java
index c8e500e174..013e51defc 100644
--- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.oracle;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.oracle;
diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java
index 3fc25eb97d..48d04457c5 100644
--- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java
@@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.datasource.OracleConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
* @Author gin
diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java b/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java
index 25c50a431f..d0e9f7cdeb 100644
--- a/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/pg/PgConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.pg;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;
diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java
index 55eebb6755..b231d74536 100644
--- a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java
@@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@@ -32,7 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("pgQuery")
diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java
index f52c1d36d2..f1ba7179aa 100644
--- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.redshift;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.pg;
diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java
index b3f54f14ff..3067caaf8e 100644
--- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java
@@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import io.dataease.provider.query.pg.PgConstants;
import io.dataease.provider.query.sqlserver.SqlServerSQLConstants;
import org.apache.commons.collections4.CollectionUtils;
@@ -33,7 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
/**
diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java
index 1817f77afa..ac5862abfa 100644
--- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java
+++ b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlServerSQLConstants.java
@@ -1,6 +1,6 @@
package io.dataease.provider.query.sqlserver;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.SQLConstants;
import static io.dataease.commons.constants.DatasourceTypes.sqlServer;
diff --git a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java
index 99f855c936..2718468fdf 100644
--- a/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java
+++ b/backend/src/main/java/io/dataease/provider/query/sqlserver/SqlserverQueryProvider.java
@@ -13,8 +13,8 @@ import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.datasource.JdbcConfiguration;
import io.dataease.dto.sqlObj.SQLObj;
-import io.dataease.provider.query.QueryProvider;
-import io.dataease.provider.query.SQLConstants;
+import io.dataease.provider.QueryProvider;
+import io.dataease.provider.SQLConstants;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -31,7 +31,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static io.dataease.provider.query.SQLConstants.TABLE_ALIAS_PREFIX;
+import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
@Service("sqlserverQuery")
public class SqlserverQueryProvider extends QueryProvider {
diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java
index 76924fd0c2..69778f5f6e 100644
--- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java
+++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java
@@ -2,7 +2,6 @@ package io.dataease.service.chart;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
-import io.dataease.auth.api.dto.CurrentUserDto;
import io.dataease.auth.entity.SysUserEntity;
import io.dataease.auth.service.AuthUserService;
import io.dataease.base.domain.*;
@@ -15,7 +14,6 @@ import io.dataease.commons.constants.CommonConstants;
import io.dataease.commons.constants.JdbcConstants;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
-import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.chart.*;
import io.dataease.controller.request.datasource.DatasourceRequest;
@@ -29,12 +27,13 @@ import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
-import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.DataSetTableUnionService;
import io.dataease.service.dataset.PermissionService;
import io.dataease.service.datasource.DatasourceService;
+import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -74,6 +73,8 @@ public class ChartViewService {
@Resource
private AuthUserService authUserService;
@Resource
+ private EngineService engineService;
+ @Resource
private ChartViewCacheMapper chartViewCacheMapper;
//默认使用非公平
@@ -530,7 +531,7 @@ public class ChartViewService {
data = datasourceProvider.getData(datasourceRequest);
} else if (table.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
datasourceRequest.setDatasource(ds);
String tableName = "ds_" + table.getId().replaceAll("-", "_");
diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java
index 4744c14637..185e68a6a5 100644
--- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableFieldsService.java
@@ -4,7 +4,7 @@ import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample;
import io.dataease.base.mapper.DatasetTableFieldMapper;
import io.dataease.commons.exception.DEException;
-import io.dataease.commons.utils.DorisTableUtils;
+import io.dataease.commons.utils.TableUtils;
import io.dataease.i18n.Translator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
@@ -36,7 +36,7 @@ public class DataSetTableFieldsService {
datasetTableField.setId(UUID.randomUUID().toString());
// 若dataeasename为空，则用MD5(id)作为dataeasename
if (StringUtils.isEmpty(datasetTableField.getDataeaseName())) {
- datasetTableField.setDataeaseName(DorisTableUtils.columnName(datasetTableField.getId()));
+ datasetTableField.setDataeaseName(TableUtils.columnName(datasetTableField.getId()));
}
if (ObjectUtils.isEmpty(datasetTableField.getLastSyncTime())) {
datasetTableField.setLastSyncTime(System.currentTimeMillis());
diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
index e95faa427a..64ade01908 100644
--- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
@@ -28,8 +28,9 @@ import io.dataease.plugins.loader.ClassloaderResponsity;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
-import io.dataease.provider.query.DDLProvider;
-import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.DDLProvider;
+import io.dataease.provider.QueryProvider;
+import io.dataease.service.engine.EngineService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -97,6 +98,8 @@ public class DataSetTableService {
private DatasetTableFieldMapper datasetTableFieldMapper;
@Resource
private PermissionService permissionService;
+ @Resource
+ private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@@ -296,8 +299,8 @@ public class DataSetTableService {
}
private void deleteDorisTable(String datasetId, DatasetTable table) throws Exception {
- String dorisTableName = DorisTableUtils.dorisName(datasetId);
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ String dorisTableName = TableUtils.tableName(datasetId);
+ Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@@ -306,12 +309,12 @@ public class DataSetTableService {
|| StringUtils.equalsIgnoreCase("union", table.getType())) {
datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName));
jdbcProvider.exec(datasourceRequest);
- datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName)));
+ datasourceRequest.setQuery(ddlProvider.dropView(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
} else {
datasourceRequest.setQuery(ddlProvider.dropTable(dorisTableName));
jdbcProvider.exec(datasourceRequest);
- datasourceRequest.setQuery(ddlProvider.dropTable(DorisTableUtils.dorisTmpName(dorisTableName)));
+ datasourceRequest.setQuery(ddlProvider.dropTable(TableUtils.tmpName(dorisTableName)));
jdbcProvider.exec(datasourceRequest);
}
}
@@ -551,11 +554,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
- String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+ String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@@ -620,11 +623,11 @@ public class DataSetTableService {
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
- String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+ String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@@ -649,11 +652,11 @@ public class DataSetTableService {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
- String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+ String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@@ -719,11 +722,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
- String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+ String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@@ -789,11 +792,11 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
}
} else {
- Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource ds = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
- String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+ String table = TableUtils.tableName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
@@ -889,7 +892,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds);
} else {
- ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDoris(dataTableInfoDTO);
}
@@ -923,10 +926,10 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
- DorisTableUtils.dorisFieldName(
+ TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableField.getFieldName(),
- DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
+ TableUtils.fieldNameShort(datasetTableField.getTableId() + "_"
+ datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@@ -955,7 +958,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDatasource(dataTableInfoDTO, list, ds);
} else {
- ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ ds = engineService.getDeEngine();
datasourceRequest.setDatasource(ds);
sql = getCustomSQLDoris(dataTableInfoDTO, list);
}
@@ -987,9 +990,9 @@ public class DataSetTableService {
for (DatasetTableField datasetTableField : checkedFieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
- DorisTableUtils.dorisFieldName(
+ TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
- || StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName(
+ || StringUtils.equalsIgnoreCase(tableField.getFieldName(), TableUtils.fieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableField.setRemarks(datasetTableField.getName());
break;
@@ -1009,7 +1012,7 @@ public class DataSetTableService {
private String getCustomSQLDoris(DataTableInfoDTO dataTableInfoDTO, List list) {
Map customInfo = new TreeMap<>();
dataTableInfoDTO.getList().forEach(ele -> {
- String table = DorisTableUtils.dorisName(ele.getTableId());
+ String table = TableUtils.tableName(ele.getTableId());
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(ele.getTableId());
if (ObjectUtils.isEmpty(datasetTable)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
@@ -1020,7 +1023,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
- + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))
+ + TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName()))
.toArray(String[]::new);
customInfo.put(table, array);
});
@@ -1046,12 +1049,12 @@ public class DataSetTableService {
}
join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation()))
- .append(DorisTableUtils.dorisName(dto.getTargetTableId()))
+ .append(TableUtils.tableName(dto.getTargetTableId()))
.append(" ON ")
- .append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".")
+ .append(TableUtils.tableName(dto.getSourceTableId())).append(".")
.append(sourceField.getDataeaseName())
.append(" = ")
- .append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".")
+ .append(TableUtils.tableName(dto.getTargetTableId())).append(".")
.append(targetField.getDataeaseName());
}
}
@@ -1059,16 +1062,16 @@ public class DataSetTableService {
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
- return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId()))
+ return MessageFormat.format("SELECT {0} FROM {1}", f, TableUtils.tableName(first.getTableId()))
+ join.toString();
} else {
if (StringUtils
- .isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) {
+ .isEmpty(StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","))) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
return MessageFormat.format("SELECT {0} FROM {1}",
- StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","),
- DorisTableUtils.dorisName(first.getTableId()));
+ StringUtils.join(customInfo.get(TableUtils.tableName(first.getTableId())), ","),
+ TableUtils.tableName(first.getTableId()));
}
}
@@ -1089,7 +1092,7 @@ public class DataSetTableService {
}
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
- + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName()))
+ + TableUtils.fieldName(ele.getTableId() + "_" + f.getOriginName()))
.toArray(String[]::new);
customInfo.put(table, array);
}
@@ -1175,7 +1178,7 @@ public class DataSetTableService {
for (UnionDTO unionDTO : union) {
// doris 使用数据集id做表名，拼sql将用到该名称
String tableId = unionDTO.getCurrentDs().getId();
- String table = DorisTableUtils.dorisName(tableId);
+ String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@@ -1193,7 +1196,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
- + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ + TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@@ -1225,7 +1228,7 @@ public class DataSetTableService {
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
- join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId()))
+ join.append(" ").append(joinType).append(" ").append(TableUtils.tableName(currentTable.getId()))
.append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
@@ -1235,10 +1238,10 @@ public class DataSetTableService {
DatasetTableField currentField = dataSetTableFieldsService
.get(unionItemDTO.getCurrentField().getId());
- join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".")
+ join.append(TableUtils.tableName(parentTable.getId())).append(".")
.append(parentField.getDataeaseName())
.append(" = ")
- .append(DorisTableUtils.dorisName(currentTable.getId())).append(".")
+ .append(TableUtils.tableName(currentTable.getId())).append(".")
.append(currentField.getDataeaseName());
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
@@ -1249,15 +1252,15 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
- DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString();
+ TableUtils.tableName(union.get(0).getCurrentDs().getId())) + join.toString();
} else {
- String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())),
+ String f = StringUtils.join(checkedInfo.get(TableUtils.tableName(union.get(0).getCurrentDs().getId())),
",");
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_union_ds_no_checked"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f,
- DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId()));
+ TableUtils.tableName(union.get(0).getCurrentDs().getId()));
}
Map map = new HashMap<>();
map.put("sql", sql);
@@ -1272,7 +1275,7 @@ public class DataSetTableService {
for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i);
String tableId = unionDTO.getCurrentDs().getId();
- String table = DorisTableUtils.dorisName(tableId);
+ String table = TableUtils.tableName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(
@@ -1282,7 +1285,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
- + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
+ + TableUtils.fieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@@ -1329,7 +1332,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
- + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ + TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@@ -1430,7 +1433,7 @@ public class DataSetTableService {
String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
- + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
+ + TableUtils.fieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
@@ -1451,7 +1454,7 @@ public class DataSetTableService {
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
- datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
+ datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
datasetTableField.setType(filed.getFieldType());
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
@@ -1501,7 +1504,7 @@ public class DataSetTableService {
List listByIds = dataSetTableFieldsService
.getListByIdsEach(ele.getCheckedFields());
listByIds.forEach(f -> f.setDataeaseName(
- DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())));
+ TableUtils.fieldName(ele.getTableId() + "_" + f.getDataeaseName())));
fieldList.addAll(listByIds);
});
for (int i = 0; i < fieldList.size(); i++) {
@@ -1514,7 +1517,7 @@ public class DataSetTableService {
dataSetTableFieldsService.batchEdit(fieldList);
// custom 创建doris视图
if (datasetTable.getMode() == 1) {
- createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
+ createDorisView(TableUtils.tableName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId())));
}
return;
@@ -1541,7 +1544,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
- DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
+ TableUtils.fieldName(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@@ -1551,7 +1554,7 @@ public class DataSetTableService {
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
if (datasetTable.getMode() == 1) {
- ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ ds = engineService.getDeEngine();
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
@@ -1564,14 +1567,14 @@ public class DataSetTableService {
List join = (List) sqlMap.get("join");
// custom 创建doris视图
- createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql);
+ createDorisView(TableUtils.tableName(datasetTable.getId()), sql);
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
- DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
+ TableUtils.fieldName(field.getTableId() + "_" + field.getDataeaseName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@@ -1595,7 +1598,7 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) {
for (TableField tableField : fields) {
if (StringUtils.equalsIgnoreCase(
- DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
+ TableUtils.fieldNameShort(field.getTableId() + "_" + field.getOriginName()),
tableField.getFieldName())) {
tableField.setRemarks(field.getName());
break;
@@ -1637,7 +1640,7 @@ public class DataSetTableService {
if (datasetTable.getMode() == 1 && StringUtils.equalsIgnoreCase("union", datasetTable.getType())) {
datasetTableField.setDataeaseName(filed.getFieldName());
} else {
- datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
+ datasetTableField.setDataeaseName(TableUtils.columnName(filed.getFieldName()));
}
datasetTableField.setType(filed.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
@@ -1665,7 +1668,7 @@ public class DataSetTableService {
}
private void createDorisView(String dorisTableName, String customSql) throws Exception {
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
@@ -2165,12 +2168,12 @@ public class DataSetTableService {
}
public Boolean checkDorisTableIsExists(String id) throws Exception {
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource dorisDatasource = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
QueryProvider qp = ProviderFactory.getQueryProvider(dorisDatasource.getType());
- datasourceRequest.setQuery(qp.searchTable(DorisTableUtils.dorisName(id)));
+ datasourceRequest.setQuery(qp.searchTable(TableUtils.tableName(id)));
List data = jdbcProvider.getData(datasourceRequest);
return CollectionUtils.isNotEmpty(data);
}
diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
index ebf6e5bb9e..9795f86261 100644
--- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
@@ -12,6 +12,7 @@ import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
import io.dataease.commons.constants.DatasourceTypes;
import io.dataease.controller.request.datasource.ApiDefinition;
+import io.dataease.provider.DDLProvider;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory;
@@ -22,7 +23,8 @@ import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
-import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.QueryProvider;
+import io.dataease.service.engine.EngineService;
import io.dataease.service.message.DeMsgutil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
@@ -94,6 +96,8 @@ public class ExtractDataService {
private DatasourceService datasourceService;
@Resource
private ExtChartViewMapper extChartViewMapper;
+ @Resource
+ private EngineService engineService;
private static final String lastUpdateTime = "${__last_update_time__}";
private static final String currentUpdateTime = "${__current_update_time__}";
@@ -111,13 +115,7 @@ public class ExtractDataService {
private String user;
@Value("${carte.passwd:cluster}")
private String passwd;
- private static final String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
- "Column_Fields" +
- "UNIQUE KEY(dataease_uuid)\n" +
- "DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
- "PROPERTIES(\"replication_num\" = \"1\");";
- private static final String dropTableSql = "DROP TABLE IF EXISTS TABLE_NAME;";
private static final String shellScript = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
"if [ $? -eq 0 ] ; then\n" +
" failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
@@ -162,7 +160,7 @@ public class ExtractDataService {
}
UpdateType updateType = UpdateType.valueOf(type);
DatasetTableTaskLog datasetTableTaskLog;
- if(datasetTableFields == null){
+ if (datasetTableFields == null) {
datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
}
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
@@ -175,32 +173,36 @@ public class ExtractDataService {
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
- String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
+
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
- createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
- createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
- generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
- generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
+ createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
+ createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
Long execTime = System.currentTimeMillis();
- extractData(datasetTable, "all_scope");
- replaceTable(DorisTableUtils.dorisName(datasetTableId));
+ if (!engineService.isSimpleMode()) {
+ generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
+ generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
+ extractData(datasetTable, "all_scope");
+ } else {
+ extractExcelDataForSimpleMode(datasetTable, "all_scope");
+ }
+ replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
- if(ops.equalsIgnoreCase("替换")){
+ if (ops.equalsIgnoreCase("替换")) {
List oldFileds = getDatasetTableFields(datasetTable.getId());
List toAdd = new ArrayList<>();
List toDelete = new ArrayList<>();
for (DatasetTableField oldFiled : oldFileds) {
boolean delete = true;
for (DatasetTableField datasetTableField : datasetTableFields) {
- if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){
+ if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
delete = false;
}
}
- if(delete){
+ if (delete) {
toDelete.add(oldFiled);
}
}
@@ -208,11 +210,11 @@ public class ExtractDataService {
for (DatasetTableField datasetTableField : datasetTableFields) {
boolean add = true;
for (DatasetTableField oldFiled : oldFileds) {
- if(oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())){
+ if (oldFiled.getDataeaseName().equalsIgnoreCase(datasetTableField.getDataeaseName()) && oldFiled.getDeExtractType().equals(datasetTableField.getDeExtractType())) {
add = false;
}
}
- if(add){
+ if (add) {
toAdd.add(datasetTableField);
}
}
@@ -222,7 +224,7 @@ public class ExtractDataService {
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
- dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
+ dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
deleteExcelFile(datasetTable, datasetTableIds);
@@ -232,10 +234,14 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
- generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
- generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
Long execTime = System.currentTimeMillis();
- extractData(datasetTable, "incremental_add");
+ if (!engineService.isSimpleMode()) {
+ generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
+ generateJobFile("incremental_add", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
+ extractData(datasetTable, "incremental_add");
+ } else {
+ extractExcelDataForSimpleMode(datasetTable, "incremental_add");
+ }
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
@@ -257,21 +263,6 @@ public class ExtractDataService {
}
}
- private List getDatasetTableFields(String datasetTableId){
- List datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
- datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
- datasetTableFields.sort((o1, o2) -> {
- if (o1.getColumnIndex() == null) {
- return -1;
- }
- if (o2.getColumnIndex() == null) {
- return 1;
- }
- return o1.getColumnIndex().compareTo(o2.getColumnIndex());
- });
- return datasetTableFields;
- }
-
public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) {
DatasetTable datasetTable = getDatasetTable(datasetTableId);
if (datasetTable == null) {
@@ -306,19 +297,17 @@ public class ExtractDataService {
datasource.setType(datasetTable.getType());
}
List datasetTableFields = getDatasetTableFields(datasetTable.getId());
- String dorisTableColumnSql = createDorisTableColumnSql(datasetTableFields);
-
boolean msg = false;
JobStatus lastExecStatus = JobStatus.Completed;
Long execTime = null;
switch (updateType) {
case all_scope: // 全量更新
try {
- createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
- createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
+ createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
+ createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
execTime = System.currentTimeMillis();
extractData(datasetTable, datasource, datasetTableFields, "all_scope", null);
- replaceTable(DorisTableUtils.dorisName(datasetTableId));
+ replaceTable(TableUtils.tableName(datasetTableId));
saveSuccessLog(datasetTableTaskLog);
msg = true;
lastExecStatus = JobStatus.Completed;
@@ -328,21 +317,39 @@ public class ExtractDataService {
lastExecStatus = JobStatus.Error;
execTime = null;
} finally {
- try { deleteFile("all_scope", datasetTableId); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
- try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
- try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){
+ try {
+ deleteFile("all_scope", datasetTableId);
+ } catch (Exception ignore) {
+ System.out.println(ignore.getMessage());
+ }
+ try {
+ sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg);
+ } catch (Exception ignore) {
+ System.out.println(ignore.getMessage());
+ }
+ try {
+ dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus);
+ } catch (Exception ignore) {
+ System.out.println(ignore.getMessage());
+ }
+ try {
+ updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
+ } catch (Exception ignore) {
+ System.out.println(ignore.getMessage());
+ }
+ try {
+ dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
+ } catch (Exception ignore) {
System.out.println(ignore.getMessage());
}
- try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
- try { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); }catch (Exception ignore){ System.out.println(ignore.getMessage());}
}
break;
case add_scope: // 增量更新
try {
- if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
+ if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) {
extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null);
- }else{
+ } else {
DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
if (datasetTable.getLastUpdateTime() == null || datasetTable.getLastUpdateTime() == 0) {
throw new Exception("未进行全量同步");
@@ -370,10 +377,23 @@ public class ExtractDataService {
lastExecStatus = JobStatus.Error;
execTime = null;
} finally {
- try { deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); }catch (Exception ignore){}
- try { sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg); }catch (Exception ignore){}
- try { dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus); }catch (Exception ignore){}
- try { updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime); }catch (Exception ignore){}
+ try {
+ deleteFile("incremental_add", datasetTableId);
+ deleteFile("incremental_delete", datasetTableId);
+ } catch (Exception ignore) {
+ }
+ try {
+ sendWebMsg(datasetTable, datasetTableTask, datasetTableTaskLog, msg);
+ } catch (Exception ignore) {
+ }
+ try {
+ dataSetTableTaskService.updateTaskStatus(datasetTableTask, lastExecStatus);
+ } catch (Exception ignore) {
+ }
+ try {
+ updateTableStatus(datasetTableId, datasetTable, lastExecStatus, execTime);
+ } catch (Exception ignore) {
+ }
}
break;
}
@@ -385,43 +405,62 @@ public class ExtractDataService {
}
- private void extractData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception{
- if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
- extractDataByDE(datasetTable, datasource, datasetTableFields, extractType);
+ private List getDatasetTableFields(String datasetTableId) {
+ List datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
+ datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
+ datasetTableFields.sort((o1, o2) -> {
+ if (o1.getColumnIndex() == null) {
+ return -1;
+ }
+ if (o2.getColumnIndex() == null) {
+ return 1;
+ }
+ return o1.getColumnIndex().compareTo(o2.getColumnIndex());
+ });
+ return datasetTableFields;
+ }
+
+ private void extractData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception {
+ if (datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())) {
+ extractApiData(datasetTable, datasource, datasetTableFields, extractType);
return;
}
extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL);
}
- private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType)throws Exception{
+ private void extractApiData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType) throws Exception {
List lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class);
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
- if(CollectionUtils.isEmpty(lists)){
+ if (CollectionUtils.isEmpty(lists)) {
throw new Exception("未找到API数据表");
}
- if(lists.size() > 1 ){
+ if (lists.size() > 1) {
throw new Exception("存在重名的API数据表");
}
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable());
- Map result = datasourceProvider.fetchResultAndField(datasourceRequest);
+ Map result = datasourceProvider.fetchResultAndField(datasourceRequest);
List dataList = result.get("dataList");
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
- DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
+ if (engineService.isSimpleMode()) {
+ extractDataForSimpleMode(extractType, datasetTable.getId(), dataList);
+ }
+
+ Datasource engine = engineService.getDeEngine();
+ DorisConfiguration dorisConfiguration = new Gson().fromJson(engine.getConfiguration(), DorisConfiguration.class);
String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid";
String dataFile = null;
String script = null;
switch (extractType) {
case "all_scope":
- dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
- script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())));
+ dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
+ script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
break;
default:
- dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
- script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()));
+ dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
+ script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
break;
}
@@ -429,8 +468,8 @@ public class ExtractDataService {
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) {
String content = "";
- for (int i=0;i< strings.length;i++){
- content = content + strings[i] + separator;
+ for (int i = 0; i < strings.length; i++) {
+ content = content + strings[i] + separator;
}
content = content + Md5Utils.md5(content);
bw.write(content);
@@ -452,7 +491,7 @@ public class ExtractDataService {
try {
Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh");
process.waitFor();
- if(process.waitFor() != 0){
+ if (process.waitFor() != 0) {
BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream()));
String errMsg = "";
String line = "";
@@ -465,16 +504,45 @@ public class ExtractDataService {
}
throw new Exception(errMsg);
}
- }catch (Exception e){
+ } catch (Exception e) {
throw e;
- }finally {
+ } finally {
File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
FileUtils.forceDelete(deleteFile);
}
}
- private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL)throws Exception{
+ private void extractDataForSimpleMode(String extractType, String datasetId, List dataList) throws Exception {
+ String tableName;
+ switch (extractType) {
+ case "all_scope":
+ tableName = TableUtils.tmpName(TableUtils.tableName(datasetId));
+ break;
+ default:
+ tableName = TableUtils.tableName(datasetId);
+ break;
+ }
+ Datasource engine = engineService.getDeEngine();
+ JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
+ DatasourceRequest datasourceRequest = new DatasourceRequest();
+ datasourceRequest.setDatasource(engine);
+ DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
+ int pageNumber = 100; //一次插入 100条
+ int totalPage;
+ if (dataList.size() % pageNumber > 0) {
+ totalPage = dataList.size() / pageNumber + 1;
+ } else {
+ totalPage = dataList.size() / pageNumber;
+ }
+
+ for (int page = 1; page <= totalPage; page++) {
+ datasourceRequest.setQuery(ddlProvider.insertSql(tableName, dataList, page, pageNumber));
+ jdbcProvider.exec(datasourceRequest);
+ }
+ }
+
+ private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception {
generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
extractData(datasetTable, extractType);
@@ -542,70 +610,43 @@ public class ExtractDataService {
}
- private String createDorisTableColumnSql(final List datasetTableFields) {
- StringBuilder Column_Fields = new StringBuilder("dataease_uuid varchar(50), `");
- for (DatasetTableField datasetTableField : datasetTableFields) {
- Column_Fields.append(datasetTableField.getDataeaseName()).append("` ");
- Integer size = datasetTableField.getSize() * 3;
- if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
- size = 65533;
- }
- switch (datasetTableField.getDeExtractType()) {
- case 0:
- Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
- break;
- case 1:
- size = size < 50? 50 : size;
- Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
- break;
- case 2:
- Column_Fields.append("bigint").append(",`");
- break;
- case 3:
- Column_Fields.append("DOUBLE").append(",`");
- break;
- case 4:
- Column_Fields.append("TINYINT(length)".replace("length", String.valueOf(size))).append(",`");
- break;
- default:
- Column_Fields.append("varchar(length)".replace("length", String.valueOf(size))).append(",`");
- break;
- }
- }
- Column_Fields = new StringBuilder(Column_Fields.substring(0, Column_Fields.length() - 2));
- Column_Fields = new StringBuilder("(" + Column_Fields + ")\n");
- return Column_Fields.toString();
- }
-
- private void createDorisTable(String dorisTableName, String dorisTableColumnSql) throws Exception {
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ private void createEngineTable(String tableName, List datasetTableFields) throws Exception {
+ Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
- datasourceRequest.setDatasource(dorisDatasource);
- datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTableColumnSql));
+ datasourceRequest.setDatasource(engine);
+ DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
+ datasourceRequest.setQuery(ddlProvider.createTableSql(tableName, datasetTableFields));
jdbcProvider.exec(datasourceRequest);
}
- private void dropDorisTable(String dorisTableName) {
+ private void dropDorisTable(String tableName) {
try {
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
- datasourceRequest.setDatasource(dorisDatasource);
- datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName));
+ datasourceRequest.setDatasource(engine);
+ DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
+ datasourceRequest.setQuery(ddlProvider.dropTable(tableName));
jdbcProvider.exec(datasourceRequest);
} catch (Exception ignore) {
}
}
private void replaceTable(String dorisTableName) throws Exception {
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
- datasourceRequest.setDatasource(dorisDatasource);
- datasourceRequest.setQuery("ALTER TABLE DORIS_TABLE REPLACE WITH TABLE DORIS_TMP_TABLE PROPERTIES('swap' = 'false');".replace("DORIS_TABLE", dorisTableName).replace("DORIS_TMP_TABLE", DorisTableUtils.dorisTmpName(dorisTableName)));
- jdbcProvider.exec(datasourceRequest);
+ datasourceRequest.setDatasource(engine);
+ DDLProvider ddlProvider = ProviderFactory.getDDLProvider(engine.getType());
+ String[] replaceTableSql = ddlProvider.replaceTable(dorisTableName).split(";");
+ for (int i = 0; i < replaceTableSql.length; i++) {
+ if (StringUtils.isNotEmpty(replaceTableSql[i])) {
+ datasourceRequest.setQuery(replaceTableSql[i]);
+ jdbcProvider.exec(datasourceRequest);
+ }
+ }
}
private DatasetTable getDatasetTable(String datasetTableId) {
@@ -661,8 +702,26 @@ public class ExtractDataService {
return datasetTableTaskLog;
}
+ private void extractExcelDataForSimpleMode(DatasetTable datasetTable, String extractType) throws Exception {
+ List data = new ArrayList<>();
+ DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
+ List excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
+ ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
+ for (ExcelSheetData excelSheetData : excelSheetDataList) {
+ excelXlsxReader.process(new FileInputStream(excelSheetData.getPath()));
+ for (ExcelSheetData sheet : excelXlsxReader.totalSheets) {
+ if (sheet.getExcelLable().equalsIgnoreCase(excelSheetData.getExcelLable())) {
+ for (List dataItem : sheet.getData()) {
+ data.add(dataItem.toArray(new String[dataItem.size()]));
+ }
+ }
+ }
+ }
+ extractDataForSimpleMode(extractType, datasetTable.getId(), data);
+ }
+
private void extractData(DatasetTable datasetTable, String extractType) throws Exception {
- if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
+ if (StringUtils.isNotEmpty(datasetTable.getDataSourceId())) {
datasourceService.validate(datasetTable.getDataSourceId());
}
KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class);
@@ -671,16 +730,16 @@ public class ExtractDataService {
JobMeta jobMeta = null;
switch (extractType) {
case "all_scope":
- jobMeta = repository.loadJob("job_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
- transMeta = repository.loadTransformation("trans_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
+ jobMeta = repository.loadJob("job_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
+ transMeta = repository.loadTransformation("trans_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_add":
- jobMeta = repository.loadJob("job_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
- transMeta = repository.loadTransformation("trans_add_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
+ jobMeta = repository.loadJob("job_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
+ transMeta = repository.loadTransformation("trans_add_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
case "incremental_delete":
- jobMeta = repository.loadJob("job_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
- transMeta = repository.loadTransformation("trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
+ jobMeta = repository.loadJob("job_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, null);
+ transMeta = repository.loadTransformation("trans_delete_" + TableUtils.tableName(datasetTable.getId()), repositoryDirectoryInterface, null, true, "");
break;
default:
break;
@@ -738,27 +797,30 @@ public class ExtractDataService {
}
private void generateJobFile(String extractType, DatasetTable datasetTable, String columnFields) throws Exception {
+ if (engineService.isSimpleMode()) {
+ return;
+ }
String outFile;
String jobName = null;
String script = null;
- Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+ Datasource dorisDatasource = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = columnFields + ",dataease_uuid";
switch (extractType) {
case "all_scope":
- outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
- jobName = "job_" + DorisTableUtils.dorisName(datasetTable.getId());
- script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())), root_path + outFile + "." + extention);
+ outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
+ jobName = "job_" + TableUtils.tableName(datasetTable.getId());
+ script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
break;
case "incremental_add":
- outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
- jobName = "job_add_" + DorisTableUtils.dorisName(datasetTable.getId());
- script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
+ outFile = TableUtils.addName(datasetTable.getId());
+ jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
+ script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
break;
case "incremental_delete":
- outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
- script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()), root_path + outFile + "." + extention);
- jobName = "job_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
+ outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
+ script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
+ jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
break;
default:
break;
@@ -816,12 +878,15 @@ public class ExtractDataService {
datasourceRequest.setQuery(qp.wrapSql(sql));
List dorisFields = new ArrayList<>();
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
- dorisFields.add(DorisTableUtils.columnName(filed));
+ dorisFields.add(TableUtils.columnName(filed));
});
return String.join(",", dorisFields);
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) throws Exception {
+ if (engineService.isSimpleMode()) {
+ return;
+ }
TransMeta transMeta = new TransMeta();
String outFile = null;
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
@@ -902,18 +967,18 @@ public class ExtractDataService {
switch (extractType) {
case "all_scope":
- transName = "trans_" + DorisTableUtils.dorisName(datasetTable.getId());
- outFile = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
+ transName = "trans_" + TableUtils.tableName(datasetTable.getId());
+ outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
case "incremental_add":
- transName = "trans_add_" + DorisTableUtils.dorisName(datasetTable.getId());
- outFile = DorisTableUtils.dorisAddName(datasetTable.getId());
+ transName = "trans_add_" + TableUtils.tableName(datasetTable.getId());
+ outFile = TableUtils.addName(datasetTable.getId());
transMeta.setName(transName);
break;
case "incremental_delete":
- transName = "trans_delete_" + DorisTableUtils.dorisName(datasetTable.getId());
- outFile = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(datasetTable.getId()));
+ transName = "trans_delete_" + TableUtils.tableName(datasetTable.getId());
+ outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
transMeta.setName(transName);
break;
default:
@@ -965,7 +1030,7 @@ public class ExtractDataService {
return fromStep;
}
- private StepMeta excelInputStep(String Info, List datasetTableFields){
+ private StepMeta excelInputStep(String Info, List datasetTableFields) {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
List excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
@@ -975,10 +1040,10 @@ public class ExtractDataService {
List files = new ArrayList<>();
List filesRequired = new ArrayList<>();
for (ExcelSheetData excelSheetData : excelSheetDataList) {
- if(!sheetNames.contains(excelSheetData.getExcelLable())){
+ if (!sheetNames.contains(excelSheetData.getExcelLable())) {
sheetNames.add(excelSheetData.getExcelLable());
}
- if(!files.contains(excelSheetData.getPath())){
+ if (!files.contains(excelSheetData.getPath())) {
files.add(excelSheetData.getPath());
filesRequired.add("Y");
}
@@ -992,7 +1057,7 @@ public class ExtractDataService {
excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
}
excelInputMeta.setPassword("Encrypted");
- excelInputMeta.setFileName( files.toArray(new String[files.size()]));
+ excelInputMeta.setFileName(files.toArray(new String[files.size()]));
excelInputMeta.setFileRequired(filesRequired.toArray(new String[filesRequired.size()]));
excelInputMeta.setStartsWithHeader(true);
excelInputMeta.setIgnoreEmptyRows(true);
@@ -1024,9 +1089,9 @@ public class ExtractDataService {
textFileOutputMeta.setSeparator(separator);
textFileOutputMeta.setExtension(extention);
- if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name()) ) {
+ if (datasource.getType().equalsIgnoreCase(DatasourceTypes.oracle.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
- for(int i=0;i< datasetTableFields.size();i++){
+ for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getOriginName());
textFileField.setType("String");
@@ -1038,9 +1103,9 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
- }else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())){
+ } else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.mysql.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
- for(int i=0;i< datasetTableFields.size();i++){
+ for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getDataeaseName());
if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_TIME)) {
@@ -1058,9 +1123,9 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
- }else if(datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) {
+ } else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.excel.name())) {
TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1];
- for(int i=0;i< datasetTableFields.size();i++){
+ for (int i = 0; i < datasetTableFields.size(); i++) {
TextFileField textFileField = new TextFileField();
textFileField.setName(datasetTableFields.get(i).getDataeaseName());
if (datasetTableFields.get(i).getDeExtractType().equals(DeTypeConstants.DE_INT)) {
@@ -1078,7 +1143,7 @@ public class ExtractDataService {
outputFields[datasetTableFields.size()] = textFileField;
textFileOutputMeta.setOutputFields(outputFields);
- }else {
+ } else {
textFileOutputMeta.setOutputFields(new TextFileField[0]);
}
@@ -1093,7 +1158,7 @@ public class ExtractDataService {
String excelCompletion = "";
for (DatasetTableField datasetTableField : datasetTableFields) {
- if(datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)){
+ if (datasetTableField.getDeExtractType().equals(DeTypeConstants.DE_BINARY)) {
handleBinaryTypeCode.append("\n").append(handleBinaryType.replace("FIELD", datasetTableField.getDataeaseName()));
}
}
@@ -1134,7 +1199,7 @@ public class ExtractDataService {
}
public void deleteFile(String type, String dataSetTableId) {
- if(kettleFilesKeep){
+ if (kettleFilesKeep) {
return;
}
String transName = null;
@@ -1143,19 +1208,19 @@ public class ExtractDataService {
switch (type) {
case "all_scope":
- transName = "trans_" + DorisTableUtils.dorisName(dataSetTableId);
- jobName = "job_" + DorisTableUtils.dorisName(dataSetTableId);
- fileName = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(dataSetTableId));
+ transName = "trans_" + TableUtils.tableName(dataSetTableId);
+ jobName = "job_" + TableUtils.tableName(dataSetTableId);
+ fileName = TableUtils.tmpName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_add":
- transName = "trans_add_" + DorisTableUtils.dorisName(dataSetTableId);
- jobName = "job_add_" + DorisTableUtils.dorisName(dataSetTableId);
- fileName = DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(dataSetTableId));
+ transName = "trans_add_" + TableUtils.tableName(dataSetTableId);
+ jobName = "job_add_" + TableUtils.tableName(dataSetTableId);
+ fileName = TableUtils.addName(TableUtils.tableName(dataSetTableId));
break;
case "incremental_delete":
- transName = "trans_delete_" + DorisTableUtils.dorisName(dataSetTableId);
- jobName = "job_delete_" + DorisTableUtils.dorisName(dataSetTableId);
- fileName = DorisTableUtils.dorisDeleteName(DorisTableUtils.dorisName(dataSetTableId));
+ transName = "trans_delete_" + TableUtils.tableName(dataSetTableId);
+ jobName = "job_delete_" + TableUtils.tableName(dataSetTableId);
+ fileName = TableUtils.deleteName(TableUtils.tableName(dataSetTableId));
break;
default:
break;
@@ -1165,29 +1230,33 @@ public class ExtractDataService {
deleteFile(root_path + transName + ".ktr");
}
- private void deleteExcelFile(DatasetTable datasetTable, ListdatasetTableIds){
+ private void deleteExcelFile(DatasetTable datasetTable, List datasetTableIds) {
List datasetTables = dataSetTableService.list(datasetTableIds);
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
Boolean allIsFinished = true;
for (DatasetTable table : datasetTables) {
- for(ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()){
- if(data.getPath().equalsIgnoreCase(excelSheetData.getPath())){
- if(StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())){
+ for (ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
+ if (data.getPath().equalsIgnoreCase(excelSheetData.getPath())) {
+ if (StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) {
allIsFinished = false;
}
}
}
}
- if(allIsFinished){
+ if (allIsFinished) {
deleteFile(excelSheetData.getPath());
- }else {
- try { Thread.sleep(5000); }catch (Exception ignore){}
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (Exception ignore) {
+ }
deleteExcelFile(datasetTable, datasetTableIds);
}
}
}
- private void deleteFile(String filePath){
- if(StringUtils.isEmpty(filePath)){
+
+ private void deleteFile(String filePath) {
+ if (StringUtils.isEmpty(filePath)) {
return;
}
try {
@@ -1196,6 +1265,7 @@ public class ExtractDataService {
} catch (Exception e) {
}
}
+
public boolean isKettleRunning() {
try {
if (!InetAddress.getByName(carte).isReachable(1000)) {
diff --git a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
index ea36b7d970..20c54d4786 100644
--- a/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
@@ -5,7 +5,6 @@ import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.Datasource;
import io.dataease.commons.constants.ColumnPermissionConstants;
-import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
import io.dataease.i18n.Translator;
import io.dataease.provider.datasource.DatasourceProvider;
@@ -15,7 +14,8 @@ import io.dataease.service.dataset.*;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
-import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.QueryProvider;
+import io.dataease.service.engine.EngineService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@@ -40,6 +40,8 @@ public class DirectFieldService implements DataSetFieldService {
private DataSetTableUnionService dataSetTableUnionService;
@Resource
private PermissionService permissionService;
+ @Resource
+ private EngineService engineService;
@Override
public List