dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ck 支持直连模式；es支持 6.*

Browse Source
This commit is contained in:
taojinlong 2021-09-22 18:46:29 +08:00
parent 848e960a44
commit c19f63590b
23 changed files with 243 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/pom.xml
View File

@ -342,6 +342,12 @@
<artifactId>spring-boot-starter-data-ldap</artifactId> <artifactId>spring-boot-starter-data-ldap</artifactId>
</dependency>--> </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.dataease.kettle</groupId>-->
<!-- <artifactId>ck-plugin</artifactId>-->
<!-- <version>1.0</version>-->
<!-- </dependency>-->
<dependency> <dependency>
<groupId>ru.yandex.clickhouse</groupId> <groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId> <artifactId>clickhouse-jdbc</artifactId>

2
backend/src/main/java/io/dataease/base/domain/Datasource.java
View File

@ -21,8 +21,6 @@ public class Datasource implements Serializable {
private String status; private String status;
private String computeType;
private String configuration; private String configuration;
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;

70
backend/src/main/java/io/dataease/base/domain/DatasourceExample.java
View File

@ -643,76 +643,6 @@ public class DatasourceExample {
addCriterion("`status` not between", value1, value2, "status"); addCriterion("`status` not between", value1, value2, "status");
return (Criteria) this; return (Criteria) this;
} }
public Criteria andComputeTypeIsNull() {
addCriterion("compute_type is null");
return (Criteria) this;
}
public Criteria andComputeTypeIsNotNull() {
addCriterion("compute_type is not null");
return (Criteria) this;
}
public Criteria andComputeTypeEqualTo(String value) {
addCriterion("compute_type =", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeNotEqualTo(String value) {
addCriterion("compute_type <>", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeGreaterThan(String value) {
addCriterion("compute_type >", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeGreaterThanOrEqualTo(String value) {
addCriterion("compute_type >=", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeLessThan(String value) {
addCriterion("compute_type <", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeLessThanOrEqualTo(String value) {
addCriterion("compute_type <=", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeLike(String value) {
addCriterion("compute_type like", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeNotLike(String value) {
addCriterion("compute_type not like", value, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeIn(List<String> values) {
addCriterion("compute_type in", values, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeNotIn(List<String> values) {
addCriterion("compute_type not in", values, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeBetween(String value1, String value2) {
addCriterion("compute_type between", value1, value2, "computeType");
return (Criteria) this;
}
public Criteria andComputeTypeNotBetween(String value1, String value2) {
addCriterion("compute_type not between", value1, value2, "computeType");
return (Criteria) this;
}
} }
public static class Criteria extends GeneratedCriteria { public static class Criteria extends GeneratedCriteria {

31
backend/src/main/java/io/dataease/base/mapper/DatasourceMapper.xml
View File

@ -10,7 +10,6 @@
<result column="update_time" jdbcType="BIGINT" property="updateTime" /> <result column="update_time" jdbcType="BIGINT" property="updateTime" />
<result column="create_by" jdbcType="VARCHAR" property="createBy" /> <result column="create_by" jdbcType="VARCHAR" property="createBy" />
<result column="status" jdbcType="VARCHAR" property="status" /> <result column="status" jdbcType="VARCHAR" property="status" />
<result column="compute_type" jdbcType="VARCHAR" property="computeType" />
</resultMap> </resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.base.domain.Datasource"> <resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="io.dataease.base.domain.Datasource">
<result column="configuration" jdbcType="LONGVARCHAR" property="configuration" /> <result column="configuration" jdbcType="LONGVARCHAR" property="configuration" />
@ -74,7 +73,7 @@
</where> </where>
</sql> </sql>
<sql id="Base_Column_List"> <sql id="Base_Column_List">
id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`, compute_type id, `name`, `desc`, `type`, create_time, update_time, create_by, `status`
</sql> </sql>
<sql id="Blob_Column_List"> <sql id="Blob_Column_List">
configuration configuration
@ -130,12 +129,12 @@
<insert id="insert" parameterType="io.dataease.base.domain.Datasource"> <insert id="insert" parameterType="io.dataease.base.domain.Datasource">
insert into datasource (id, `name`, `desc`, insert into datasource (id, `name`, `desc`,
`type`, create_time, update_time, `type`, create_time, update_time,
create_by, `status`, compute_type, create_by, `status`, configuration
configuration) )
values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR}, values (#{id,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{desc,jdbcType=VARCHAR},
#{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT}, #{type,jdbcType=VARCHAR}, #{createTime,jdbcType=BIGINT}, #{updateTime,jdbcType=BIGINT},
#{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{computeType,jdbcType=VARCHAR}, #{createBy,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR}, #{configuration,jdbcType=LONGVARCHAR}
#{configuration,jdbcType=LONGVARCHAR}) )
</insert> </insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.Datasource"> <insert id="insertSelective" parameterType="io.dataease.base.domain.Datasource">
insert into datasource insert into datasource
@ -164,9 +163,6 @@
<if test="status != null"> <if test="status != null">
`status`, `status`,
</if> </if>
<if test="computeType != null">
compute_type,
</if>
<if test="configuration != null"> <if test="configuration != null">
configuration, configuration,
</if> </if>
@ -196,9 +192,6 @@
<if test="status != null"> <if test="status != null">
#{status,jdbcType=VARCHAR}, #{status,jdbcType=VARCHAR},
</if> </if>
<if test="computeType != null">
#{computeType,jdbcType=VARCHAR},
</if>
<if test="configuration != null"> <if test="configuration != null">
#{configuration,jdbcType=LONGVARCHAR}, #{configuration,jdbcType=LONGVARCHAR},
</if> </if>
@ -237,9 +230,6 @@
<if test="record.status != null"> <if test="record.status != null">
`status` = #{record.status,jdbcType=VARCHAR}, `status` = #{record.status,jdbcType=VARCHAR},
</if> </if>
<if test="record.computeType != null">
compute_type = #{record.computeType,jdbcType=VARCHAR},
</if>
<if test="record.configuration != null"> <if test="record.configuration != null">
configuration = #{record.configuration,jdbcType=LONGVARCHAR}, configuration = #{record.configuration,jdbcType=LONGVARCHAR},
</if> </if>
@ -258,7 +248,6 @@
update_time = #{record.updateTime,jdbcType=BIGINT}, update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR}, create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR}, `status` = #{record.status,jdbcType=VARCHAR},
compute_type = #{record.computeType,jdbcType=VARCHAR},
configuration = #{record.configuration,jdbcType=LONGVARCHAR} configuration = #{record.configuration,jdbcType=LONGVARCHAR}
<if test="_parameter != null"> <if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" /> <include refid="Update_By_Example_Where_Clause" />
@ -273,8 +262,7 @@
create_time = #{record.createTime,jdbcType=BIGINT}, create_time = #{record.createTime,jdbcType=BIGINT},
update_time = #{record.updateTime,jdbcType=BIGINT}, update_time = #{record.updateTime,jdbcType=BIGINT},
create_by = #{record.createBy,jdbcType=VARCHAR}, create_by = #{record.createBy,jdbcType=VARCHAR},
`status` = #{record.status,jdbcType=VARCHAR}, `status` = #{record.status,jdbcType=VARCHAR}
compute_type = #{record.computeType,jdbcType=VARCHAR}
<if test="_parameter != null"> <if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" /> <include refid="Update_By_Example_Where_Clause" />
</if> </if>
@ -303,9 +291,6 @@
<if test="status != null"> <if test="status != null">
`status` = #{status,jdbcType=VARCHAR}, `status` = #{status,jdbcType=VARCHAR},
</if> </if>
<if test="computeType != null">
compute_type = #{computeType,jdbcType=VARCHAR},
</if>
<if test="configuration != null"> <if test="configuration != null">
configuration = #{configuration,jdbcType=LONGVARCHAR}, configuration = #{configuration,jdbcType=LONGVARCHAR},
</if> </if>
@ -321,7 +306,6 @@
update_time = #{updateTime,jdbcType=BIGINT}, update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR}, create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR}, `status` = #{status,jdbcType=VARCHAR},
compute_type = #{computeType,jdbcType=VARCHAR},
configuration = #{configuration,jdbcType=LONGVARCHAR} configuration = #{configuration,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR}
</update> </update>
@ -333,8 +317,7 @@
create_time = #{createTime,jdbcType=BIGINT}, create_time = #{createTime,jdbcType=BIGINT},
update_time = #{updateTime,jdbcType=BIGINT}, update_time = #{updateTime,jdbcType=BIGINT},
create_by = #{createBy,jdbcType=VARCHAR}, create_by = #{createBy,jdbcType=VARCHAR},
`status` = #{status,jdbcType=VARCHAR}, `status` = #{status,jdbcType=VARCHAR}
compute_type = #{computeType,jdbcType=VARCHAR}
where id = #{id,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR}
</update> </update>
</mapper> </mapper>

3
backend/src/main/java/io/dataease/config/CommonConfig.java
View File

@ -4,7 +4,10 @@ import com.alibaba.fastjson.JSONObject;
import com.fit2cloud.autoconfigure.QuartzAutoConfiguration; import com.fit2cloud.autoconfigure.QuartzAutoConfiguration;
import io.dataease.base.domain.Datasource; import io.dataease.base.domain.Datasource;
import io.dataease.commons.utils.CommonThreadPool; import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.kettle.databaseplugin.ClickhouseDatabaseMeta;
import org.pentaho.di.core.KettleEnvironment; import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.plugins.DatabasePluginType;
import org.pentaho.di.core.plugins.PluginRegistry;
import org.pentaho.di.repository.filerep.KettleFileRepository; import org.pentaho.di.repository.filerep.KettleFileRepository;
import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta; import org.pentaho.di.repository.filerep.KettleFileRepositoryMeta;
import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.AutoConfigureBefore;

2
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
View File

@ -7,7 +7,7 @@ public enum DatasourceTypes {
sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""),
doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""),
oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""),
ch("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"), ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
es("es", "es", "", "\"", "\"", "\"", "\""); es("es", "es", "", "\"", "\"", "\"", "\"");
private String feature; private String feature;

2
backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java
View File

@ -10,5 +10,7 @@ public class EsConfigDTO {
private String url; private String url;
private String username; private String username;
private String password; private String password;
private String version;
private String uri;
private String dataSourceType = "es"; private String dataSourceType = "es";
} }

1
backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java
View File

@ -12,6 +12,7 @@ public class EsReponse {
private String cursor; private String cursor;
private Integer status; private Integer status;
private Error error; private Error error;
private String version;
@Data @Data
public class Error{ public class Error{

57
backend/src/main/java/io/dataease/datasource/provider/EsProvider.java
View File

@ -1,5 +1,7 @@
package io.dataease.datasource.provider; package io.dataease.datasource.provider;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.mchange.v2.c3p0.ComboPooledDataSource; import com.mchange.v2.c3p0.ComboPooledDataSource;
import io.dataease.commons.utils.HttpClientConfig; import io.dataease.commons.utils.HttpClientConfig;
@ -10,6 +12,7 @@ import io.dataease.datasource.dto.es.Requst;
import io.dataease.datasource.dto.es.RequstWithCursor; import io.dataease.datasource.dto.es.RequstWithCursor;
import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DataEaseException; import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
import io.dataease.provider.es.EsQueryProvider; import io.dataease.provider.es.EsQueryProvider;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@ -54,7 +57,7 @@ public class EsProvider extends DatasourceProvider {
Requst requst = new Requst(); Requst requst = new Requst();
requst.setQuery(dsr.getQuery()); requst.setQuery(dsr.getQuery());
requst.setFetch_size(dsr.getFetchSize()); requst.setFetch_size(dsr.getFetchSize());
String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + "_sql?format=json" : esConfigDTO.getUrl() + "/" + "_sql?format=json"; String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + esConfigDTO.getUri() + "?format=json" : esConfigDTO.getUrl() + "/" + esConfigDTO.getUri() + "?format=json";
String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig);
EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
@ -84,7 +87,7 @@ public class EsProvider extends DatasourceProvider {
public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception { public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception {
List<String[]> list = new LinkedList<>(); List<String[]> list = new LinkedList<>();
try { try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
list = fetchResult(response); list = fetchResult(response);
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(e); DataEaseException.throwException(e);
@ -110,7 +113,7 @@ public class EsProvider extends DatasourceProvider {
public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception { public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> tableFileds = new ArrayList<>(); List<TableFiled> tableFileds = new ArrayList<>();
try { try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
tableFileds = fetchResultField(response); tableFileds = fetchResultField(response);
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(e); DataEaseException.throwException(e);
@ -140,7 +143,7 @@ public class EsProvider extends DatasourceProvider {
public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception { public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
Map<String, List> result = new HashMap<>(); Map<String, List> result = new HashMap<>();
try { try {
String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
result.put("dataList", fetchResult(response)); result.put("dataList", fetchResult(response));
result.put("fieldList", fetchResultField(response)); result.put("fieldList", fetchResultField(response));
} catch (Exception e) { } catch (Exception e) {
@ -157,7 +160,7 @@ public class EsProvider extends DatasourceProvider {
public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception { public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<String> tables = new ArrayList<>(); List<String> tables = new ArrayList<>();
try { try {
String response = exexQuery(datasourceRequest, "show tables", "_sql?format=json"); String response = exexQuery(datasourceRequest, "show tables", "?format=json");
tables = fetchTables(response); tables = fetchTables(response);
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(e); DataEaseException.throwException(e);
@ -176,6 +179,9 @@ public class EsProvider extends DatasourceProvider {
if(row.length == 3 && row[1].equalsIgnoreCase("TABLE") && row[2].equalsIgnoreCase("INDEX")){ if(row.length == 3 && row[1].equalsIgnoreCase("TABLE") && row[2].equalsIgnoreCase("INDEX")){
tables.add(row[0]); tables.add(row[0]);
} }
if(row.length == 2 && row[1].equalsIgnoreCase("BASE TABLE")){
tables.add(row[0]);
}
} }
return tables; return tables;
} }
@ -189,7 +195,7 @@ public class EsProvider extends DatasourceProvider {
public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception { public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> tableFileds = new ArrayList<>(); List<TableFiled> tableFileds = new ArrayList<>();
try { try {
String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "_sql?format=json"); String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "?format=json");
tableFileds = fetchResultField(response); tableFileds = fetchResultField(response);
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(e); DataEaseException.throwException(e);
@ -199,16 +205,34 @@ public class EsProvider extends DatasourceProvider {
@Override @Override
public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
String response = exexGetQuery(datasourceRequest);
String version = JSONObject.parseObject(response).getJSONObject("version").getString("number");
if(Integer.valueOf(version.substring(0,1)) < 6 ){
throw new Exception(Translator.get("i18n_es_limit"));
}
if(Integer.valueOf(version.substring(2,3)) < 3 ){
throw new Exception(Translator.get("i18n_es_limit"));
}
if(Integer.valueOf(version.substring(0,1)) == 6 ) {
esConfigDTO.setUri("_xpack/sql");
}
if(Integer.valueOf(version.substring(0,1)) == 7 ) {
esConfigDTO.setUri("_sql");
}
datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfigDTO));
getTables(datasourceRequest); getTables(datasourceRequest);
} }
private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){ private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){
EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class); EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
uri = esConfigDTO.getUri()+uri;
HttpClientConfig httpClientConfig = new HttpClientConfig(); HttpClientConfig httpClientConfig = new HttpClientConfig();
String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword(); if(StringUtils.isNotEmpty(esConfigDTO.getUsername()) && StringUtils.isNotEmpty(esConfigDTO.getPassword())){
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
}
Requst requst = new Requst(); Requst requst = new Requst();
requst.setQuery(sql); requst.setQuery(sql);
@ -218,4 +242,17 @@ public class EsProvider extends DatasourceProvider {
return response; return response;
} }
private String exexGetQuery(DatasourceRequest datasourceRequest){
EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class);
HttpClientConfig httpClientConfig = new HttpClientConfig();
if(StringUtils.isNotEmpty(esConfigDTO.getUsername()) && StringUtils.isNotEmpty(esConfigDTO.getPassword())){
String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword();
byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8));
httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth));
}
String response = HttpClientUtil.get(esConfigDTO.getUrl(), httpClientConfig);
return response;
}
} }

37
backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
View File

@ -7,6 +7,7 @@ import io.dataease.datasource.dto.*;
import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DataEaseException; import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator; import io.dataease.i18n.Translator;
import io.dataease.provider.QueryProvider;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.beans.PropertyVetoException; import java.beans.PropertyVetoException;
@ -127,7 +128,7 @@ public class JdbcProvider extends DatasourceProvider {
connection = getConnectionFromPool(datasourceRequest); connection = getConnectionFromPool(datasourceRequest);
Statement stat = connection.createStatement(); Statement stat = connection.createStatement();
rs = stat.executeQuery(datasourceRequest.getQuery()); rs = stat.executeQuery(datasourceRequest.getQuery());
return fetchResultField(rs); return fetchResultField(rs, datasourceRequest);
} catch (SQLException e) { } catch (SQLException e) {
DataEaseException.throwException(e); DataEaseException.throwException(e);
} catch (Exception e) { } catch (Exception e) {
@ -152,7 +153,7 @@ public class JdbcProvider extends DatasourceProvider {
Statement stat = connection.createStatement(); Statement stat = connection.createStatement();
rs = stat.executeQuery(datasourceRequest.getQuery()); rs = stat.executeQuery(datasourceRequest.getQuery());
dataList = fetchResult(rs); dataList = fetchResult(rs);
fieldList = fetchResultField(rs); fieldList = fetchResultField(rs, datasourceRequest);
result.put("dataList", dataList); result.put("dataList", dataList);
result.put("fieldList", fieldList); result.put("fieldList", fieldList);
return result; return result;
@ -168,10 +169,11 @@ public class JdbcProvider extends DatasourceProvider {
return new HashMap<>(); return new HashMap<>();
} }
private List<TableFiled> fetchResultField(ResultSet rs) throws Exception { private List<TableFiled> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
List<TableFiled> fieldList = new ArrayList<>(); List<TableFiled> fieldList = new ArrayList<>();
ResultSetMetaData metaData = rs.getMetaData(); ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount(); int columnCount = metaData.getColumnCount();
QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
for (int j = 0; j < columnCount; j++) { for (int j = 0; j < columnCount; j++) {
String f = metaData.getColumnName(j + 1); String f = metaData.getColumnName(j + 1);
String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f; String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f;
@ -180,7 +182,12 @@ public class JdbcProvider extends DatasourceProvider {
field.setFieldName(l); field.setFieldName(l);
field.setRemarks(l); field.setRemarks(l);
field.setFieldType(t); field.setFieldType(t);
field.setFieldSize(metaData.getColumnDisplaySize(j + 1));
if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
field.setFieldSize(qp.transFieldSize(t));
}else {
field.setFieldSize(metaData.getColumnDisplaySize(j + 1));
}
if(t.equalsIgnoreCase("LONG")){field.setFieldSize(65533);} //oracle LONG if(t.equalsIgnoreCase("LONG")){field.setFieldSize(65533);} //oracle LONG
if(StringUtils.isNotEmpty(t) && t.toLowerCase().contains("date") && field.getFieldSize() < 50 ){ if(StringUtils.isNotEmpty(t) && t.toLowerCase().contains("date") && field.getFieldSize() < 50 ){
field.setFieldSize(50); field.setFieldSize(50);
@ -251,19 +258,19 @@ public class JdbcProvider extends DatasourceProvider {
while (resultSet.next()) { while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME"); String tableName = resultSet.getString("TABLE_NAME");
String database = null; String database = null;
if(datasourceRequest.getDatasource().getType().equalsIgnoreCase("ch")){ if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
database = resultSet.getString("TABLE_SCHEM"); database = resultSet.getString("TABLE_SCHEM");
}else { }else {
database = resultSet.getString("TABLE_CAT"); database = resultSet.getString("TABLE_CAT");
} }
if(database != null){ if(database != null){
if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) { if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) {
TableFiled tableFiled = getTableFiled(resultSet); TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
list.add(tableFiled); list.add(tableFiled);
} }
}else { }else {
if (tableName.equals(datasourceRequest.getTable())) { if (tableName.equals(datasourceRequest.getTable())) {
TableFiled tableFiled = getTableFiled(resultSet); TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
list.add(tableFiled); list.add(tableFiled);
} }
} }
@ -281,7 +288,7 @@ public class JdbcProvider extends DatasourceProvider {
return list; return list;
} }
private TableFiled getTableFiled(ResultSet resultSet) throws SQLException { private TableFiled getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
TableFiled tableFiled = new TableFiled(); TableFiled tableFiled = new TableFiled();
String colName = resultSet.getString("COLUMN_NAME"); String colName = resultSet.getString("COLUMN_NAME");
tableFiled.setFieldName(colName); tableFiled.setFieldName(colName);
@ -290,13 +297,19 @@ public class JdbcProvider extends DatasourceProvider {
remarks = colName; remarks = colName;
} }
tableFiled.setRemarks(remarks); tableFiled.setRemarks(remarks);
tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE")));
String dbType = resultSet.getString("TYPE_NAME").toUpperCase(); String dbType = resultSet.getString("TYPE_NAME").toUpperCase();
tableFiled.setFieldType(dbType); tableFiled.setFieldType(dbType);
if(dbType.equalsIgnoreCase("LONG")){tableFiled.setFieldSize(65533);} if(dbType.equalsIgnoreCase("LONG")){tableFiled.setFieldSize(65533);}
if(StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50 ){ if(StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50 ){
tableFiled.setFieldSize(50); tableFiled.setFieldSize(50);
} }
if(datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())){
QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
tableFiled.setFieldSize(qp.transFieldSize(dbType));
}else {
tableFiled.setFieldSize(Integer.valueOf(resultSet.getString("COLUMN_SIZE")));
}
return tableFiled; return tableFiled;
} }
@ -442,7 +455,7 @@ public class JdbcProvider extends DatasourceProvider {
driver = pgConfigration.getDriver(); driver = pgConfigration.getDriver();
jdbcurl = pgConfigration.getJdbc(); jdbcurl = pgConfigration.getJdbc();
break; break;
case ch: case ck:
CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class); CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
username = chConfigration.getUsername(); username = chConfigration.getUsername();
password = chConfigration.getPassword(); password = chConfigration.getPassword();
@ -506,7 +519,7 @@ public class JdbcProvider extends DatasourceProvider {
dataSource.setJdbcUrl(pgConfigration.getJdbc()); dataSource.setJdbcUrl(pgConfigration.getJdbc());
jdbcDTO = pgConfigration; jdbcDTO = pgConfigration;
break; break;
case ch: case ck:
CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class); CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
dataSource.setUser(chConfigration.getUsername()); dataSource.setUser(chConfigration.getUsername());
dataSource.setDriverClass(chConfigration.getDriver()); dataSource.setDriverClass(chConfigration.getDriver());
@ -568,7 +581,7 @@ public class JdbcProvider extends DatasourceProvider {
throw new Exception(Translator.get("i18n_schema_is_empty")); throw new Exception(Translator.get("i18n_schema_is_empty"));
} }
return "SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg%' AND tablename NOT LIKE 'sql_%' AND schemaname='SCHEMA' ;".replace("SCHEMA", pgConfigration.getSchema()); return "SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg%' AND tablename NOT LIKE 'sql_%' AND schemaname='SCHEMA' ;".replace("SCHEMA", pgConfigration.getSchema());
case ch: case ck:
CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class); CHConfigration chConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfigration.class);
return "SELECT name FROM system.tables where database='DATABASE';".replace("DATABASE", chConfigration.getDataBase()); return "SELECT name FROM system.tables where database='DATABASE';".replace("DATABASE", chConfigration.getDataBase());
default: default:

4
backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java
View File

@ -52,8 +52,8 @@ public class ProviderFactory implements ApplicationContextAware {
return context.getBean("oracleQuery", QueryProvider.class); return context.getBean("oracleQuery", QueryProvider.class);
case es: case es:
return context.getBean("esQuery", QueryProvider.class); return context.getBean("esQuery", QueryProvider.class);
case ch: case ck:
return context.getBean("chQuery", QueryProvider.class); return context.getBean("ckQuery", QueryProvider.class);
default: default:
return context.getBean("mysqlQuery", QueryProvider.class); return context.getBean("mysqlQuery", QueryProvider.class);
} }

7
backend/src/main/java/io/dataease/datasource/service/DatasourceService.java
View File

@ -65,9 +65,10 @@ public class DatasourceService {
datasource.setUpdateTime(currentTimeMillis); datasource.setUpdateTime(currentTimeMillis);
datasource.setCreateTime(currentTimeMillis); datasource.setCreateTime(currentTimeMillis);
datasource.setCreateBy(String.valueOf(AuthUtils.getUser().getUsername())); datasource.setCreateBy(String.valueOf(AuthUtils.getUser().getUsername()));
checkAndUpdateDatasourceStatus(datasource);
datasourceMapper.insertSelective(datasource); datasourceMapper.insertSelective(datasource);
handleConnectionPool(datasource, "add"); handleConnectionPool(datasource, "add");
checkAndUpdateDatasourceStatus(datasource);
return datasource; return datasource;
} }
@ -129,9 +130,9 @@ public class DatasourceService {
checkName(datasource); checkName(datasource);
datasource.setCreateTime(null); datasource.setCreateTime(null);
datasource.setUpdateTime(System.currentTimeMillis()); datasource.setUpdateTime(System.currentTimeMillis());
checkAndUpdateDatasourceStatus(datasource);
datasourceMapper.updateByPrimaryKeySelective(datasource); datasourceMapper.updateByPrimaryKeySelective(datasource);
handleConnectionPool(datasource, "edit"); handleConnectionPool(datasource, "edit");
checkAndUpdateDatasourceStatus(datasource);
} }
public ResultHolder validate(Datasource datasource) throws Exception { public ResultHolder validate(Datasource datasource) throws Exception {
@ -256,10 +257,8 @@ public class DatasourceService {
datasourceRequest.setDatasource(datasource); datasourceRequest.setDatasource(datasource);
datasourceProvider.checkStatus(datasourceRequest); datasourceProvider.checkStatus(datasourceRequest);
datasource.setStatus("Success"); datasource.setStatus("Success");
datasourceMapper.updateByPrimaryKeySelective(datasource);
} catch (Exception e) { } catch (Exception e) {
datasource.setStatus("Error"); datasource.setStatus("Error");
datasourceMapper.updateByPrimaryKeySelective(datasource);
} }
} }

3
backend/src/main/java/io/dataease/provider/QueryProvider.java
View File

@ -54,6 +54,9 @@ public abstract class QueryProvider {
public abstract String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList); public abstract String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList);
public Integer transFieldSize(String type){
return 50;
};
/** /**
* 单指标汇总 * 单指标汇总
* *

10
backend/src/main/java/io/dataease/provider/ch/CHConstants.java → backend/src/main/java/io/dataease/provider/ck/CKConstants.java
View File

@ -1,17 +1,17 @@
package io.dataease.provider.ch; package io.dataease.provider.ck;
import io.dataease.provider.SQLConstants; import io.dataease.provider.SQLConstants;
import static io.dataease.datasource.constants.DatasourceTypes.ch; import static io.dataease.datasource.constants.DatasourceTypes.ck;
/** /**
* @Author gin * @Author gin
* @Date 2021/7/8 7:22 下午 * @Date 2021/7/8 7:22 下午
*/ */
public class CHConstants extends SQLConstants { public class CKConstants extends SQLConstants {
public static final String KEYWORD_TABLE = ch.getKeywordPrefix() + "%s" + ch.getKeywordSuffix(); public static final String KEYWORD_TABLE = ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix();
public static final String KEYWORD_FIX = "%s." + ch.getKeywordPrefix() + "%s" + ch.getKeywordSuffix(); public static final String KEYWORD_FIX = "%s." + ck.getKeywordPrefix() + "%s" + ck.getKeywordSuffix();
public static final String toInt32 = "toInt32(%s)"; public static final String toInt32 = "toInt32(%s)";

159
backend/src/main/java/io/dataease/provider/ch/CHQueryProvider.java → backend/src/main/java/io/dataease/provider/ck/CKQueryProvider.java
View File

@ -1,4 +1,4 @@
package io.dataease.provider.ch; package io.dataease.provider.ck;
import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableFieldExample; import io.dataease.base.domain.DatasetTableFieldExample;
@ -32,8 +32,8 @@ import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX;
* @Author gin * @Author gin
* @Date 2021/5/17 2:43 下午 * @Date 2021/5/17 2:43 下午
*/ */
@Service("chQuery") @Service("ckQuery")
public class CHQueryProvider extends QueryProvider { public class CKQueryProvider extends QueryProvider {
@Resource @Resource
private DatasetTableFieldMapper datasetTableFieldMapper; private DatasetTableFieldMapper datasetTableFieldMapper;
@ -80,6 +80,25 @@ public class CHQueryProvider extends QueryProvider {
} }
} }
@Override
public Integer transFieldSize(String field){
Integer type = transFieldType(field);
switch (type) {
case 0:
return 65533;
case 1:
return 60;
case 2:
return 0;
case 3:
return 0;
case 4:
return 0;
default:
return 65533;
}
}
@Override @Override
public String createSQLPreview(String sql, String orderBy) { public String createSQLPreview(String sql, String orderBy) {
return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 0,1000"; return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 0,1000";
@ -88,7 +107,7 @@ public class CHQueryProvider extends QueryProvider {
@Override @Override
public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds) { public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds) {
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
@ -100,9 +119,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(f.getOriginName(), tableObj); originField = calcFieldRegex(f.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
String fieldName = ""; String fieldName = "";
@ -110,29 +129,29 @@ public class CHQueryProvider extends QueryProvider {
if (f.getDeExtractType() == DeTypeConstants.DE_TIME) { if (f.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) { if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) {
if(f.getType().equalsIgnoreCase("DATE")){ if(f.getType().equalsIgnoreCase("DATE")){
fieldName = String.format(CHConstants.toInt32, String.format(CHConstants.toDateTime, originField)) + "*1000"; fieldName = String.format(CKConstants.toInt32, String.format(CKConstants.toDateTime, originField)) + "*1000";
}else { }else {
fieldName = String.format(CHConstants.toInt32, originField) + "*1000"; fieldName = String.format(CKConstants.toInt32, originField) + "*1000";
} }
} else { } else {
fieldName = originField; fieldName = originField;
} }
} else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) { } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) {
if (f.getDeType() == DeTypeConstants.DE_INT) { if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format(CHConstants.toInt64, originField); fieldName = String.format(CKConstants.toInt64, originField);
} else if (f.getDeType() == DeTypeConstants.DE_FLOAT) { } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) {
fieldName = String.format(CHConstants.toFloat64, originField); fieldName = String.format(CKConstants.toFloat64, originField);
} else if (f.getDeType() == DeTypeConstants.DE_TIME) { } else if (f.getDeType() == DeTypeConstants.DE_TIME) {
fieldName = String.format(CHConstants.toDateTime, originField); fieldName = String.format(CKConstants.toDateTime, originField);
} else { } else {
fieldName = originField; fieldName = originField;
} }
} else { } else {
if (f.getDeType() == DeTypeConstants.DE_TIME) { if (f.getDeType() == DeTypeConstants.DE_TIME) {
String cast = String.format(CHConstants.toFloat64, originField); String cast = String.format(CKConstants.toFloat64, originField);
fieldName = String.format(CHConstants.toDateTime, cast); fieldName = String.format(CKConstants.toDateTime, cast);
} else if (f.getDeType() == DeTypeConstants.DE_INT) { } else if (f.getDeType() == DeTypeConstants.DE_INT) {
fieldName = String.format(CHConstants.toInt64, originField); fieldName = String.format(CKConstants.toInt64, originField);
} else { } else {
fieldName = originField; fieldName = originField;
} }
@ -180,7 +199,7 @@ public class CHQueryProvider extends QueryProvider {
@Override @Override
public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) { public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) {
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
@ -194,9 +213,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj); originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段 // 处理横轴字段
@ -224,9 +243,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj); originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段 // 处理纵轴字段
@ -274,7 +293,7 @@ public class CHQueryProvider extends QueryProvider {
ST st = stg.getInstanceOf("querySql"); ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder() SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(CHConstants.BRACKETS, sql)) .tableName(String.format(CKConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build(); .build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@ -286,7 +305,7 @@ public class CHQueryProvider extends QueryProvider {
@Override @Override
public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) { public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds) {
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
@ -300,9 +319,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj); originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段 // 处理横轴字段
@ -347,7 +366,7 @@ public class CHQueryProvider extends QueryProvider {
ST st = stg.getInstanceOf("previewSql"); ST st = stg.getInstanceOf("previewSql");
st.add("isGroup", false); st.add("isGroup", false);
SQLObj tableSQL = SQLObj.builder() SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(CHConstants.BRACKETS, sql)) .tableName(String.format(CKConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build(); .build();
if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders);
@ -369,7 +388,7 @@ public class CHQueryProvider extends QueryProvider {
@Override @Override
public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds) { public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds) {
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
@ -386,9 +405,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj); originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段 // 处理横轴字段
@ -416,9 +435,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj); originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段 // 处理纵轴字段
@ -466,7 +485,7 @@ public class CHQueryProvider extends QueryProvider {
ST st = stg.getInstanceOf("querySql"); ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder() SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(CHConstants.BRACKETS, sql)) .tableName(String.format(CKConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build(); .build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@ -483,7 +502,7 @@ public class CHQueryProvider extends QueryProvider {
@Override @Override
public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds) { public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds) {
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> xFields = new ArrayList<>(); List<SQLObj> xFields = new ArrayList<>();
@ -497,9 +516,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(x.getOriginName(), tableObj); originField = calcFieldRegex(x.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段 // 处理横轴字段
@ -530,9 +549,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj); originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段 // 处理纵轴字段
@ -580,7 +599,7 @@ public class CHQueryProvider extends QueryProvider {
ST st = stg.getInstanceOf("querySql"); ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder() SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(CHConstants.BRACKETS, sql)) .tableName(String.format(CKConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build(); .build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@ -603,7 +622,7 @@ public class CHQueryProvider extends QueryProvider {
public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) { public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartCustomFilterDTO> customFilter, List<ChartExtFilterRequest> extFilterRequestList) {
// 字段汇总 排序等 // 字段汇总 排序等
SQLObj tableObj = SQLObj.builder() SQLObj tableObj = SQLObj.builder()
.tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CHConstants.KEYWORD_TABLE, table)) .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(CKConstants.KEYWORD_TABLE, table))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build(); .build();
List<SQLObj> yFields = new ArrayList<>(); List<SQLObj> yFields = new ArrayList<>();
@ -617,9 +636,9 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originField = calcFieldRegex(y.getOriginName(), tableObj); originField = calcFieldRegex(y.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} else { } else {
originField = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); originField = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName());
} }
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段 // 处理纵轴字段
@ -662,7 +681,7 @@ public class CHQueryProvider extends QueryProvider {
ST st = stg.getInstanceOf("querySql"); ST st = stg.getInstanceOf("querySql");
SQLObj tableSQL = SQLObj.builder() SQLObj tableSQL = SQLObj.builder()
.tableName(String.format(CHConstants.BRACKETS, sql)) .tableName(String.format(CKConstants.BRACKETS, sql))
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build(); .build();
if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres);
@ -757,17 +776,17 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj); originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else { } else {
originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} }
if (field.getDeType() == DeTypeConstants.DE_TIME) { if (field.getDeType() == DeTypeConstants.DE_TIME) {
if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
whereName = String.format(CHConstants.toDateTime, originName); whereName = String.format(CKConstants.toDateTime, originName);
} }
if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) { if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) {
String cast = String.format(CHConstants.toFloat64, originName); String cast = String.format(CKConstants.toFloat64, originName);
whereName = String.format(CHConstants.toDateTime, cast); whereName = String.format(CKConstants.toDateTime, cast);
} }
if (field.getDeExtractType() == 1) { if (field.getDeExtractType() == 1) {
whereName = originName; whereName = originName;
@ -776,7 +795,7 @@ public class CHQueryProvider extends QueryProvider {
whereName = originName; whereName = originName;
} }
if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) { if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) {
whereValue = CHConstants.WHERE_VALUE_NULL; whereValue = CKConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) { } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originName); whereTerm = String.format(whereTerm, originName);
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) {
@ -784,7 +803,7 @@ public class CHQueryProvider extends QueryProvider {
} else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) {
whereValue = "'%" + value + "%'"; whereValue = "'%" + value + "%'";
} else { } else {
whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, value); whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, value);
} }
list.add(SQLObj.builder() list.add(SQLObj.builder()
.whereField(whereName) .whereField(whereName)
@ -814,18 +833,18 @@ public class CHQueryProvider extends QueryProvider {
// 解析origin name中有关联的字段生成sql表达式 // 解析origin name中有关联的字段生成sql表达式
originName = calcFieldRegex(field.getOriginName(), tableObj); originName = calcFieldRegex(field.getOriginName(), tableObj);
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} else { } else {
originName = String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); originName = String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName());
} }
if (field.getDeType() == DeTypeConstants.DE_TIME) { if (field.getDeType() == DeTypeConstants.DE_TIME) {
if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) {
whereName = String.format(CHConstants.toDateTime, originName); whereName = String.format(CKConstants.toDateTime, originName);
} }
if (field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) { if (field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) {
String cast = String.format(CHConstants.toFloat64, originName); String cast = String.format(CKConstants.toFloat64, originName);
whereName = String.format(CHConstants.toDateTime, cast); whereName = String.format(CKConstants.toDateTime, cast);
} }
if (field.getDeExtractType() == 1) { if (field.getDeExtractType() == 1) {
whereName = originName; whereName = originName;
@ -844,12 +863,12 @@ public class CHQueryProvider extends QueryProvider {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0))));
String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1))));
whereValue = String.format(CHConstants.WHERE_BETWEEN, startTime, endTime); whereValue = String.format(CKConstants.WHERE_BETWEEN, startTime, endTime);
} else { } else {
whereValue = String.format(CHConstants.WHERE_BETWEEN, value.get(0), value.get(1)); whereValue = String.format(CKConstants.WHERE_BETWEEN, value.get(0), value.get(1));
} }
} else { } else {
whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, value.get(0)); whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, value.get(0));
} }
list.add(SQLObj.builder() list.add(SQLObj.builder()
.whereField(whereName) .whereField(whereName)
@ -897,13 +916,13 @@ public class CHQueryProvider extends QueryProvider {
if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { if (x.getDeExtractType() == DeTypeConstants.DE_TIME) {
if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) { if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) {
if(x.getType().equalsIgnoreCase("DATE")){ if(x.getType().equalsIgnoreCase("DATE")){
fieldName = String.format(CHConstants.toInt32, String.format(CHConstants.toDateTime, originField)) + "*1000"; fieldName = String.format(CKConstants.toInt32, String.format(CKConstants.toDateTime, originField)) + "*1000";
}else { }else {
fieldName = String.format(CHConstants.toInt32, originField) + "*1000"; fieldName = String.format(CKConstants.toInt32, originField) + "*1000";
} }
} else if (x.getDeType() == DeTypeConstants.DE_TIME) { } else if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
fieldName = String.format(CHConstants.formatDateTime, originField, format); fieldName = String.format(CKConstants.formatDateTime, originField, format);
} else { } else {
fieldName = originField; fieldName = originField;
} }
@ -911,9 +930,9 @@ public class CHQueryProvider extends QueryProvider {
if (x.getDeType() == DeTypeConstants.DE_TIME) { if (x.getDeType() == DeTypeConstants.DE_TIME) {
String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); String format = transDateFormat(x.getDateStyle(), x.getDatePattern());
if (x.getDeExtractType() == DeTypeConstants.DE_STRING) { if (x.getDeExtractType() == DeTypeConstants.DE_STRING) {
fieldName = String.format(CHConstants.formatDateTime, String.format(CHConstants.toDateTime, originField), format); fieldName = String.format(CKConstants.formatDateTime, String.format(CKConstants.toDateTime, originField), format);
} else { } else {
fieldName = String.format(CHConstants.formatDateTime, String.format(CHConstants.toDateTime, String.format(CHConstants.toFloat64, originField)), format); fieldName = String.format(CKConstants.formatDateTime, String.format(CKConstants.toDateTime, String.format(CKConstants.toFloat64, originField)), format);
} }
} else { } else {
fieldName = originField; fieldName = originField;
@ -929,17 +948,17 @@ public class CHQueryProvider extends QueryProvider {
private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) {
String fieldName = ""; String fieldName = "";
if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) {
fieldName = CHConstants.AGG_COUNT; fieldName = CKConstants.AGG_COUNT;
} else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) {
fieldName = String.format(CHConstants.AGG_FIELD, y.getSummary(), originField); fieldName = String.format(CKConstants.AGG_FIELD, y.getSummary(), originField);
} else { } else {
if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) {
String cast = y.getDeType() == 2? String.format(CHConstants.toInt64, originField) : String.format(CHConstants.toFloat64, originField); String cast = y.getDeType() == 2? String.format(CKConstants.toInt64, originField) : String.format(CKConstants.toFloat64, originField);
String agg = String.format(CHConstants.AGG_FIELD, y.getSummary(), cast); String agg = String.format(CKConstants.AGG_FIELD, y.getSummary(), cast);
fieldName = String.format(CHConstants.toDecimal, agg); fieldName = String.format(CKConstants.toDecimal, agg);
} else { } else {
String cast = y.getDeType() == 2 ? String.format(CHConstants.toInt64, originField) : String.format(CHConstants.toFloat64, originField); String cast = y.getDeType() == 2 ? String.format(CKConstants.toInt64, originField) : String.format(CKConstants.toFloat64, originField);
fieldName = String.format(CHConstants.AGG_FIELD, y.getSummary(), cast); fieldName = String.format(CKConstants.AGG_FIELD, y.getSummary(), cast);
} }
} }
return SQLObj.builder() return SQLObj.builder()
@ -956,7 +975,7 @@ public class CHQueryProvider extends QueryProvider {
String whereValue = ""; String whereValue = "";
// 原始类型不是时间，在de中被转成时间的字段做处理 // 原始类型不是时间，在de中被转成时间的字段做处理
if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) {
whereValue = CHConstants.WHERE_VALUE_NULL; whereValue = CKConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originField); whereTerm = String.format(whereTerm, originField);
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) {
@ -964,7 +983,7 @@ public class CHQueryProvider extends QueryProvider {
} else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) {
whereValue = "'%" + f.getValue() + "%'"; whereValue = "'%" + f.getValue() + "%'";
} else { } else {
whereValue = String.format(CHConstants.WHERE_VALUE_VALUE, f.getValue()); whereValue = String.format(CKConstants.WHERE_VALUE_VALUE, f.getValue());
} }
list.add(SQLObj.builder() list.add(SQLObj.builder()
.whereField(fieldAlias) .whereField(fieldAlias)
@ -995,7 +1014,7 @@ public class CHQueryProvider extends QueryProvider {
List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); List<DatasetTableField> calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
for (DatasetTableField ele : calcFields) { for (DatasetTableField ele : calcFields) {
originField = originField.replaceAll("\\[" + ele.getId() + "]", originField = originField.replaceAll("\\[" + ele.getId() + "]",
String.format(CHConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); String.format(CKConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName()));
} }
return originField; return originField;
} }

13
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -469,6 +469,7 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds)); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
System.out.println(datasourceRequest.getQuery());
datasourceRequest.setPage(page); datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
datasourceRequest.setPageSize(pageSize); datasourceRequest.setPageSize(pageSize);
@ -503,8 +504,12 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds)); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
System.out.println(datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
for(int i=0;i< data.size(); i++){
System.out.println(data.get(i)[2]);
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
DEException.throwException(e.getMessage()); DEException.throwException(e.getMessage());
@ -558,8 +563,12 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds)); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
System.out.println(datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
for(int i=0;i< data.size(); i++){
System.out.println(data.get(i)[2]);
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
DEException.throwException(e.getMessage()); DEException.throwException(e.getMessage());
@ -585,8 +594,12 @@ public class DataSetTableService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds)); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
System.out.println(datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
for(int i=0;i< data.size(); i++){
System.out.println(data.get(i)[2]);
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
DEException.throwException(e.getMessage()); DEException.throwException(e.getMessage());

26
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -31,6 +31,9 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.pentaho.di.cluster.SlaveServer; import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.database.DatabaseMeta; import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.plugins.DatabasePluginType;
import org.pentaho.di.core.plugins.PluginInterface;
import org.pentaho.di.core.plugins.PluginRegistry;
import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.util.HttpClientManager; import org.pentaho.di.core.util.HttpClientManager;
import org.pentaho.di.job.Job; import org.pentaho.di.job.Job;
@ -312,6 +315,7 @@ public class ExtractDataService {
msg = true; msg = true;
lastExecStatus = JobStatus.Completed; lastExecStatus = JobStatus.Completed;
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace();
saveErrorLog(datasetTableId, taskId, e); saveErrorLog(datasetTableId, taskId, e);
msg = false; msg = false;
lastExecStatus = JobStatus.Error; lastExecStatus = JobStatus.Error;
@ -456,7 +460,7 @@ public class ExtractDataService {
for (DatasetTableField datasetTableField : datasetTableFields) { for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` "; Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
Integer size = datasetTableField.getSize() * 3; Integer size = datasetTableField.getSize() * 3;
if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { if (datasetTableField.getSize() == 0 || datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) {
size = 65533; size = 65533;
} }
switch (datasetTableField.getDeExtractType()) { switch (datasetTableField.getDeExtractType()) {
@ -468,7 +472,7 @@ public class ExtractDataService {
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`"; Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`";
break; break;
case 2: case 2:
Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(size)) + ",`"; Column_Fields = Column_Fields + "bigint" + ",`";
break; break;
case 3: case 3:
Column_Fields = Column_Fields + "DOUBLE" + ",`"; Column_Fields = Column_Fields + "DOUBLE" + ",`";
@ -616,7 +620,8 @@ public class ExtractDataService {
Thread.sleep(1000); Thread.sleep(1000);
} }
if (!transStatus.getStatusDescription().equals("Finished")) { if (!transStatus.getStatusDescription().equals("Finished")) {
DataEaseException.throwException((transStatus.getLoggingString())); DataEaseException.throwException(transStatus.getLoggingString());
return;
} }
executing = true; executing = true;
@ -637,11 +642,6 @@ public class ExtractDataService {
} }
} }
private boolean isExitFile(String fileName) {
File file = new File(root_path + fileName);
return file.exists();
}
private SlaveServer getSlaveServer() { private SlaveServer getSlaveServer() {
SlaveServer remoteSlaveServer = new SlaveServer(); SlaveServer remoteSlaveServer = new SlaveServer();
remoteSlaveServer.setHostname(carte);// 设置远程IP remoteSlaveServer.setHostname(carte);// 设置远程IP
@ -805,6 +805,16 @@ public class ExtractDataService {
inputStep = inputStep(transMeta, selectSQL); inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle); udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle);
break; break;
case ck:
CHConfigration chConfigration = new Gson().fromJson(datasource.getConfiguration(), CHConfigration.class);
dataMeta = new DatabaseMeta("db", "ORACLE", "Native", chConfigration.getHost().trim(), chConfigration.getDataBase().trim(), chConfigration.getPort().toString(), chConfigration.getUsername(), chConfigration.getPassword());
// dataMeta.addExtraOption("MYSQL", "characterEncoding", "UTF-8");
dataMeta.setDatabaseType("Clickhouse");
transMeta.addDatabase(dataMeta);
selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, DatasourceTypes.ck);
break;
case excel: case excel:
inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields); inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields);
udjcStep = udjc(datasetTableFields, DatasourceTypes.excel); udjcStep = udjc(datasetTableFields, DatasourceTypes.excel);

1
backend/src/main/resources/i18n/messages_en_US.properties
View File

@ -286,3 +286,4 @@ i18n_Stopped=END
i18n_Exec=Running i18n_Exec=Running
i18n_no_trigger=The current setting does not trigger task generation. i18n_no_trigger=The current setting does not trigger task generation.
i18n_dataset_field_delete=Union field deleted,please set again and redo. i18n_dataset_field_delete=Union field deleted,please set again and redo.
i18n_es_limit=Elasticsearch version cannot be less than 6.3

2
backend/src/main/resources/i18n/messages_zh_CN.properties
View File

@ -285,4 +285,4 @@ i18n_Stopped=执行结束
i18n_Exec=运行中 i18n_Exec=运行中
i18n_no_trigger=当前设置没有触发任务生成 i18n_no_trigger=当前设置没有触发任务生成
i18n_dataset_field_delete=该自定义数据集有关联字段被删除，请重新确认关联关系并重做该数据集 i18n_dataset_field_delete=该自定义数据集有关联字段被删除，请重新确认关联关系并重做该数据集
i18n_es_limit=Elasticsearch 版本不能小于6.3

1
backend/src/main/resources/i18n/messages_zh_TW.properties
View File

@ -288,3 +288,4 @@ i18n_Stopped=執行結束
i18n_Exec=運行中 i18n_Exec=運行中
i18n_no_trigger=当前设置没有触发任务生成 當前設置沒有觸發任務生成. i18n_no_trigger=当前设置没有触发任务生成 當前設置沒有觸發任務生成.
i18n_dataset_field_delete=該自定義數據集有關聯字段被刪除，請重新確認關聯關系並重做該數據集 i18n_dataset_field_delete=該自定義數據集有關聯字段被刪除，請重新確認關聯關系並重做該數據集
i18n_es_limit=Elasticsearch 版本不能小於6.3

4
frontend/src/views/dataset/add/AddDB.vue
View File

@ -28,8 +28,8 @@
</el-form-item> </el-form-item>
<el-form-item class="form-item"> <el-form-item class="form-item">
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini"> <el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.direct_connect')" value="0" :disabled="selectedDatasource.computeType==='EXTRACT'"/> <el-option :label="$t('dataset.direct_connect')" value="0" />
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.computeType==='DIRECT'" /> <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck'" />
</el-select> </el-select>
</el-form-item> </el-form-item>

4
frontend/src/views/dataset/add/AddSQL.vue
View File

@ -32,8 +32,8 @@
</el-form-item> </el-form-item>
<el-form-item v-if="!param.tableId" class="form-item"> <el-form-item v-if="!param.tableId" class="form-item">
<el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini"> <el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
<el-option :label="$t('dataset.direct_connect')" value="0" :disabled="selectedDatasource.computeType==='EXTRACT'"/> <el-option :label="$t('dataset.direct_connect')" value="0" />
<el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.computeType==='DIRECT'" /> <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck'" />
</el-select> </el-select>
</el-form-item> </el-form-item>

39
frontend/src/views/system/datasource/form.vue
View File

@ -24,17 +24,6 @@
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item :label="$t('datasource.data_mode')" prop="type">
<el-select v-model="form.computeType" :placeholder="$t('datasource.please_choose_data_type')" class="select-width" :disabled="formType=='modify' || (formType==='add' && params && !!params.computeMode)">
<el-option
v-for="item in compute_mode"
:key="item.type"
:label="item.label"
:value="item.type"
/>
</el-select>
</el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.host')" prop="configuration.host"> <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.host')" prop="configuration.host">
<el-input v-model="form.configuration.host" autocomplete="off" /> <el-input v-model="form.configuration.host" autocomplete="off" />
</el-form-item> </el-form-item>
@ -50,10 +39,10 @@
<el-radio v-model="form.configuration.connectionType" label="serviceName">{{ $t('datasource.oracle_service_name') }}</el-radio> <el-radio v-model="form.configuration.connectionType" label="serviceName">{{ $t('datasource.oracle_service_name') }}</el-radio>
</el-form-item> </el-form-item>
<el-form-item :label="$t('datasource.user_name')" prop="configuration.username"> <el-form-item :label="$t('datasource.user_name')" >
<el-input v-model="form.configuration.username" autocomplete="off" /> <el-input v-model="form.configuration.username" autocomplete="off" />
</el-form-item> </el-form-item>
<el-form-item :label="$t('datasource.password')" prop="configuration.password"> <el-form-item :label="$t('datasource.password')" >
<el-input v-model="form.configuration.password" autocomplete="off" show-password /> <el-input v-model="form.configuration.password" autocomplete="off" show-password />
</el-form-item> </el-form-item>
<el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.port')" prop="configuration.port"> <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.port')" prop="configuration.port">
@ -164,14 +153,9 @@ export default {
{ name: 'sqlServer', label: 'SQL Server', type: 'jdbc' }, { name: 'sqlServer', label: 'SQL Server', type: 'jdbc' },
{ name: 'pg', label: 'PostgreSQL', type: 'jdbc' }, { name: 'pg', label: 'PostgreSQL', type: 'jdbc' },
{ name: 'es', label: 'Elasticsearch', type: 'es' }, { name: 'es', label: 'Elasticsearch', type: 'es' },
{ name: 'ch', label: 'ClickHouse', type: 'jdbc' } { name: 'ck', label: 'ClickHouse', type: 'jdbc' }
], ],
schemas: [], schemas: [],
compute_mode: [
{type: "DIRECT", label: this.$t('datasource.direct')},
{type: "EXTRACT", label: this.$t('datasource.extract')},
{type: "ALL", label: this.$t('datasource.all_compute_mode')}
],
canEdit: false, canEdit: false,
originConfiguration: {} originConfiguration: {}
} }
@ -222,6 +206,14 @@ export default {
this.$refs.dsForm.resetFields() this.$refs.dsForm.resetFields()
}, },
save() { save() {
if(this.form.type !== 'es' && !this.form.configuration.username){
this.$message.error(this.$t('datasource.please_input_user_name'))
return
}
if(this.form.type !== 'es' && !this.form.configuration.username){
this.$message.error(this.$t('datasource.please_input_password'))
return
}
if (!this.form.configuration.schema && (this.form.type === 'oracle' || this.form.type === 'sqlServer')) { if (!this.form.configuration.schema && (this.form.type === 'oracle' || this.form.type === 'sqlServer')) {
this.$message.error(this.$t('datasource.please_choose_schema')) this.$message.error(this.$t('datasource.please_choose_schema'))
return return
@ -312,15 +304,6 @@ export default {
this.form.configuration.dataSourceType = this.allTypes[i].type this.form.configuration.dataSourceType = this.allTypes[i].type
} }
} }
if(this.form.type === 'es'){
this.compute_mode = [{type: "DIRECT", label: this.$t('datasource.direct')}];
}else {
this.compute_mode = [
{type: "DIRECT", label: this.$t('datasource.direct')},
{type: "EXTRACT", label: this.$t('datasource.extract')},
{type: "ALL", label: this.$t('datasource.all_compute_mode')}
];
}
}, },
backToList() { backToList() {
this.$emit('switch-component', { }) this.$emit('switch-component', { })