From a045f51515808116a07dd982cfce19a844e807e5 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 22 Mar 2022 19:33:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=EF=BC=88doris=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dto/datasource/DorisConfiguration.java | 2 +- .../provider/datasource/JdbcProvider.java | 1 - .../query/mongodb/MongoQueryProvider.java | 1 - .../service/engine/EngineService.java | 82 +++-- frontend/src/lang/zh.js | 2 + .../system/SysParam/ClusterModeSetting.vue | 291 ++++++++++++++++++ 6 files changed, 356 insertions(+), 23 deletions(-) create mode 100644 frontend/src/views/system/SysParam/ClusterModeSetting.vue diff --git a/backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java b/backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java index f3edc95760..757c58b41a 100644 --- a/backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java +++ b/backend/src/main/java/io/dataease/dto/datasource/DorisConfiguration.java @@ -7,7 +7,7 @@ import lombok.Setter; @Setter public class DorisConfiguration extends MysqlConfiguration { - private Integer httpPort; + private Integer httpPort = 8030; private Integer replicationNum = 1; private Integer bucketNum = 10; diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index af64cfca60..4786df662f 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -487,7 +487,6 @@ public class JdbcProvider extends DatasourceProvider { break; case impala: ImpalaConfiguration impalaConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), ImpalaConfiguration.class); - System.out.println(new Gson().toJson(impalaConfiguration)); username = impalaConfiguration.getUsername(); password = impalaConfiguration.getPassword(); driver = impalaConfiguration.getDriver(); diff --git a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java index eec743a7eb..d7569fc37d 100644 --- a/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mongodb/MongoQueryProvider.java @@ -43,7 +43,6 @@ public class MongoQueryProvider extends QueryProvider { @Override public Integer transFieldType(String field) { - System.out.println(field); field = field.toUpperCase(); switch (field) { case "CHAR": diff --git a/backend/src/main/java/io/dataease/service/engine/EngineService.java b/backend/src/main/java/io/dataease/service/engine/EngineService.java index db635e721b..5eb454b702 100644 --- a/backend/src/main/java/io/dataease/service/engine/EngineService.java +++ b/backend/src/main/java/io/dataease/service/engine/EngineService.java @@ -1,14 +1,20 @@ package io.dataease.service.engine; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.google.gson.Gson; +import com.google.gson.JsonArray; import io.dataease.base.domain.Datasource; import io.dataease.base.domain.DeEngine; import io.dataease.base.domain.DeEngineExample; import io.dataease.base.mapper.DeEngineMapper; import io.dataease.commons.utils.BeanUtils; +import io.dataease.commons.utils.HttpClientConfig; +import io.dataease.commons.utils.HttpClientUtil; import io.dataease.controller.ResultHolder; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.dto.DatasourceDTO; +import io.dataease.dto.datasource.DorisConfiguration; import io.dataease.listener.util.CacheUtils; import io.dataease.provider.ProviderFactory; import io.dataease.provider.datasource.DatasourceProvider; @@ -20,7 +26,9 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; +import java.util.Base64; import java.util.List; +import java.util.Optional; import java.util.UUID; @Service @@ -33,38 +41,38 @@ public class EngineService { @Resource private DatasourceService datasource; - public Boolean isLocalMode(){ + public Boolean isLocalMode() { return env.getProperty("engine_mode", "local").equalsIgnoreCase("local"); } - public Boolean isSimpleMode(){ + public Boolean isSimpleMode() { return env.getProperty("engine_mode", "local").equalsIgnoreCase("simple"); } - public Boolean isClusterMode(){ + public Boolean isClusterMode() { return env.getProperty("engine_mode", "local").equalsIgnoreCase("cluster"); } - public String mode(){ + public String mode() { return env.getProperty("engine_mode", "local"); } - public DeEngine info(){ + public DeEngine info() { DeEngineExample deEngineExample = new DeEngineExample(); - if(isClusterMode()){ + if (isClusterMode()) { deEngineExample.createCriteria().andTypeEqualTo("engine_doris"); - }else { + } else { deEngineExample.createCriteria().andTypeEqualTo("engine_mysql"); } List deEngines = deEngineMapper.selectByExampleWithBLOBs(deEngineExample); - if(CollectionUtils.isEmpty(deEngines)){ + if (CollectionUtils.isEmpty(deEngines)) { return new DeEngine(); } return deEngines.get(0); } public ResultHolder validate(DatasourceDTO datasource) throws Exception { - if(StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())){ + if (StringUtils.isEmpty(datasource.getType()) || StringUtils.isEmpty(datasource.getConfiguration())) { throw new Exception("未完整设置数据引擎"); } try { @@ -72,17 +80,49 @@ public class EngineService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); datasourceProvider.checkStatus(datasourceRequest); - return ResultHolder.success(datasource); - }catch (Exception e){ + } catch (Exception e) { return ResultHolder.error("Engine is invalid: " + e.getMessage()); } + + if (datasource.getType().equalsIgnoreCase("engine_doris")) { + DorisConfiguration dorisConfiguration = new Gson().fromJson(datasource.getConfiguration(), DorisConfiguration.class); + HttpClientConfig httpClientConfig = new HttpClientConfig(); + String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((dorisConfiguration.getUsername() + + ":" + dorisConfiguration.getPassword()).getBytes()); + httpClientConfig.addHeader("Authorization", authValue); + String response; + try { + response = HttpClientUtil.get("http://" + dorisConfiguration.getHost() + ":" + dorisConfiguration.getHttpPort() + "/api/backends", httpClientConfig); + }catch (Exception e){ + return ResultHolder.error("Engine is invalid: " + e.getMessage()); + } + + JSONArray backends = Optional.ofNullable(JSONObject.parseObject(response).getJSONObject("data")).orElse(new JSONObject()).getJSONArray("backends"); + if(CollectionUtils.isEmpty(backends)){ + return ResultHolder.error("Engine is invalid: no backends found."); + } + + Integer alives = 0; + for (int i = 0; i < backends.size(); i++) { + JSONObject kv = backends.getJSONObject(i); + if (kv.getBoolean("is_alive")) { + alives ++; + } + } + + if(alives < dorisConfiguration.getReplicationNum()){ + return ResultHolder.error("Engine params is invalid: 副本数量不能大于节点数量."); + } + } + + return ResultHolder.success(datasource); } public ResultHolder save(DeEngine engine) throws Exception { - if(StringUtils.isEmpty(engine.getId())){ + if (StringUtils.isEmpty(engine.getId())) { engine.setId(UUID.randomUUID().toString()); deEngineMapper.insert(engine); - }else { + } else { deEngineMapper.updateByPrimaryKeyWithBLOBs(engine); datasource.handleConnectionPool(getDeEngine(), "delete"); } @@ -91,17 +131,19 @@ public class EngineService { return ResultHolder.success(engine); } - private void setDs(DeEngine engine){ - CacheUtils.put("ENGINE", "engine", engine, null, null); + private void setDs(DeEngine engine) { + Datasource datasource = new Datasource(); + BeanUtils.copyBean(datasource, engine); + CacheUtils.put("ENGINE", "engine", datasource, null, null); } - public Datasource getDeEngine() throws Exception{ + public Datasource getDeEngine() throws Exception { Object catcheEngine = CacheUtils.get("ENGINE", "engine"); - if(catcheEngine != null){ + if (catcheEngine != null) { return (Datasource) catcheEngine; } - if(isLocalMode()){ + if (isLocalMode()) { JSONObject jsonObject = new JSONObject(); jsonObject.put("dataSourceType", "jdbc"); jsonObject.put("dataBase", env.getProperty("doris.db", "doris")); @@ -118,9 +160,9 @@ public class EngineService { engine.setType("engine_doris"); engine.setConfiguration(jsonObject.toJSONString()); setDs(engine); - }else { + } else { List deEngines = deEngineMapper.selectByExampleWithBLOBs(new DeEngineExample()); - if(CollectionUtils.isEmpty(deEngines)){ + if (CollectionUtils.isEmpty(deEngines)) { throw new Exception("未设置数据引擎"); } setDs(deEngines.get(0)); diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 532f27b309..a1089daca1 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1303,6 +1303,8 @@ export default { host: '主机名/IP地址', doris_host: 'Doris 地址', port: '端口', + query_port: 'Query Port', + http_port: 'Http Port', datasource_url: '地址', please_input_datasource_url: '请输入 Elasticsearch 地址，如: http://es_host:es_port', please_input_data_base: '请输入数据库名称', diff --git a/frontend/src/views/system/SysParam/ClusterModeSetting.vue b/frontend/src/views/system/SysParam/ClusterModeSetting.vue new file mode 100644 index 0000000000..f0a5f9b2dc --- /dev/null +++ b/frontend/src/views/system/SysParam/ClusterModeSetting.vue @@ -0,0 +1,291 @@ + + + + +