dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10712 from dataease/pr@dev-v2@refactor_ds

Browse Source 
refactor: 数据源接口抽象
This commit is contained in:
Junjun 2024-07-02 16:38:28 +08:00 committed by GitHub
commit ccc8a2c6fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 356 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
core/core-backend/pom.xml
View File

@ -38,23 +38,6 @@
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>
<!--calcite核心包-->
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>${calcite-core.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
</exclusion>
</exclusions>
<classifier>de</classifier>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/DefaultChartHandler.java
View File

@ -15,9 +15,9 @@ import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.plugin.AbstractChartPlugin;
import io.dataease.extensions.view.util.ChartDataUtil;
import io.dataease.extensions.view.util.FieldUtil;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/YoyChartHandler.java
View File

@ -8,9 +8,9 @@ import io.dataease.engine.trans.ExtWhere2Str;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.FieldUtil;
import io.dataease.utils.JsonUtil;
import org.apache.commons.lang3.StringUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/bar/BarHandler.java
View File

@ -4,9 +4,9 @@ import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/bar/RangeBarHandler.java
View File

@ -2,9 +2,9 @@ package io.dataease.chart.charts.impl.bar;
import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.BooleanUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/line/LineHandler.java
View File

@ -4,9 +4,9 @@ import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Component;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/line/StackAreaHandler.java
View File

@ -5,9 +5,9 @@ import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/map/SymbolicMapHandler.java
View File

@ -9,9 +9,9 @@ import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.ChartDataUtil;
import io.dataease.extensions.view.util.FieldUtil;
import io.dataease.utils.BeanUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/mix/MixHandler.java
View File

@ -5,9 +5,9 @@ import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/numeric/NumericalChartHandler.java
View File

@ -10,9 +10,9 @@ import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.FieldUtil;
import io.dataease.i18n.Translator;
import io.dataease.utils.BeanUtils;

2
core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java
View File

@ -8,9 +8,9 @@ import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.ChartDataUtil;
import io.dataease.extensions.view.util.FieldUtil;
import lombok.Getter;

2
core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java
View File

@ -20,10 +20,10 @@ import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.factory.PluginsChartFactory;
import io.dataease.extensions.view.filter.FilterTreeObj;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.plugin.AbstractChartPlugin;
import io.dataease.extensions.view.util.FieldUtil;
import io.dataease.i18n.Translator;

2
core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java
View File

@ -4,10 +4,10 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.filter.FilterTreeObj;
import io.dataease.api.chart.vo.ViewSelectorVO;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.chart.dao.auto.entity.CoreChartView;
import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper;
import io.dataease.chart.dao.ext.mapper.ExtChartViewMapper;

4
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java
View File

@ -22,20 +22,20 @@ import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.utils.DatasourceUtils;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.*;
import io.dataease.engine.utils.SQLUtils;
import io.dataease.engine.utils.Utils;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.extensions.datasource.factory.ProviderFactory;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.dto.ChartExtRequest;
import io.dataease.extensions.view.dto.ColumnPermissionItem;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.i18n.Translator;
import io.dataease.utils.AuthUtils;
import io.dataease.utils.BeanUtils;

2
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java
View File

@ -3,12 +3,12 @@ package io.dataease.dataset.manage;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.dto.ChartExtRequest;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.api.dataset.union.*;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.api.permissions.auth.dto.BusiPerCheckDTO;
import io.dataease.commons.utils.SqlparserUtils;
import io.dataease.constant.AuthEnum;

2
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetTableFieldManage.java
View File

@ -11,8 +11,8 @@ import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.utils.Utils;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.ColumnPermissionItem;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.i18n.Translator;
import io.dataease.utils.AuthUtils;
import io.dataease.utils.BeanUtils;

4
core/core-backend/src/main/java/io/dataease/dataset/utils/SqlUtils.java
View File

@ -1,11 +1,11 @@
package io.dataease.dataset.utils;
import com.google.common.collect.ImmutableList;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.extensions.view.model.SQLMeta;
import org.apache.calcite.config.Lex;
import org.apache.calcite.sql.*;
import org.apache.calcite.sql.dialect.*;

2
core/core-backend/src/main/java/io/dataease/dataset/utils/TableUtils.java
View File

@ -1,8 +1,8 @@
package io.dataease.dataset.utils;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.utils.Md5Utils;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.commons.lang3.StringUtils;

292
core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java
View File

@ -24,6 +24,7 @@ import org.apache.calcite.adapter.jdbc.JdbcSchema;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.sql.SqlDialect;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
@ -110,26 +111,6 @@ public class CalciteProvider extends Provider {
return tables;
}
private DatasetTableDTO getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
DatasetTableDTO tableDesc = new DatasetTableDTO();
tableDesc.setDatasourceId(datasourceRequest.getDatasource().getId());
tableDesc.setType("db");
tableDesc.setTableName(resultSet.getString(1));
if (resultSet.getMetaData().getColumnCount() > 1) {
tableDesc.setName(resultSet.getString(2));
} else {
tableDesc.setName(resultSet.getString(1));
}
return tableDesc;
}
private List<String> getDriver() {
List<String> drivers = new ArrayList<>();
Map<String, DatasourceConfiguration> beansOfType = CommonBeanFactory.getApplicationContext().getBeansOfType((DatasourceConfiguration.class));
beansOfType.keySet().forEach(key -> drivers.add(beansOfType.get(key).getDriver()));
return drivers;
}
@Override
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(datasourceRequest.getDatasource().getType());
@ -203,6 +184,151 @@ public class CalciteProvider extends Provider {
return map;
}
@Override
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
List<TableField> datasetTableFields = new ArrayList<>();
DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(datasourceSchemaDTO);
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
String table = datasourceRequest.getTable();
if (StringUtils.isEmpty(table)) {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
datasetTableFields.addAll(getField(resultSet, datasourceRequest));
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
} else {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest));
while (resultSet.next()) {
TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet);
boolean repeat = false;
for (TableField ele : datasetTableFields) {
if (StringUtils.equalsIgnoreCase(ele.getOriginName(), tableFieldDesc.getOriginName())) {
repeat = true;
break;
}
}
if (!repeat) {
datasetTableFields.add(tableFieldDesc);
}
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
return datasetTableFields;
}
@Override
public Connection getConnection(DatasourceDTO coreDatasource) throws DEException {
DatasourceConfiguration configuration = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case doris:
case TiDB:
case mariadb:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Mysql.class);
break;
case impala:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Impala.class);
break;
case sqlServer:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Sqlserver.class);
break;
case oracle:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Oracle.class);
break;
case db2:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Db2.class);
break;
case pg:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Pg.class);
break;
case redshift:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Redshift.class);
break;
case ck:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), CK.class);
break;
case h2:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), H2.class);
break;
default:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Mysql.class);
}
Properties props = new Properties();
if (StringUtils.isNotBlank(configuration.getUsername())) {
props.setProperty("user", configuration.getUsername());
}
if (StringUtils.isNotBlank(configuration.getPassword())) {
props.setProperty("password", configuration.getPassword());
}
String driverClassName = configuration.getDriver();
ExtendedJdbcClassLoader jdbcClassLoader = extendedJdbcClassLoader;
Connection conn = null;
try {
Driver driverClass = (Driver) jdbcClassLoader.loadClass(driverClassName).newInstance();
conn = driverClass.connect(configuration.getJdbc(), props);
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return conn;
}
private DatasetTableDTO getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
DatasetTableDTO tableDesc = new DatasetTableDTO();
tableDesc.setDatasourceId(datasourceRequest.getDatasource().getId());
tableDesc.setType("db");
tableDesc.setTableName(resultSet.getString(1));
if (resultSet.getMetaData().getColumnCount() > 1) {
tableDesc.setName(resultSet.getString(2));
} else {
tableDesc.setName(resultSet.getString(1));
}
return tableDesc;
}
private List<String> getDriver() {
List<String> drivers = new ArrayList<>();
Map<String, DatasourceConfiguration> beansOfType = CommonBeanFactory.getApplicationContext().getBeansOfType((DatasourceConfiguration.class));
beansOfType.keySet().forEach(key -> drivers.add(beansOfType.get(key).getDriver()));
return drivers;
}
public Map<String, Object> jdbcFetchResultField(DatasourceRequest datasourceRequest) throws DEException {
DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(value);
@ -549,73 +675,6 @@ public class CalciteProvider extends Provider {
return tableField;
}
@Override
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
List<TableField> datasetTableFields = new ArrayList<>();
DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(datasourceSchemaDTO);
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
String table = datasourceRequest.getTable();
if (StringUtils.isEmpty(table)) {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(datasourceRequest.getQuery());
datasetTableFields.addAll(getField(resultSet, datasourceRequest));
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
} else {
ResultSet resultSet = null;
try (Connection con = getConnection(datasourceRequest.getDatasource());
Statement statement = getStatement(con, 30)) {
if (DatasourceConfiguration.DatasourceType.valueOf(datasourceSchemaDTO.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest));
while (resultSet.next()) {
TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet);
boolean repeat = false;
for (TableField ele : datasetTableFields) {
if (StringUtils.equalsIgnoreCase(ele.getOriginName(), tableFieldDesc.getOriginName())) {
repeat = true;
break;
}
}
if (!repeat) {
datasetTableFields.add(tableFieldDesc);
}
}
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
return datasetTableFields;
}
public Connection initConnection(Map<Long, DatasourceSchemaDTO> dsMap) {
Connection connection = getCalciteConnection();
CalciteConnection calciteConnection = null;
@ -988,65 +1047,6 @@ public class CalciteProvider extends Provider {
}
}
@Override
public Connection getConnection(DatasourceDTO coreDatasource) throws DEException {
DatasourceConfiguration configuration = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case doris:
case TiDB:
case mariadb:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Mysql.class);
break;
case impala:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Impala.class);
break;
case sqlServer:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Sqlserver.class);
break;
case oracle:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Oracle.class);
break;
case db2:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Db2.class);
break;
case pg:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Pg.class);
break;
case redshift:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Redshift.class);
break;
case ck:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), CK.class);
break;
case h2:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), H2.class);
break;
default:
configuration = JsonUtil.parseObject(coreDatasource.getConfiguration(), Mysql.class);
}
Properties props = new Properties();
if (StringUtils.isNotBlank(configuration.getUsername())) {
props.setProperty("user", configuration.getUsername());
}
if (StringUtils.isNotBlank(configuration.getPassword())) {
props.setProperty("password", configuration.getPassword());
}
String driverClassName = configuration.getDriver();
ExtendedJdbcClassLoader jdbcClassLoader = extendedJdbcClassLoader;
Connection conn = null;
try {
Driver driverClass = (Driver) jdbcClassLoader.loadClass(driverClassName).newInstance();
conn = driverClass.connect(configuration.getJdbc(), props);
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return conn;
}
public Statement getStatement(Connection connection, int queryTimeout) {
if (connection == null) {
DEException.throwException("Failed to get connection!");

4
core/core-backend/src/main/java/io/dataease/engine/sql/SQLProvider.java
View File

@ -1,8 +1,8 @@
package io.dataease.engine.sql;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.ChartViewDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.engine.constant.SQLConstants;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java
View File

@ -1,13 +1,13 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.filter.FilterTreeItem;
import io.dataease.extensions.view.filter.FilterTreeObj;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/Dimension2SQLObj.java
View File

@ -1,14 +1,14 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.ChartViewFieldDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/ExtWhere2Str.java
View File

@ -1,12 +1,12 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java
View File

@ -1,15 +1,15 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.func.FunctionConstant;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

4
core/core-backend/src/main/java/io/dataease/engine/trans/Order2SQLObj.java
View File

@ -2,13 +2,13 @@ package io.dataease.engine.trans;
import io.dataease.api.chart.dto.DeSortField;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/Quota2SQLObj.java
View File

@ -1,14 +1,14 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.ChartViewFieldDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

6
core/core-backend/src/main/java/io/dataease/engine/trans/Table2SQLObj.java
View File

@ -1,9 +1,9 @@
package io.dataease.engine.trans;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
/**
* @Author Junjun

6
core/core-backend/src/main/java/io/dataease/engine/trans/WhereTree2Str.java
View File

@ -1,15 +1,15 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeItem;
import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeObj;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.engine.utils.Utils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;

4
core/core-backend/src/main/java/io/dataease/engine/utils/Utils.java
View File

@ -2,12 +2,12 @@ package io.dataease.engine.utils;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.i18n.Translator;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

2
sdk/api/api-base/src/main/java/io/dataease/api/dataset/union/UnionParamDTO.java
View File

@ -1,7 +1,7 @@
package io.dataease.api.dataset.union;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.extensions.datasource.model.SQLObj;
import lombok.Data;
import java.io.Serializable;

20
sdk/extensions/extensions-datasource/pom.xml
View File

@ -11,6 +11,26 @@
<artifactId>extensions-datasource</artifactId>
<dependencies>
<!--calcite核心包-->
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>${calcite-core.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
</exclusion>
</exclusions>
<classifier>de</classifier>
</dependency>
</dependencies>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>

32
core/core-backend/src/main/java/io/dataease/engine/constant/SqlPlaceholderConstants.java → sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/constant/SqlPlaceholderConstants.java
View File

@ -1,16 +1,16 @@
package io.dataease.engine.constant;
/**
* @Author Junjun
*/
public class SqlPlaceholderConstants {
public static final String TABLE_PLACEHOLDER = "SELECT * FROM DE_PLACEHOLDER_TABLE_0";
public static final String KEYWORD_PREFIX_REGEX = "[`'\"\\[]?";
public static final String KEYWORD_SUFFIX_REGEX = "[`'\"\\]]?";
public static final String TABLE_PLACEHOLDER_REGEX = "SELECT \\* FROM " + KEYWORD_PREFIX_REGEX + "DE_PLACEHOLDER_TABLE_0" + KEYWORD_SUFFIX_REGEX;
public static final String CALC_FIELD_PLACEHOLDER = "DE_CALC_FIELD_PLACEHOLDER_%s";
}
package io.dataease.extensions.datasource.constant;
/**
* @Author Junjun
*/
public class SqlPlaceholderConstants {
public static final String TABLE_PLACEHOLDER = "SELECT * FROM DE_PLACEHOLDER_TABLE_0";
public static final String KEYWORD_PREFIX_REGEX = "[`'\"\\[]?";
public static final String KEYWORD_SUFFIX_REGEX = "[`'\"\\]]?";
public static final String TABLE_PLACEHOLDER_REGEX = "SELECT \\* FROM " + KEYWORD_PREFIX_REGEX + "DE_PLACEHOLDER_TABLE_0" + KEYWORD_SUFFIX_REGEX;
public static final String CALC_FIELD_PLACEHOLDER = "DE_CALC_FIELD_PLACEHOLDER_%s";
}

2
sdk/extensions/extensions-view/src/main/java/io/dataease/extensions/view/model/SQLMeta.java → sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/model/SQLMeta.java
View File

@ -1,4 +1,4 @@
package io.dataease.extensions.view.model;
package io.dataease.extensions.datasource.model;
import lombok.Data;

2
sdk/extensions/extensions-view/src/main/java/io/dataease/extensions/view/model/SQLObj.java → sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/model/SQLObj.java
View File

@ -1,4 +1,4 @@
package io.dataease.extensions.view.model;
package io.dataease.extensions.datasource.model;
import lombok.AllArgsConstructor;
import lombok.Builder;

124
sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java
View File

@ -1,19 +1,31 @@
package io.dataease.extensions.datasource.provider;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.extensions.datasource.constant.SqlPlaceholderConstants;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import org.apache.calcite.config.Lex;
import org.apache.calcite.sql.SqlDialect;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.dialect.*;
import org.apache.calcite.sql.parser.SqlParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
/**
* @Author Junjun
*/
public abstract class Provider {
public static Logger logger = LoggerFactory.getLogger(Provider.class);
public abstract List<String> getSchema(DatasourceRequest datasourceRequest);
public abstract List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest);
@ -25,4 +37,108 @@ public abstract class Provider {
public abstract Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException;
public abstract List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException;
public String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap) {
logger.info("calcite sql: " + sql);
if (crossDs) {
return sql;
}
String s = transSqlDialect(sql, dsMap);
String tableDialect = sqlMeta.getTableDialect();
s = replaceTablePlaceHolder(s, tableDialect);
return replaceCalcFieldPlaceHolder(s, sqlMeta);
}
public String transSqlDialect(String sql, Map<Long, DatasourceSchemaDTO> dsMap) throws DEException {
try {
DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue();
SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA));
SqlNode sqlNode = parser.parseStmt();
return sqlNode.toSqlString(getDialect(value)).toString();
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return null;
}
public String replaceTablePlaceHolder(String s, String placeholder) {
s = s.replaceAll("\r\n", " ")
.replaceAll("\n", " ")
.replaceAll(SqlPlaceholderConstants.TABLE_PLACEHOLDER_REGEX, Matcher.quoteReplacement(placeholder))
.replaceAll("ASYMMETRIC", "")
.replaceAll("SYMMETRIC", "");
return s;
}
public String replaceCalcFieldPlaceHolder(String s, SQLMeta sqlMeta) {
Map<String, String> fieldsDialect = new HashMap<>();
if (sqlMeta.getXFieldsDialect() != null && !sqlMeta.getXFieldsDialect().isEmpty()) {
fieldsDialect.putAll(sqlMeta.getXFieldsDialect());
}
if (sqlMeta.getYFieldsDialect() != null && !sqlMeta.getYFieldsDialect().isEmpty()) {
fieldsDialect.putAll(sqlMeta.getYFieldsDialect());
}
if (sqlMeta.getCustomWheresDialect() != null && !sqlMeta.getCustomWheresDialect().isEmpty()) {
fieldsDialect.putAll(sqlMeta.getCustomWheresDialect());
}
if (sqlMeta.getExtWheresDialect() != null && !sqlMeta.getExtWheresDialect().isEmpty()) {
fieldsDialect.putAll(sqlMeta.getExtWheresDialect());
}
if (sqlMeta.getWhereTreesDialect() != null && !sqlMeta.getWhereTreesDialect().isEmpty()) {
fieldsDialect.putAll(sqlMeta.getWhereTreesDialect());
}
if (!fieldsDialect.isEmpty()) {
for (Map.Entry<String, String> ele : fieldsDialect.entrySet()) {
s = s.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + ele.getKey() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, Matcher.quoteReplacement(ele.getValue()));
}
}
return s;
}
public SqlDialect getDialect(DatasourceSchemaDTO coreDatasource) {
SqlDialect sqlDialect = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {
case mysql:
case mongo:
case StarRocks:
case TiDB:
case mariadb:
sqlDialect = MysqlSqlDialect.DEFAULT;
break;
case doris:
sqlDialect = DorisSqlDialect.DEFAULT;
break;
case impala:
sqlDialect = ImpalaSqlDialect.DEFAULT;
break;
case sqlServer:
sqlDialect = MssqlSqlDialect.DEFAULT;
break;
case oracle:
sqlDialect = OracleSqlDialect.DEFAULT;
break;
case db2:
sqlDialect = Db2SqlDialect.DEFAULT;
break;
case pg:
sqlDialect = PostgresqlSqlDialect.DEFAULT;
break;
case redshift:
sqlDialect = RedshiftSqlDialect.DEFAULT;
break;
case ck:
sqlDialect = ClickHouseSqlDialect.DEFAULT;
break;
case h2:
sqlDialect = H2SqlDialect.DEFAULT;
break;
default:
sqlDialect = MysqlSqlDialect.DEFAULT;
}
return sqlDialect;
}
}

6
sdk/extensions/extensions-view/src/main/java/io/dataease/extensions/view/plugin/AbstractChartPlugin.java
View File

@ -1,12 +1,13 @@
package io.dataease.extensions.view.plugin;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import java.util.List;
import java.util.Map;
public abstract class AbstractChartPlugin {
/**
@ -49,9 +50,12 @@ public abstract class AbstractChartPlugin {
*
* @param view 原视图对象
* @param calcResult 计算结果
* @param view 原视图对象
* @param calcResult 计算结果
* @param formatResult 轴字段处理结果
* @param filterResult 过滤条件处理结果
* @return 返回前端的视图，建议数据放在 data 中，过滤条件放在 filter 中，其他字段可自行添加，可参考主工程。
*/
public abstract ChartViewDTO buildChart(ChartViewDTO view, ChartCalcDataResult calcResult, AxisFormatResult formatResult, CustomFilterResult filterResult);
}