dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(数据源): 支持ES数据源

Browse Source
This commit is contained in:
junjun 2024-10-12 14:16:58 +08:00
parent 43815fa9a5
commit f60a937881
13 changed files with 172 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java
View File

@ -565,14 +565,20 @@ public class DatasetDataManage {
provider = ProviderFactory.getProvider(dsList.getFirst());
}
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String querySQL;
if (multFieldValuesRequest.getResultMode() == 0) {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, true, 0, 1000);
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"), 0, 1000);
} else {
querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, true);
querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"));
}
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
logger.debug("calcite data enum sql: " + querySQL);
@ -812,15 +818,21 @@ public class DatasetDataManage {
provider = ProviderFactory.getProvider(dsList.getFirst());
}
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
ExtWhere2Str.extWhere2sqlOjb(sqlMeta, extFilterList, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String querySQL;
if (request.getResultMode() == 0) {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1, 0, 1000);
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1 && !StringUtils.equalsIgnoreCase(dsType, "es"), 0, 1000);
} else {
querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1);
querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1 && !StringUtils.equalsIgnoreCase(dsType, "es"));
}
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
logger.debug("calcite data enum sql: " + querySQL);

7
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java
View File

@ -152,17 +152,22 @@ public class DatasetSQLManage {
f.setDatasetTableId(datasetTable.getId());
String prefix = "";
String suffix = "";
DsTypeDTO datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
if (Objects.equals(f.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
if (isCross) {
prefix = "`";
suffix = "`";
} else {
DsTypeDTO datasourceType = getDatasourceType(dsMap, datasetTable.getDatasourceId());
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
}
}
if (StringUtils.equalsIgnoreCase(datasourceType.getType(), "es")) {
return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix;
} else {
return table.getTableAlias() + "." + prefix + f.getOriginName() + suffix + " AS " + prefix + alias + suffix;
}
})
.toArray(String[]::new);
checkedInfo.put(table.getTableAlias(), array);

12
core/core-backend/src/main/java/io/dataease/datasource/provider/EsProvider.java
View File

@ -83,8 +83,8 @@ public class EsProvider extends Provider {
Map<String, Object> result = new HashMap<>();
try {
String response = execQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
result.put("dataList", fetchResultData(response));
result.put("fieldList", fetchResultField4Sql(response));
result.put("data", fetchResultData(response));
result.put("fields", fetchResultField4Sql(response));
} catch (Exception e) {
e.printStackTrace();
DEException.throwException(e);
@ -96,7 +96,13 @@ public class EsProvider extends Provider {
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) {
List<TableField> tableFields = new ArrayList<>();
try {
String response = execQuery(datasourceRequest, "select * from " + datasourceRequest.getTable() + " limit 0", "?format=json");
String sql;
if (datasourceRequest.getTable() != null) {
sql = "select * from " + datasourceRequest.getTable() + " limit 0";
} else {
sql = datasourceRequest.getQuery();
}
String response = execQuery(datasourceRequest, sql, "?format=json");
tableFields = fetchResultField4Sql(response);
} catch (Exception e) {
DEException.throwException(e);

17
core/core-backend/src/main/java/io/dataease/engine/trans/CustomWhere2Str.java
View File

@ -78,6 +78,13 @@ public class CustomWhere2Str {
if (ObjectUtils.isEmpty(field)) {
return null;
}
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
Map<String, String> paramMap = Utils.mergeParam(fieldParam, chartParam);
String whereName = "";
String originName;
@ -91,10 +98,18 @@ public class CustomWhere2Str {
originName = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
// 此处获取标准格式的日期

17
core/core-backend/src/main/java/io/dataease/engine/trans/Dimension2SQLObj.java
View File

@ -33,6 +33,13 @@ public class Dimension2SQLObj {
List<SQLObj> xFields = new ArrayList<>();
List<SQLObj> xOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO x = fields.get(i);
@ -47,10 +54,18 @@ public class Dimension2SQLObj {
originField = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias, isCross));

17
core/core-backend/src/main/java/io/dataease/engine/trans/ExtWhere2Str.java
View File

@ -29,6 +29,13 @@ public class ExtWhere2Str {
Map<String, String> paramMap = Utils.mergeParam(fieldParam, chartParam);
List<SQLObj> list = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (ObjectUtils.isNotEmpty(fields)) {
for (ChartExtFilterDTO request : fields) {
List<String> value = request.getValue();
@ -58,10 +65,18 @@ public class ExtWhere2Str {
originName = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {

17
core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java
View File

@ -30,6 +30,13 @@ public class Field2SQLObj {
Map<String, String> paramMap = Utils.mergeParam(fieldParam, chartParam);
List<SQLObj> xFields = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (ObjectUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
DatasetTableFieldDTO x = fields.get(i);
@ -51,10 +58,18 @@ public class Field2SQLObj {
}
}
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
// 处理横轴字段
xFields.add(getXFields(x, originField, fieldAlias, isCross));

17
core/core-backend/src/main/java/io/dataease/engine/trans/Order2SQLObj.java
View File

@ -44,14 +44,29 @@ public class Order2SQLObj {
private static SQLObj buildSortField(DeSortField f, SQLObj tableObj, int i, List<DatasetTableFieldDTO> originFields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap, List<CalParam> fieldParam, List<CalParam> chartParam, PluginManageApi pluginManage) {
Map<String, String> paramMap = Utils.mergeParam(fieldParam, chartParam);
String originField;
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (ObjectUtils.isNotEmpty(f.getExtField()) && Objects.equals(f.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
originField = Utils.calcFieldRegex(f.getOriginName(), tableObj, originFields, isCross, dsMap, paramMap, pluginManage);
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && Objects.equals(f.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);
String fieldName = "";
// 处理横轴字段

17
core/core-backend/src/main/java/io/dataease/engine/trans/Quota2SQLObj.java
View File

@ -33,6 +33,13 @@ public class Quota2SQLObj {
List<String> yWheres = new ArrayList<>();
List<SQLObj> yOrders = new ArrayList<>();
Map<String, String> fieldsDialect = new HashMap<>();
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (!CollectionUtils.isEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
ChartViewFieldDTO y = fields.get(i);
@ -47,10 +54,18 @@ public class Quota2SQLObj {
originField = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_COPY)) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);
// 处理纵轴字段
SQLObj ySQLObj = getYFields(y, originField, fieldAlias);

17
core/core-backend/src/main/java/io/dataease/engine/trans/WhereTree2Str.java
View File

@ -97,6 +97,13 @@ public class WhereTree2Str {
Map<String, String> paramMap = Utils.mergeParam(fieldParam, chartParam);
String whereName = "";
String originName;
String dsType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
dsType = next.getValue().getType();
}
if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_CALC)) {
// 解析origin name中有关联的字段生成sql表达式
String calcFieldExp = Utils.calcFieldRegex(field.getOriginName(), tableObj, originFields, isCross, dsMap, paramMap, pluginManage);
@ -107,10 +114,18 @@ public class WhereTree2Str {
originName = calcFieldExp;
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_COPY)) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
} else {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
}
}
if (field.getDeType() == 1) {
if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) {
whereName = String.format(SQLConstants.DE_STR_TO_DATE, originName, StringUtils.isNotEmpty(field.getDateFormat()) ? field.getDateFormat() : SQLConstants.DEFAULT_DATE_FORMAT);

2
de-xpack

@ -1 +1 @@
Subproject commit 7612ed2de6f9c2412b60ce1497b7b02a0f0a8f1b
Subproject commit 08cc93e3fb3dc9579759a738dad6a1726577f026

2
pom.xml
View File

@ -30,7 +30,7 @@
<mybatis-plus.version>3.5.6</mybatis-plus.version>
<h2.version>2.2.220</h2.version>
<knife4j.version>4.4.0</knife4j.version>
<calcite-core.version>1.35.15</calcite-core.version>
<calcite-core.version>1.35.16</calcite-core.version>
<commons-dbcp2.version>2.6.0</commons-dbcp2.version>
<antlr.version>3.5.2</antlr.version>
<java-jwt.version>3.12.1</java-jwt.version>

19
sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java
View File

@ -212,7 +212,7 @@ public abstract class Provider {
sqlDialect = new MssqlSqlDialect(MssqlSqlDialect.DEFAULT_CONTEXT, coreDatasource.getDsVersion());
break;
case oracle:
sqlDialect = OracleSqlDialect.DEFAULT;
sqlDialect = new OracleSqlDialect(OracleSqlDialect.DEFAULT_CONTEXT, coreDatasource.getDsVersion());
break;
case db2:
sqlDialect = Db2SqlDialect.DEFAULT;
@ -229,6 +229,9 @@ public abstract class Provider {
case h2:
sqlDialect = H2SqlDialect.DEFAULT;
break;
case es:
sqlDialect = EsSqlDialect.DEFAULT;
break;
default:
sqlDialect = MysqlSqlDialect.DEFAULT;
}
@ -259,26 +262,26 @@ public abstract class Provider {
}
}
public void startSshSession(DatasourceConfiguration configuration, ConnectionObj connectionObj, Long datacourseId) throws Exception {
public void startSshSession(DatasourceConfiguration configuration, ConnectionObj connectionObj, Long datasourceId) throws Exception {
if (configuration.isUseSSH()) {
if (datacourseId == null) {
if (datasourceId == null) {
configuration.setLPort(getLport(null));
connectionObj.setLPort(configuration.getLPort());
connectionObj.setConfiguration(configuration);
Session session = initSession(configuration);
connectionObj.setSession(session);
} else {
Integer lport = Provider.getLPorts().get(datacourseId);
Integer lport = Provider.getLPorts().get(datasourceId);
configuration.setLPort(lport);
if (lport != null) {
if (Provider.getSessions().get(datacourseId) == null || !Provider.getSessions().get(datacourseId).isConnected()) {
if (Provider.getSessions().get(datasourceId) == null || !Provider.getSessions().get(datasourceId).isConnected()) {
Session session = initSession(configuration);
Provider.getSessions().put(datacourseId, session);
Provider.getSessions().put(datasourceId, session);
}
} else {
configuration.setLPort(getLport(datacourseId));
configuration.setLPort(getLport(datasourceId));
Session session = initSession(configuration);
Provider.getSessions().put(datacourseId, session);
Provider.getSessions().put(datasourceId, session);
}
configuration.setLPort(lport);
}