dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: doris数据源过滤器选项排序

Browse Source
This commit is contained in:
fit2cloud-chenyw 2023-11-15 10:32:02 +08:00
parent e308e1c249
commit 7c8a3e231a
3 changed files with 38 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/backend/src/main/java/io/dataease/provider/engine/doris/DorisQueryProvider.java
View File

@ -102,7 +102,6 @@ public class DorisQueryProvider extends QueryProvider {
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
int originSize = fields.size();
List<String> fieldList = fields.stream().map(DatasetTableField::getId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(sortFields)) {
sortFields.forEach(item -> {
@ -206,17 +205,8 @@ public class DorisQueryProvider extends QueryProvider {
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
st.add("table", tableSQL);
List<SQLObj> outFieldList = new ArrayList<>();
for (int i = 0; i < originSize; i++) {
SQLObj fieldObj = xFields.get(i);
String outFieldAlias = tableSQL.getTableAlias() + "." + fieldObj.getFieldAlias();
outFieldList.add(SQLObj.builder().fieldName(outFieldAlias).fieldAlias(fieldObj.getFieldAlias()).build());
}
st.add("groups", outFieldList);
if (CollectionUtils.isNotEmpty(xOrders)) {
st.add("orders", xOrders);
return st.render() + " LIMIT 0, 10000000";
}
return st.render();
}

10
core/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java
View File

@ -161,7 +161,6 @@ public class MysqlQueryProvider extends QueryProvider {
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 0))
.build();
List<SQLObj> xFields = new ArrayList<>();
int originSize = fields.size();
List<String> fieldList = fields.stream().map(DatasetTableField::getId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(sortFields)) {
sortFields.forEach(item -> {
@ -259,17 +258,8 @@ public class MysqlQueryProvider extends QueryProvider {
.tableAlias(String.format(TABLE_ALIAS_PREFIX, 1))
.build();
st.add("table", tableSQL);
List<SQLObj> outFieldList = new ArrayList<>();
for (int i = 0; i < originSize; i++) {
SQLObj fieldObj = xFields.get(i);
String outFieldAlias = tableSQL.getTableAlias() + "." + fieldObj.getFieldAlias();
outFieldList.add(SQLObj.builder().fieldName(outFieldAlias).fieldAlias(fieldObj.getFieldAlias()).build());
}
st.add("groups", outFieldList);
if (CollectionUtils.isNotEmpty(xOrders)) {
st.add("orders", xOrders);
return st.render() + " LIMIT 0, 10000000";
}
return st.render();
}

43
core/backend/src/main/java/io/dataease/service/dataset/impl/direct/DirectFieldService.java
View File

@ -1,6 +1,8 @@
package io.dataease.service.dataset.impl.direct;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.ReflectUtil;
import com.google.gson.Gson;
import io.dataease.commons.exception.DEException;
import io.dataease.commons.model.BaseTreeNode;
@ -11,6 +13,7 @@ import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.DeSortDTO;
import io.dataease.i18n.Translator;
import io.dataease.plugins.common.base.domain.ChartViewWithBLOBs;
import io.dataease.plugins.common.base.domain.DatasetTable;
import io.dataease.plugins.common.base.domain.DatasetTableField;
import io.dataease.plugins.common.base.domain.Datasource;
@ -87,15 +90,14 @@ public class DirectFieldService implements DataSetFieldService {
}
String id = sortDTO.getId();
String sortStr = StringUtils.equalsIgnoreCase("chineseDesc", id) ? "desc" : "asc";
List<Object> result = CollectionUtil.sort(list, (v1, v2) -> {
return CollectionUtil.sort(list, (v1, v2) -> {
Collator instance = Collator.getInstance(Locale.CHINESE);
if (StringUtils.equals("desc", sortStr)) {
return instance.compare(v2, v1);
}
return instance.compare(v1, v2);
});
return result;
}
private String formatTableByKeyword(String keyword, String originTable, List<DatasetTableField> fields) {
@ -155,10 +157,12 @@ public class DirectFieldService implements DataSetFieldService {
}
}
DatasourceRequest datasourceRequest = new DatasourceRequest();
Provider datasourceProvider = null;
String createSQL = null;
ChartViewWithBLOBs view = new ChartViewWithBLOBs();
view.setResultCount(1000);
view.setResultMode("custom");
if (datasetTable.getMode() == 0) {// 直连
if (StringUtils.isEmpty(datasetTable.getDataSourceId())) return null;
Datasource ds = datasourceService.get(datasetTable.getDataSourceId());
@ -200,7 +204,14 @@ public class DirectFieldService implements DataSetFieldService {
}
createSQL = qp.createQuerySQLAsTmp(sql, permissionFields, !needSort, customFilter, rowPermissionsTree, deSortFields);
}
if (StringUtils.equalsAny(ds.getType(), "ds_doris", "mysql")) {
Object[] args = new Object[]{createSQL, view};
createSQL = ReflectUtil.invoke(qp, "sqlLimit", args);
datasourceRequest.setQuery(createSQL);
} else {
datasourceRequest.setQuery(qp.createSQLPreview(createSQL, null));
}
} else if (datasetTable.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = engineService.getDeEngine();
@ -212,8 +223,26 @@ public class DirectFieldService implements DataSetFieldService {
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
String formatSql = formatTableByKeyword(keyword, tableName, permissionFields);
createSQL = qp.createQuerySQL(formatSql, permissionFields, !needSort, null, customFilter, rowPermissionsTree, deSortFields);
datasourceRequest.setQuery(qp.createSQLPreview(createSQL, null));
Object[] args = new Object[]{createSQL, view};
createSQL = ReflectUtil.invoke(qp, "sqlLimit", args);
datasourceRequest.setQuery(createSQL);
}
int originSize = permissionFields.size();
boolean existExtSortField = false;
if (CollectionUtils.isNotEmpty(deSortFields)) {
List<String> fieldList = permissionFields.stream().map(DatasetTableField::getId).collect(Collectors.toList());
for (DeSortField sField : deSortFields) {
int indexOf = fieldList.indexOf(sField.getId());
if (indexOf == -1) {
existExtSortField = true;
permissionFields.add(sField);
} else {
permissionFields.set(indexOf, sField);
}
}
}
LogUtil.info(datasourceRequest.getQuery());
datasourceRequest.setPermissionFields(permissionFields);
assert datasourceProvider != null;
@ -222,7 +251,9 @@ public class DirectFieldService implements DataSetFieldService {
return rows.stream().map(row -> row[0]).distinct().collect(Collectors.toList());
}
Set<String> pkSet = new HashSet<>();
if (CollectionUtils.isNotEmpty(rows) && existExtSortField && originSize > 0) {
rows = rows.stream().map(row -> ArrayUtil.sub(row, 0, originSize)).collect(Collectors.toList());
}
List<BaseTreeNode> treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList());
List tree = TreeUtils.mergeDuplicateTree(treeNodes, TreeUtils.DEFAULT_ROOT);
return tree;