From 6546696f98f3bc3dc657a0a535b33f186e37ef5e Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 21 Jun 2024 18:31:04 +0800 Subject: [PATCH 01/23] =?UTF-8?q?fix:=20impala=20jdbc=E5=B1=95=E7=A4=BA?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/datasource/provider/CalciteProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java index 947ec1c0cd..e0af11c435 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java @@ -393,7 +393,7 @@ public class CalciteProvider { configuration = JsonUtil.parseObject(datasourceDTO.getConfiguration(), Impala.class); if (StringUtils.isNotEmpty(configuration.getUrlType()) && configuration.getUrlType().equalsIgnoreCase("jdbcUrl")) { if (configuration.getJdbcUrl().contains("password=")) { - String[] params = configuration.getJdbcUrl().split(";")[1].split("&"); + String[] params = configuration.getJdbcUrl().split(";"); String pd = ""; for (int i = 0; i < params.length; i++) { if (params[i].contains("password=")) { From 06536a7b61630f71cdf500e19c49b656bfa7a935 Mon Sep 17 00:00:00 2001 From: junjun Date: Mon, 24 Jun 2024 16:58:22 +0800 Subject: [PATCH 02/23] =?UTF-8?q?feat(=E4=BB=AA=E8=A1=A8=E6=9D=BF):=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8B=E6=8B=89=E6=A0=91=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/manage/DatasetDataManage.java | 127 +++++++++++++++++- .../dataset/server/DatasetDataServer.java | 12 ++ .../dataease/dataset/utils/DatasetUtils.java | 56 ++++++++ .../dataease/api/dataset/DatasetDataApi.java | 5 + .../api/dataset/dto/BaseTreeNodeDTO.java | 28 ++++ 5 files changed, 221 insertions(+), 7 deletions(-) create mode 100644 core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java create mode 100644 sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 6fe82437da..7d0967455d 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -1,13 +1,10 @@ package io.dataease.dataset.manage; -import io.dataease.extensions.view.dto.ChartExtFilterDTO; import io.dataease.api.chart.dto.ColumnPermissionItem; import io.dataease.api.chart.dto.DeSortField; -import io.dataease.extensions.view.dto.ChartExtRequest; import io.dataease.api.dataset.dto.*; import io.dataease.api.dataset.union.DatasetGroupInfoDTO; import io.dataease.api.dataset.union.DatasetTableInfoDTO; -import io.dataease.extensions.view.model.SQLMeta; import io.dataease.api.ds.vo.TableField; import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; import io.dataease.auth.bo.TokenUserBO; @@ -16,16 +13,13 @@ import io.dataease.chart.utils.ChartDataBuild; import io.dataease.commons.utils.SqlparserUtils; import io.dataease.dataset.constant.DatasetTableType; import io.dataease.dataset.dto.DatasourceSchemaDTO; -import io.dataease.dataset.utils.FieldUtils; -import io.dataease.dataset.utils.SqlUtils; -import io.dataease.dataset.utils.TableUtils; +import io.dataease.dataset.utils.*; import io.dataease.datasource.dao.auto.entity.CoreDatasource; import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper; import io.dataease.datasource.manage.EngineManage; import io.dataease.datasource.provider.CalciteProvider; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.utils.DatasourceUtils; -import io.dataease.extensions.view.dto.DatasetTableFieldDTO; import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.constant.SQLConstants; import io.dataease.engine.constant.SqlPlaceholderConstants; @@ -34,11 +28,16 @@ import io.dataease.engine.trans.*; import io.dataease.engine.utils.SQLUtils; import io.dataease.engine.utils.Utils; import io.dataease.exception.DEException; +import io.dataease.extensions.view.dto.ChartExtFilterDTO; +import io.dataease.extensions.view.dto.ChartExtRequest; +import io.dataease.extensions.view.dto.DatasetTableFieldDTO; import io.dataease.extensions.view.dto.SqlVariableDetails; +import io.dataease.extensions.view.model.SQLMeta; import io.dataease.i18n.Translator; import io.dataease.utils.AuthUtils; import io.dataease.utils.BeanUtils; import io.dataease.utils.JsonUtil; +import io.dataease.utils.TreeUtils; import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -733,4 +732,118 @@ public class DatasetDataManage { } return previewData; } + + public List getFieldValueTree(List ids) throws Exception { + if (ids.isEmpty()) { + DEException.throwException("no field selected."); + } + // 根据前端传的查询组件field ids，获取所有字段枚举值并去重合并 + List> list = new ArrayList<>(); + List fields = new ArrayList<>(); + + // 根据图表计算字段，获取数据集 + List allFields = new ArrayList<>(); + DatasetTableFieldDTO field = datasetTableFieldManage.selectById(ids.getFirst()); + Long datasetGroupId = field.getDatasetGroupId(); + if (field.getChartId() != null) { + allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId())); + } + DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.get(datasetGroupId, null); + + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest()); + String sql = (String) sqlMap.get("sql"); + + allFields.addAll(datasetGroupInfoDTO.getAllFields()); + + Map dsMap = (Map) sqlMap.get("dsMap"); + boolean crossDs = Utils.isCrossDs(dsMap); + if (!crossDs) { + sql = Utils.replaceSchemaAlias(sql, dsMap); + } + + // build query sql + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); + + for (Long id : ids) { + DatasetTableFieldDTO f = datasetTableFieldManage.selectById(id); + if (f == null) { + DEException.throwException(Translator.get("i18n_no_field")); + } + // 获取allFields + fields.add(f); + } + + Map desensitizationList = new HashMap<>(); + fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null); + if (ObjectUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_no_column_permission")); + } + buildFieldName(sqlMap, fields); + + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean needOrder = Utils.isNeedOrder(dsList); + + List rowPermissionsTree = new ArrayList<>(); + TokenUserBO user = AuthUtils.getUser(); + if (user != null) { + rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId()); + } + + Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap); + WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap); + Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap); + String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, 1000); + querySQL = SqlUtils.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); + logger.info("filter tree sql: " + querySQL); + + // 通过数据源请求数据 + // 调用数据源的calcite获得data + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setQuery(querySQL); + datasourceRequest.setDsList(dsMap); + Map data = calciteProvider.fetchResultField(datasourceRequest); + List rows = (List) data.get("data"); + + // 重新构造data + Set pkSet = new HashSet<>(); +// if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(rows) && existExtSortField && originSize > 0) { +// rows = rows.stream().map(row -> ArrayUtils.subarray(row, 0, originSize)).collect(Collectors.toList()); +// } + rows = rows.stream().filter(row -> { + int length = row.length; + boolean allEmpty = true; + for (String s : row) { + if (StringUtils.isNotBlank(s)) { + allEmpty = false; + } + } + return !allEmpty; + }).toList(); + List treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList()); + List tree = DatasetUtils.mergeDuplicateTree(treeNodes, "root"); + return tree; + } + + private List buildTreeNode(String[] row, Set pkSet) { + List nodes = new ArrayList<>(); + List parentPkList = new ArrayList<>(); + for (int i = 0; i < row.length; i++) { + String text = row[i]; + + parentPkList.add(text); + String val = String.join(TreeUtils.SEPARATOR, parentPkList); + String parentVal = i == 0 ? TreeUtils.DEFAULT_ROOT : row[i - 1]; + String pk = String.join(TreeUtils.SEPARATOR, parentPkList); + if (pkSet.contains(pk)) continue; + pkSet.add(pk); + BaseTreeNodeDTO node = new BaseTreeNodeDTO(val, parentVal, StringUtils.isNotBlank(text) ? text.trim() : text, pk + TreeUtils.SEPARATOR + i); + nodes.add(node); + } + return nodes; + + } } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java index 60e9740c32..7f13ed268c 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetDataServer.java @@ -1,6 +1,7 @@ package io.dataease.dataset.server; import io.dataease.api.dataset.DatasetDataApi; +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; import io.dataease.api.dataset.dto.DatasetTableDTO; import io.dataease.api.dataset.dto.EnumValueRequest; import io.dataease.api.dataset.dto.PreviewSqlDTO; @@ -70,4 +71,15 @@ public class DatasetDataServer implements DatasetDataApi { public Long getDatasetCount(DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception { return datasetDataManage.getDatasetTotal(datasetGroupInfoDTO.getId()); } + + @Override + public List getFieldValueTree(List ids) throws Exception { + try { + return datasetDataManage.getFieldValueTree(ids); + } catch (Exception e) { + e.printStackTrace(); + LogUtil.error(e); + return null; + } + } } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java b/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java new file mode 100644 index 0000000000..d11571a87e --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/dataset/utils/DatasetUtils.java @@ -0,0 +1,56 @@ +package io.dataease.dataset.utils; + +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; +import io.dataease.utils.TreeUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @Author Junjun + */ +public class DatasetUtils { + public final static String SEPARATOR = "-de-"; + + public static List mergeDuplicateTree(List tree, String... rootPid) { + Assert.notNull(rootPid, "Root Pid cannot be null"); + if (CollectionUtils.isEmpty(tree)) { + return null; + } + List result = new ArrayList<>(); + // 构建id-节点map映射 + Map treePidMap = tree.stream().collect(Collectors.toMap(node -> node.getNodeType(), t -> t)); + tree.stream().filter(item -> ObjectUtils.isNotEmpty(item.getId())).forEach(node -> { + + String nodeType = node.getNodeType(); + String[] links = nodeType.split(SEPARATOR); + int length = links.length; + int level = Integer.parseInt(links[length - 1]); + // 判断根节点 + if (Arrays.asList(rootPid).contains(node.getPid()) && 0 == level) { + result.add(node); + } else { + //找到父元素 + String[] pLinks = new String[level]; + System.arraycopy(links, 0, pLinks, 0, level); + String parentType = Arrays.stream(pLinks).collect(Collectors.joining(SEPARATOR)) + TreeUtils.SEPARATOR + (level - 1); + BaseTreeNodeDTO parentNode = treePidMap.get(parentType); + if (parentNode == null) { + // 可能出现 rootPid 更高的节点 这个操作相当于截断 + return; + } + if (parentNode.getChildren() == null) { + parentNode.setChildren(new ArrayList()); + } + parentNode.getChildren().add(node); + } + }); + return result; + } +} diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java index 3ea305171c..fde19bcaa7 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DatasetDataApi.java @@ -1,6 +1,7 @@ package io.dataease.api.dataset; import com.github.xiaoymin.knife4j.annotations.ApiSupport; +import io.dataease.api.dataset.dto.BaseTreeNodeDTO; import io.dataease.api.dataset.dto.DatasetTableDTO; import io.dataease.api.dataset.dto.EnumValueRequest; import io.dataease.api.dataset.dto.PreviewSqlDTO; @@ -47,4 +48,8 @@ public interface DatasetDataApi { @Operation(summary = "获取数据集总数据量", hidden = true) @PostMapping("getDatasetCount") Long getDatasetCount(@RequestBody DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception; + + @Operation(summary = "获取下拉树数据", hidden = true) + @PostMapping("getFieldTree") + List getFieldValueTree(@RequestBody List ids) throws Exception; } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java new file mode 100644 index 0000000000..c6768e00c4 --- /dev/null +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/dto/BaseTreeNodeDTO.java @@ -0,0 +1,28 @@ +package io.dataease.api.dataset.dto; + + +import lombok.Data; + +import java.util.List; + +@Data +public class BaseTreeNodeDTO { + + private String id; + + private String pid; + + private String text; + + private String nodeType; + + private List children; + + public BaseTreeNodeDTO(String id, String pid, String text, String nodeType) { + this.id = id; + this.pid = pid; + this.text = text; + this.nodeType = nodeType; + } + +} From 6bd64d49e714804da00ee048d626fd2b655385da Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 25 Jun 2024 09:44:41 +0800 Subject: [PATCH 03/23] =?UTF-8?q?fix(X-Pack):=20=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E8=8F=9C=E5=8D=95=E6=97=A0=E6=B3=95=E5=B1=95?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-backend/pom.xml | 17 +++++++++++++++++ de-xpack | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/core-backend/pom.xml b/core/core-backend/pom.xml index f648cd5038..1b5596c1d3 100644 --- a/core/core-backend/pom.xml +++ b/core/core-backend/pom.xml @@ -148,6 +148,23 @@ flexmark-all ${flexmark.version} + + + io.dataease + xpack-permissions + ${project.version} + + + io.dataease + xpack-sync + ${project.version} + + + io.dataease + xpack-base + ${project.version} + + diff --git a/de-xpack b/de-xpack index 178f8f5220..05e2378fc7 160000 --- a/de-xpack +++ b/de-xpack @@ -1 +1 @@ -Subproject commit 178f8f52209c1ffb81b58c8055733d411aef054a +Subproject commit 05e2378fc74d5029527436d5965cf1d706223cd7 From 1a3090b7c6304760d1302a629f930ffa4b11c882 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Mon, 24 Jun 2024 18:09:06 +0800 Subject: [PATCH 04/23] =?UTF-8?q?fix(X-Pack):=20=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E8=8F=9C=E5=8D=95=E6=97=A0=E6=B3=95=E5=B1=95?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/io/dataease/menu/manage/MenuManage.java | 2 +- .../src/layout/components/LayoutTransition.vue | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 core/core-frontend/src/layout/components/LayoutTransition.vue diff --git a/core/core-backend/src/main/java/io/dataease/menu/manage/MenuManage.java b/core/core-backend/src/main/java/io/dataease/menu/manage/MenuManage.java index 5245e80cec..3319213d2f 100644 --- a/core/core-backend/src/main/java/io/dataease/menu/manage/MenuManage.java +++ b/core/core-backend/src/main/java/io/dataease/menu/manage/MenuManage.java @@ -87,12 +87,12 @@ public class MenuManage { } private boolean isXpackMenu(CoreMenu coreMenu) { + if (coreMenu.getId().equals(21L)) return false; return coreMenu.getId().equals(7L) || coreMenu.getPid().equals(7L) || coreMenu.getId().equals(14L) || coreMenu.getId().equals(17L) || coreMenu.getId().equals(18L) - || coreMenu.getId().equals(21L) || coreMenu.getPid().equals(21L) || coreMenu.getId().equals(25L) || coreMenu.getId().equals(26L) diff --git a/core/core-frontend/src/layout/components/LayoutTransition.vue b/core/core-frontend/src/layout/components/LayoutTransition.vue new file mode 100644 index 0000000000..594a48b0ad --- /dev/null +++ b/core/core-frontend/src/layout/components/LayoutTransition.vue @@ -0,0 +1,7 @@ + + From 7988fd2106b00826c9ecf2719b59919ab45d5367 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 25 Jun 2024 09:53:26 +0800 Subject: [PATCH 05/23] =?UTF-8?q?fix(X-Pack):=20=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E8=8F=9C=E5=8D=95=E6=97=A0=E6=B3=95=E5=B1=95?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-frontend/src/router/establish.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/core-frontend/src/router/establish.ts b/core/core-frontend/src/router/establish.ts index 271dc4f375..c7e5023466 100644 --- a/core/core-frontend/src/router/establish.ts +++ b/core/core-frontend/src/router/establish.ts @@ -4,6 +4,7 @@ import { XpackComponent } from '@/components/plugin' const modules = import.meta.glob('../views/**/*.vue') export const Layout = () => import('@/layout/index.vue') const xpackComName = 'components/plugin' +export const LayoutTransition = () => import('@/layout/components/LayoutTransition.vue') // 后端控制路由生成 export const generateRoutesFn2 = (routes: AppCustomRouteRecordRaw[]): AppRouteRecordRaw[] => { const res: AppRouteRecordRaw[] = [] @@ -37,12 +38,14 @@ export const generateRoutesFn2 = (routes: AppCustomRouteRecordRaw[]): AppRouteRe let comModule = null if (route.component === xpackComName) { comModule = XpackComponent - } else { + } else if (!route.component.startsWith('Layout')) { comModule = modules[`../views/${route.component}/index.vue`] } if (route.component === 'Layout') { data.component = Layout + } else if (route.component === 'LayoutTransition') { + data.component = LayoutTransition } else if (!comModule) { } else { data.component = comModule From 6d73ba32e8e53d868655ac8d74fb33dccfbc173b Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 25 Jun 2024 10:13:32 +0800 Subject: [PATCH 06/23] =?UTF-8?q?feat(=E4=BB=AA=E8=A1=A8=E6=9D=BF):=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8B=E6=8B=89=E6=A0=91=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/dataset/manage/DatasetDataManage.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 7d0967455d..16a18de088 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -810,11 +810,7 @@ public class DatasetDataManage { // 重新构造data Set pkSet = new HashSet<>(); -// if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(rows) && existExtSortField && originSize > 0) { -// rows = rows.stream().map(row -> ArrayUtils.subarray(row, 0, originSize)).collect(Collectors.toList()); -// } rows = rows.stream().filter(row -> { - int length = row.length; boolean allEmpty = true; for (String s : row) { if (StringUtils.isNotBlank(s)) { From fd792032825bf1a62d1351113e6aff955a1abb6a Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 25 Jun 2024 10:56:07 +0800 Subject: [PATCH 07/23] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dsql=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E4=B8=AD=E6=97=A5=E6=9C=9F=E7=B1=BB=E5=9E=8B=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/io/dataease/commons/utils/SqlparserUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java b/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java index 83c744b1f4..001ac67382 100644 --- a/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java +++ b/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java @@ -561,7 +561,7 @@ public class SqlparserUtils { if (sqlVariableDetails.getOperator().equals("in")) { return "'" + String.join("','", sqlVariableDetails.getValue()) + "'"; } else if (sqlVariableDetails.getOperator().equals("between")) { - SimpleDateFormat simpleDateFormat = new SimpleDateFormat(sqlVariableDetails.getType().size() > 1 ? (String) sqlVariableDetails.getType().get(1).replace("DD", "dd") : "YYYY"); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat(sqlVariableDetails.getType().size() > 1 ? (String) sqlVariableDetails.getType().get(1).replace("DD", "dd").replace("YYYY", "yyyy") : "yyyy"); if (StringUtils.endsWith(sqlVariableDetails.getId(), START_END_SEPARATOR)) { return simpleDateFormat.format(new Date(Long.parseLong((String) sqlVariableDetails.getValue().get(1)))); } else { From 1395caedb6b1cadceff58b97070300697fd3bd25 Mon Sep 17 00:00:00 2001 From: wanghe Date: Tue, 25 Jun 2024 10:58:46 +0800 Subject: [PATCH 08/23] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 557affa2f9..7129f2ac8b 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,16 @@ DataEase 是开源的数据可视化分析工具（ BI 工具 ），帮助用户 **DataEase 支持的数据源：** -- OLTP 数据库： MySQL、Oracle、SQL Server、PostgreSQL、MariaDB、Db2、TiDB、MongoDB-BI 等; -- OLAP 数据库： ClickHouse、Apache Doris、Apache Impala、StarRocks 等; -- 数据仓库/数据湖： Amazon RedShift 等; -- 数据文件： Excel、CSV 等; +- OLTP 数据库： MySQL、Oracle、SQL Server、PostgreSQL、MariaDB、Db2、TiDB、MongoDB-BI 等； +- OLAP 数据库： ClickHouse、Apache Doris、Apache Impala、StarRocks 等； +- 数据仓库/数据湖： Amazon RedShift 等； +- 数据文件： Excel、CSV 等； - API 数据源。 ## 快速开始 ``` -# 准备一台 2核4G 以上 Linux 服务器，以 root 运行以下一键安装脚本： +# 准备一台 2 核 4G 以上的 Linux 服务器，并以 root 用户运行以下一键安装脚本： curl -sSL https://dataease.oss-cn-hangzhou.aliyuncs.com/quick_start_v2.sh | bash @@ -74,12 +74,12 @@ curl -sSL https://dataease.oss-cn-hangzhou.aliyuncs.com/quick_start_v2.sh | bash - 数据处理：[Apache Calcite](https://github.com/apache/calcite/)、[Apache SeaTunnel](https://github.com/apache/seatunnel) - 基础设施：[Docker](https://www.docker.com/) -## 我们的其他明星开源项目 +## 飞致云的其他明星项目 -- [JumpServer](https://github.com/jumpserver/jumpserver/) - 广受欢迎的开源堡垒机 - [1Panel](https://github.com/1panel-dev/1panel/) - 现代化、开源的 Linux 服务器运维管理面板 -- [Halo](https://github.com/halo-dev/halo/) - 强大易用的开源建站工具 - [MaxKB](https://github.com/1panel-dev/MaxKB/) - 基于 LLM 大语言模型的开源知识库问答系统 +- [JumpServer](https://github.com/jumpserver/jumpserver/) - 广受欢迎的开源堡垒机 +- [Halo](https://github.com/halo-dev/halo/) - 强大易用的开源建站工具 - [MeterSphere](https://github.com/metersphere/metersphere/) - 开源的测试管理和接口测试工具 ## License From d97c0477cc75847da8af9c9837fe03db79d6e317 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Tue, 25 Jun 2024 14:22:11 +0800 Subject: [PATCH 09/23] =?UTF-8?q?fix(=E6=9F=A5=E8=AF=A2=E7=BB=84=E4=BB=B6)?= =?UTF-8?q?:=20=E6=9F=A5=E8=AF=A2=E7=BB=84=E4=BB=B6=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E5=85=B3=E8=81=94=E5=A4=B1=E6=95=88bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-frontend/src/custom-component/v-query/Component.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/core-frontend/src/custom-component/v-query/Component.vue b/core/core-frontend/src/custom-component/v-query/Component.vue index eeeee78f7b..bcdd6617ca 100644 --- a/core/core-frontend/src/custom-component/v-query/Component.vue +++ b/core/core-frontend/src/custom-component/v-query/Component.vue @@ -289,6 +289,8 @@ const updateQueryCriteria = () => { ele.checkedFields = checkedFields ele.checkedFieldsMap = checkedFieldsMap } else { + if (!ele.dataset.id || ele.optionValueSource !== 1 || ![0, 2, 5].includes(+ele.displayType)) + return const checkedFields = [] datasetFieldList.value.forEach(itx => { if (itx.tableId === ele.dataset.id) { From 465607c641fffef6c95319f13c559453ccf83297 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Tue, 25 Jun 2024 15:34:41 +0800 Subject: [PATCH 10/23] =?UTF-8?q?feat(X-Pack):=20DE=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=97=B6=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8Bapisix=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=B9=B6=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-backend/pom.xml | 17 -------- de-xpack | 2 +- .../io/dataease/utils/HttpClientUtil.java | 40 +++++++++++++++++-- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/core/core-backend/pom.xml b/core/core-backend/pom.xml index 1b5596c1d3..f648cd5038 100644 --- a/core/core-backend/pom.xml +++ b/core/core-backend/pom.xml @@ -148,23 +148,6 @@ flexmark-all ${flexmark.version} - - - io.dataease - xpack-permissions - ${project.version} - - - io.dataease - xpack-sync - ${project.version} - - - io.dataease - xpack-base - ${project.version} - - diff --git a/de-xpack b/de-xpack index 05e2378fc7..68a35e7897 160000 --- a/de-xpack +++ b/de-xpack @@ -1 +1 @@ -Subproject commit 05e2378fc74d5029527436d5965cf1d706223cd7 +Subproject commit 68a35e78970ba610a3d596695b7e8017b6e31721 diff --git a/sdk/common/src/main/java/io/dataease/utils/HttpClientUtil.java b/sdk/common/src/main/java/io/dataease/utils/HttpClientUtil.java index 00814e3e86..cdc2abc6c4 100755 --- a/sdk/common/src/main/java/io/dataease/utils/HttpClientUtil.java +++ b/sdk/common/src/main/java/io/dataease/utils/HttpClientUtil.java @@ -8,10 +8,7 @@ import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPatch; -import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.*; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.HttpClientConnectionManager; @@ -225,6 +222,41 @@ public class HttpClientUtil { } } + public static String put(String url, String json, HttpClientConfig config) { + CloseableHttpClient httpClient = null; + try { + httpClient = buildHttpClient(url); + HttpPut httpPut = new HttpPut(url); + if (config == null) { + config = new HttpClientConfig(); + } + httpPut.setConfig(config.buildRequestConfig()); + Map header = config.getHeader(); + for (String key : header.keySet()) { + httpPut.addHeader(key, header.get(key)); + } + EntityBuilder entityBuilder = EntityBuilder.create(); + entityBuilder.setText(json); + entityBuilder.setContentType(ContentType.APPLICATION_JSON); + HttpEntity requestEntity = entityBuilder.build(); + httpPut.setEntity(requestEntity); + + HttpResponse response = httpClient.execute(httpPut); + return getResponseStr(response, config); + } catch (Exception e) { + logger.error("HttpClient查询失败", e); + throw new DEException(SYSTEM_INNER_ERROR.code(), "HttpClient查询失败: " + e.getMessage()); + } finally { + try { + if (httpClient != null) { + httpClient.close(); + } + } catch (Exception e) { + logger.error("HttpClient关闭连接失败", e); + } + } + } + /** * Post请求，请求内容必须为JSON格式的字符串 * From 4c2db14934ddafff5eb3cb50bf91211e7778d89e Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 25 Jun 2024 15:53:15 +0800 Subject: [PATCH 11/23] =?UTF-8?q?feat:=20API=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E6=94=AF=E6=8C=81Token=E8=AE=A4=E8=AF=81=20#9189?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/provider/ApiUtils.java | 50 ++++++++-- .../datasource/server/DatasourceServer.java | 20 +++- .../datasource/form/ApiHttpRequestDraw.vue | 30 ++++-- .../datasource/form/ApiHttpRequestForm.vue | 5 + .../data/datasource/form/ApiKeyValue.vue | 31 ++++-- .../data/datasource/form/EditorDetail.vue | 96 ++++++++++--------- .../visualized/data/datasource/form/index.vue | 9 +- .../visualized/data/datasource/form/option.ts | 2 + .../visualized/data/datasource/index.vue | 10 ++ .../io/dataease/api/ds/vo/ApiDefinition.java | 1 + .../io/dataease/api/ds/vo/DatasourceDTO.java | 1 + 11 files changed, 183 insertions(+), 72 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java index d6317d83cc..8bc6e4f259 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java @@ -41,6 +41,9 @@ public class ApiUtils { }; List apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference); for (ApiDefinition apiDefinition : apiDefinitionList) { + if (StringUtils.isNotEmpty(apiDefinition.getType()) && apiDefinition.getType().equalsIgnoreCase("params")) { + continue; + } DatasetTableDTO datasetTableDTO = new DatasetTableDTO(); datasetTableDTO.setTableName(apiDefinition.getDeTableName()); datasetTableDTO.setName(apiDefinition.getName()); @@ -59,7 +62,7 @@ public class ApiUtils { if (apiDefinition == null) { DEException.throwException("未找到"); } - String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout()); + String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest)); fieldList = getTableFields(apiDefinition); result.put("fieldList", fieldList); dataList = fetchResult(response, apiDefinition); @@ -116,19 +119,38 @@ public class ApiUtils { if (apiDefinition == null) { DEException.throwException("未找到"); } - String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout()); + String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest)); return fetchResult(response, apiDefinition); } - public static String execHttpRequest(ApiDefinition apiDefinition, int socketTimeout) { + public static String execHttpRequest(ApiDefinition apiDefinition, int socketTimeout, List paramsList) { String response = ""; HttpClientConfig httpClientConfig = new HttpClientConfig(); httpClientConfig.setSocketTimeout(socketTimeout * 1000); ApiDefinitionRequest apiDefinitionRequest = apiDefinition.getRequest(); for (Map header : apiDefinitionRequest.getHeaders()) { if (header.get("name") != null && StringUtils.isNotEmpty(header.get("name").toString()) && header.get("value") != null && StringUtils.isNotEmpty(header.get("value").toString())) { - httpClientConfig.addHeader(header.get("name").toString(), header.get("value").toString()); + if (header.get("nameType") != null && header.get("nameType").toString().equalsIgnoreCase("params")) { + String param = header.get("value").toString(); + for (ApiDefinition definition : paramsList) { + for (int i = 0; i < definition.getFields().size(); i++) { + TableField field = definition.getFields().get(i); + if (field.getOriginName().equalsIgnoreCase(param)) { + String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null); + List dataList = fetchResult(resultStr, definition); + System.out.println(dataList.get(0)[i]); + if (dataList.size() > 0) { + httpClientConfig.addHeader(header.get("name").toString(), dataList.get(0)[i]); + } + } + } + + } + } else { + httpClientConfig.addHeader(header.get("name").toString(), header.get("value").toString()); + } + } } if (apiDefinitionRequest.getAuthManager() != null @@ -194,7 +216,7 @@ public class ApiUtils { return response; } - private static void previewNum(List> field){ + private static void previewNum(List> field) { for (Map stringObjectMap : field) { JSONArray newArray = new JSONArray(); if (stringObjectMap.get("value") != null) { @@ -202,7 +224,7 @@ public class ApiUtils { TypeReference listTypeReference = new TypeReference() { }; JSONArray array = objectMapper.readValue(stringObjectMap.get("value").toString(), listTypeReference); - if(array.size() > 100){ + if (array.size() > 100) { for (int i = 0; i < Math.min(100, array.size()); i++) { newArray.add(array.get(i)); } @@ -254,8 +276,8 @@ public class ApiUtils { } int i = 0; try { - LinkedHashMap data = currentData.get(0); - }catch (Exception e){ + LinkedHashMap data = currentData.get(0); + } catch (Exception e) { DEException.throwException("数据不符合规范, " + e.getMessage()); } for (LinkedHashMap data : currentData) { @@ -540,6 +562,18 @@ public class ApiUtils { } + private static List params(DatasourceRequest datasourceRequest) { + TypeReference> listTypeReference = new TypeReference>() { + }; + List apiDefinitionListTemp = null; + try { + apiDefinitionListTemp = objectMapper.readValue(datasourceRequest.getDatasource().getConfiguration(), listTypeReference); + } catch (Exception e) { + DEException.throwException(e); + } + return apiDefinitionListTemp.stream().filter(apiDefinition -> apiDefinition.getType() != null && apiDefinition.getType().equalsIgnoreCase("params")).collect(Collectors.toList()); + } + private static ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws DEException { List apiDefinitionList = new ArrayList<>(); TypeReference> listTypeReference = new TypeReference>() { diff --git a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java index bbc252b61e..33697e9c02 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java @@ -107,6 +107,8 @@ public class DatasourceServer implements DatasourceApi { all_scope, add_scope } + private TypeReference> listTypeReference = new TypeReference>() { + }; @Resource private CommonThreadPool commonThreadPool; @@ -505,11 +507,11 @@ public class DatasourceServer implements DatasourceApi { DEException.throwException("不存在的数据源！"); } BeanUtils.copyBean(datasourceDTO, datasource); - TypeReference> listTypeReference = new TypeReference>() { - }; + if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.API.toString())) { List apiDefinitionList = JsonUtil.parseList(datasourceDTO.getConfiguration(), listTypeReference); List apiDefinitionListWithStatus = new ArrayList<>(); + List params = new ArrayList<>(); int success = 0; for (ApiDefinition apiDefinition : apiDefinitionList) { String status = null; @@ -534,9 +536,16 @@ public class DatasourceServer implements DatasourceApi { if (log != null) { apiDefinition.setUpdateTime(log.getStartTime()); } - apiDefinitionListWithStatus.add(apiDefinition); + + + if (StringUtils.isEmpty(apiDefinition.getType()) || apiDefinition.getType().equalsIgnoreCase("table")) { + apiDefinitionListWithStatus.add(apiDefinition); + } else { + params.add(apiDefinition); + } } datasourceDTO.setApiConfigurationStr(new String(Base64.getEncoder().encode(Objects.requireNonNull(JsonUtil.toJSONString(apiDefinitionListWithStatus)).toString().getBytes()))); + datasourceDTO.setParamsStr(new String(Base64.getEncoder().encode(Objects.requireNonNull(JsonUtil.toJSONString(params)).toString().getBytes()))); if (success == apiDefinitionList.size()) { datasourceDTO.setStatus("Success"); } else { @@ -847,12 +856,13 @@ public class DatasourceServer implements DatasourceApi { } public ApiDefinition checkApiDatasource(Map request) throws DEException { + ApiDefinition apiDefinition = JsonUtil.parseObject(new String(java.util.Base64.getDecoder().decode(request.get("data"))), ApiDefinition.class); - String response = ApiUtils.execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout()); + List paramsList = JsonUtil.parseList(new String(java.util.Base64.getDecoder().decode(request.get("paramsList"))), listTypeReference); + String response = ApiUtils.execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), paramsList); if (request.keySet().contains("type") && request.get("type").equals("apiStructure")) { apiDefinition.setShowApiStructure(true); } - ApiUtils.checkApiDefinition(apiDefinition, response); if (apiDefinition.getRequest().getAuthManager() != null && StringUtils.isNotBlank(apiDefinition.getRequest().getAuthManager().getUsername()) && StringUtils.isNotBlank(apiDefinition.getRequest().getAuthManager().getPassword()) && apiDefinition.getRequest().getAuthManager().getVerification().equals("Basic Auth")) { apiDefinition.getRequest().getAuthManager().setUsername(new String(Base64.getEncoder().encode(apiDefinition.getRequest().getAuthManager().getUsername().getBytes()))); diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue index 80c1bfec5a..0575cf124f 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue @@ -24,6 +24,7 @@ export interface Field { export interface ApiItem { status: string name: string + type: string deTableName?: string url: string method: string @@ -57,10 +58,12 @@ const originFieldItem = reactive({ }) let apiItemList = reactive([]) +let paramsList = reactive([]) let apiItem = reactive({ status: '', name: '', + type: 'table', url: '', method: 'GET', request: { @@ -95,6 +98,7 @@ const edit_api_item = ref(false) const active = ref(1) const loading = ref(false) const columns = shallowRef([]) +const valueList = shallowRef([]) const tableData = shallowRef([]) const apiItemBasicInfo = ref() const isNumber = (rule, value, callback) => { @@ -152,9 +156,16 @@ const rule = reactive({ }) const activeName = ref('third') provide('api-active-name', activeName) -const initApiItem = (val: ApiItem, apiList, name) => { +const initApiItem = (val: ApiItem, from, name) => { activeName.value = name - apiItemList = apiList + apiItemList = from.apiConfiguration + paramsList = from.paramsConfiguration + if (val.type !== 'params') { + valueList.value = [] + for (let i = 0; i < paramsList.length; i++) { + valueList.value = valueList.value.concat(paramsList[i].fields) + } + } Object.assign(apiItem, val) edit_api_item.value = true active.value = 0 @@ -167,9 +178,10 @@ const showApiData = () => { apiItemBasicInfo.value.validate(valid => { if (valid) { const data = Base64.encode(JSON.stringify(apiItem)) + const params = Base64.encode(JSON.stringify(paramsList)) loading.value = true cancelMap['/datasource/checkApiDatasource']?.() - checkApiItem({ data: data, type: 'apiStructure' }) + checkApiItem({ data: data, type: 'apiStructure', paramsList: params }) .then(response => { originFieldItem.jsonFields = response.data.jsonFields }) @@ -203,7 +215,7 @@ const fieldOptions = [ ] const disabledNext = ref(false) const saveItem = () => { - if (apiItem.fields.length === 0) { + if (apiItem.type !== 'params' && apiItem.fields.length === 0) { ElMessage.error(t('datasource.api_field_not_empty')) return } @@ -239,7 +251,8 @@ const next = () => { } cancelMap['/datasource/checkApiDatasource']?.() - checkApiItem({ data: Base64.encode(JSON.stringify(apiItem)) }) + const params = Base64.encode(JSON.stringify(paramsList)) + checkApiItem({ data: Base64.encode(JSON.stringify(apiItem)), paramsList: params }) .then(response => { apiItem.jsonFields = response.data.jsonFields apiItem.fields = [] @@ -465,6 +478,7 @@ defineExpose({ @@ -572,7 +586,11 @@ defineExpose({ - +