From c1509be15b626a05148aba0950bb7e65810a7064 Mon Sep 17 00:00:00 2001
From: wangjiahao <1522128093@qq.com>
Date: Wed, 30 Nov 2022 13:36:23 +0800
Subject: [PATCH] =?UTF-8?q?fix(=E5=BA=94=E7=94=A8):=20=E4=BF=AE=E5=A4=8D?=
 =?UTF-8?q?=E5=BA=94=E7=94=A8=E5=88=9B=E5=BB=BA=E9=9D=9E=E7=9B=B4=E8=BF=9E?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=9B=A0doris=E8=A1=A8=E5=88=9B?=
 =?UTF-8?q?=E5=BB=BA=E5=BB=B6=E8=BF=9F=E8=80=8C=E5=AF=BC=E8=87=B4=E7=9A=84?=
 =?UTF-8?q?=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../io/dataease/ext/ExtDataSetTableMapper.xml |  2 +-
 .../service/dataset/ExtractDataService.java   | 30 ++++++-------
 .../panel/PanelAppTemplateService.java        | 25 ++++++++++-
 .../service/panel/PanelGroupService.java      | 45 ++++++++-----------
 4 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/backend/src/main/java/io/dataease/ext/ExtDataSetTableMapper.xml b/backend/src/main/java/io/dataease/ext/ExtDataSetTableMapper.xml
index deaa2f183b..38440ec0c7 100644
--- a/backend/src/main/java/io/dataease/ext/ExtDataSetTableMapper.xml
+++ b/backend/src/main/java/io/dataease/ext/ExtDataSetTableMapper.xml
@@ -149,7 +149,7 @@
     </select>
 
     <select id="findByTableIds" resultMap="BaseResultMapDTO">
-        select dataset_table.*
+        select distinct dataset_table.*
         from dataset_table
         where dataset_table.id in
         <foreach collection="tableIds" item="item" open="(" separator="," close=")">
diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
index 3ace12e750..adff409cb7 100644
--- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
@@ -2,11 +2,16 @@ package io.dataease.service.dataset;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
-import io.dataease.ext.ExtChartViewMapper;
 import io.dataease.commons.constants.*;
 import io.dataease.commons.model.AuthURD;
 import io.dataease.commons.utils.*;
 import io.dataease.controller.request.datasource.ApiDefinition;
+import io.dataease.dto.dataset.DataTableInfoDTO;
+import io.dataease.dto.dataset.ExcelSheetData;
+import io.dataease.dto.datasource.*;
+import io.dataease.exception.DataEaseException;
+import io.dataease.ext.ExtChartViewMapper;
+import io.dataease.listener.util.CacheUtils;
 import io.dataease.plugins.common.base.domain.*;
 import io.dataease.plugins.common.constants.DatasetType;
 import io.dataease.plugins.common.constants.DatasourceTypes;
@@ -17,14 +22,9 @@ import io.dataease.plugins.datasource.entity.JdbcConfiguration;
 import io.dataease.plugins.datasource.provider.Provider;
 import io.dataease.plugins.datasource.query.QueryProvider;
 import io.dataease.provider.DDLProvider;
-import io.dataease.provider.datasource.JdbcProvider;
 import io.dataease.provider.ProviderFactory;
-import io.dataease.dto.datasource.*;
+import io.dataease.provider.datasource.JdbcProvider;
 import io.dataease.service.datasource.DatasourceService;
-import io.dataease.dto.dataset.DataTableInfoDTO;
-import io.dataease.dto.dataset.ExcelSheetData;
-import io.dataease.exception.DataEaseException;
-import io.dataease.listener.util.CacheUtils;
 import io.dataease.service.engine.EngineService;
 import io.dataease.service.kettle.KettleService;
 import io.dataease.service.message.DeMsgutil;
@@ -413,7 +413,7 @@ public class ExtractDataService {
 
     }
 
-    private List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
+    public List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
         List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
         datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
         datasetTableFields.sort((o1, o2) -> {
@@ -617,7 +617,7 @@ public class ExtractDataService {
         dataSetTableTaskLogService.save(datasetTableTaskLog, hasTask);
     }
 
-    private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
+    public void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
         Datasource engine = engineService.getDeEngine();
         JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
         DatasourceRequest datasourceRequest = new DatasourceRequest();
@@ -793,7 +793,7 @@ public class ExtractDataService {
             Thread.sleep(1000);
         }
         if (jobStatus.getStatusDescription().equals("Finished")) {
-            LogUtil.info(datasetTable.getId()+ ": " + jobStatus.getLoggingString());
+            LogUtil.info(datasetTable.getId() + ": " + jobStatus.getLoggingString());
             return;
         } else {
             DataEaseException.throwException(jobStatus.getLoggingString());
@@ -1029,7 +1029,7 @@ public class ExtractDataService {
         if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase(DatasetType.SQL.name())) {
             DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
             selectSQL = dataTableInfoDTO.getSql();
-            if(dataTableInfoDTO.isBase64Encryption()){
+            if (dataTableInfoDTO.isBase64Encryption()) {
                 selectSQL = new String(java.util.Base64.getDecoder().decode(selectSQL));
             }
             QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
@@ -1056,13 +1056,13 @@ public class ExtractDataService {
     }
 
     private List<StepMeta> excelInputStep(String Info, List<DatasetTableField> datasetTableFields) {
-        List<StepMeta>inputSteps = new ArrayList<>();
+        List<StepMeta> inputSteps = new ArrayList<>();
         DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
         List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
 
         List<String> sheetNames = new ArrayList<>();
 
-        int size =1;
+        int size = 1;
         for (ExcelSheetData excelSheetData : excelSheetDataList) {
             StepMeta fromStep = null;
             String suffix = excelSheetData.getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
@@ -1090,7 +1090,7 @@ public class ExtractDataService {
                 fromStep.setDraw(true);
                 fromStep.setLocation(100, 100 * size);
                 inputSteps.add(fromStep);
-            }else {
+            } else {
                 List<String> files = new ArrayList<>();
                 files.add(excelSheetData.getPath());
 
@@ -1240,7 +1240,7 @@ public class ExtractDataService {
             if (StringUtils.isNotEmpty(charset)) {
                 String varcharFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getDeExtractType() == 0).map(DatasetTableField::getOriginName).collect(Collectors.joining(","));
                 tmp_code = tmp_code.replace("handleCharset", handleCharset.replace("Datasource_Charset", charset).replace("Target_Charset", targetCharset).replace("varcharFields", varcharFields));
-            }else {
+            } else {
                 tmp_code = tmp_code.replace("handleCharset", "");
             }
         } else {
diff --git a/backend/src/main/java/io/dataease/service/panel/PanelAppTemplateService.java b/backend/src/main/java/io/dataease/service/panel/PanelAppTemplateService.java
index 50286f90d0..3f9adc3685 100644
--- a/backend/src/main/java/io/dataease/service/panel/PanelAppTemplateService.java
+++ b/backend/src/main/java/io/dataease/service/panel/PanelAppTemplateService.java
@@ -20,6 +20,7 @@ import io.dataease.service.chart.ChartViewService;
 import io.dataease.service.dataset.DataSetGroupService;
 import io.dataease.service.dataset.DataSetTableFieldsService;
 import io.dataease.service.dataset.DataSetTableService;
+import io.dataease.service.dataset.ExtractDataService;
 import io.dataease.service.datasource.DatasourceService;
 import io.dataease.service.staticResource.StaticResourceService;
 import org.apache.commons.lang3.StringUtils;
@@ -68,6 +69,8 @@ public class PanelAppTemplateService {
     private DataSetGroupService dataSetGroupService;
     @Resource
     private StaticResourceService staticResourceService;
+    @Resource
+    private ExtractDataService extractDataService;
 
     public List<PanelAppTemplateWithBLOBs> list(PanelAppTemplateRequest request) {
         return extPanelAppTemplateMapper.queryBaseInfo(request.getNodeType(), request.getPid());
@@ -209,6 +212,7 @@ public class PanelAppTemplateService {
                 DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
                 datasetFieldsRealMap.put(oldId, newTableField.getId());
                 datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + datasetTableField.getOriginName()), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
+                datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
             }
         }
         //数据集计算字段替换
@@ -225,6 +229,8 @@ public class PanelAppTemplateService {
                 DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
                 datasetFieldsRealMap.put(oldId, newTableField.getId());
                 datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + oldOriginName), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
+                datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
+
             }
         }
 
@@ -233,13 +239,28 @@ public class PanelAppTemplateService {
             if (DatasetType.UNION.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId())) || DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId()))) {
                 DatasetTableField updateField = new DatasetTableField();
                 updateField.setId(datasetTableField.getId());
-                updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName()));
-                dataSetTableFieldsService.updateByPrimaryKeySelective(updateField);
+                String newOriginName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName());
+                String dataeaseName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName());
+                if (StringUtils.isNotEmpty(newOriginName) || StringUtils.isNotEmpty(dataeaseName)) {
+                    updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName()));
+                    updateField.setDataeaseName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName()));
+                    dataSetTableFieldsService.updateByPrimaryKeySelective(updateField);
+                }
             }
         }
         return datasetFieldsRealMap;
     }
 
+    @Transactional(rollbackFor = Exception.class)
+    public void createDorisTable(List<DatasetTable> datasetTablesInfo) throws Exception {
+        for (DatasetTable datasetTable : datasetTablesInfo) {
+            if (1 == datasetTable.getMode() && !(DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTable.getType()) || DatasetType.UNION.name().equalsIgnoreCase(datasetTable.getType()))) {
+                List<DatasetTableField> fields = extractDataService.getDatasetTableFields(datasetTable.getId());
+                extractDataService.createEngineTable(TableUtils.tableName(datasetTable.getId()), fields);
+            }
+        }
+    }
+
     @Transactional(rollbackFor = Exception.class)
     public void resetCustomAndUnionDataset(List<DatasetTable> datasetTablesInfo, Map<String, String> datasetRealMap, Map<String, String> datasetFieldsRealMap) throws Exception {
         for (DatasetTable datasetTable : datasetTablesInfo) {
diff --git a/backend/src/main/java/io/dataease/service/panel/PanelGroupService.java b/backend/src/main/java/io/dataease/service/panel/PanelGroupService.java
index 35d2293ad3..1ed3bfa3e4 100644
--- a/backend/src/main/java/io/dataease/service/panel/PanelGroupService.java
+++ b/backend/src/main/java/io/dataease/service/panel/PanelGroupService.java
@@ -30,7 +30,6 @@ import io.dataease.listener.util.CacheUtils;
 import io.dataease.plugins.common.base.domain.*;
 import io.dataease.plugins.common.base.mapper.*;
 import io.dataease.plugins.common.constants.DeTypeConstants;
-import io.dataease.service.SystemInfoService;
 import io.dataease.service.chart.ChartViewService;
 import io.dataease.service.dataset.DataSetGroupService;
 import io.dataease.service.dataset.DataSetTableService;
@@ -135,8 +134,6 @@ public class PanelGroupService {
     private DatasetGroupMapper datasetGroupMapper;
     @Resource
     private PanelWatermarkMapper panelWatermarkMapper;
-    @Resource
-    private SystemInfoService systemInfoService;
 
     public List<PanelGroupDTO> tree(PanelGroupRequest panelGroupRequest) {
         String userId = String.valueOf(AuthUtils.getUser().getUserId());
@@ -811,29 +808,6 @@ public class PanelGroupService {
         List<ChartViewField> chartViewFieldsInfo = extChartViewFieldMapper.findByPanelId(panelId);
         //3.获取所有数据集信息
         List<DatasetTable> datasetTablesInfo = extDataSetTableMapper.findByPanelId(panelId);
-        List<String> attachTableIds = new ArrayList<>();
-        if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
-            for (DatasetTable datasetTable : datasetTablesInfo) {
-                if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
-                    DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
-                    DatasetUtils.getUnionTable(attachTableIds, dt.getUnion());
-                } else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
-                    Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
-                    List<Map> list = (List<Map>) result.get("list");
-                    if (CollectionUtils.isNotEmpty(list)) {
-                        for (Map details : list) {
-                            attachTableIds.add(String.valueOf(details.get("tableId")));
-                        }
-                    }
-                }
-            }
-            if (CollectionUtils.isNotEmpty(attachTableIds)) {
-                List<DatasetTable> attachDatasetTables = extDataSetTableMapper.findByTableIds(attachTableIds);
-                if (CollectionUtils.isNotEmpty(attachDatasetTables)) {
-                    datasetTablesInfo.addAll(attachDatasetTables);
-                }
-            }
-        }
         // dataset check
         if (CollectionUtils.isEmpty(datasetTablesInfo)) {
             return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR"));
@@ -841,6 +815,23 @@ public class PanelGroupService {
             return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET"));
         }
         List<String> allTableIds = datasetTablesInfo.stream().map(DatasetTable::getId).collect(Collectors.toList());
+        if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
+            for (DatasetTable datasetTable : datasetTablesInfo) {
+                if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
+                    DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
+                    DatasetUtils.getUnionTable(allTableIds, dt.getUnion());
+                } else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
+                    Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
+                    List<Map> list = (List<Map>) result.get("list");
+                    if (CollectionUtils.isNotEmpty(list)) {
+                        for (Map details : list) {
+                            allTableIds.add(String.valueOf(details.get("tableId")));
+                        }
+                    }
+                }
+            }
+        }
+        datasetTablesInfo = extDataSetTableMapper.findByTableIds(allTableIds);
         //4.获取所有数据集字段信息
         List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByTableIds(allTableIds);
         //5.获取所有任务信息
@@ -917,6 +908,8 @@ public class PanelGroupService {
 
         Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap, datasetTypeRealMap, datasetFieldsMd5FormatRealMap);
 
+        panelAppTemplateService.createDorisTable(datasetTablesInfo);
+        
         panelAppTemplateService.resetCustomAndUnionDataset(datasetTablesInfo, datasetsRealMap, datasetFieldsRealMap);
 
         Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, datasetFieldsMd5FormatRealMap, newPanelId);