From 260d09a1a01324f23d66921e82bb411cd6a147d7 Mon Sep 17 00:00:00 2001
From: taojinlong <jinlong@fit2cloud.com>
Date: Fri, 7 May 2021 16:01:04 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8A=BD=E5=8F=96=20excel=20=E6=95=B0?=
 =?UTF-8?q?=E6=8D=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../base/domain/DatasetTableField.java        |   2 +
 .../base/domain/DatasetTableFieldExample.java |  70 ++++++++++
 .../base/mapper/DatasetTableFieldMapper.xml   |  31 +++--
 .../io/dataease/commons/utils/Md5Utils.java   |  41 ++++++
 .../datasource/constants/DatasourceTypes.java |   2 +-
 .../service/dataset/DataSetTableService.java  |  15 +++
 .../service/dataset/ExtractDataService.java   | 124 ++++++++++--------
 .../db/migration/V16__dataset_table_field.sql |   2 +
 .../src/main/resources/generatorConfig.xml    |   2 +-
 9 files changed, 225 insertions(+), 64 deletions(-)
 create mode 100644 backend/src/main/java/io/dataease/commons/utils/Md5Utils.java
 create mode 100644 backend/src/main/resources/db/migration/V16__dataset_table_field.sql

diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
index 360ab4b24e..1b69178841 100644
--- a/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
+++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
@@ -16,6 +16,8 @@ public class DatasetTableField implements Serializable {
 
     private String name;
 
+    private String dataeaseName;
+
     private String type;
 
     private Integer size;
diff --git a/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java b/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java
index 462a9a2b31..a6154b55c7 100644
--- a/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java
+++ b/backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java
@@ -384,6 +384,76 @@ public class DatasetTableFieldExample {
             return (Criteria) this;
         }
 
+        public Criteria andDataeaseNameIsNull() {
+            addCriterion("dataease_name is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameIsNotNull() {
+            addCriterion("dataease_name is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameEqualTo(String value) {
+            addCriterion("dataease_name =", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotEqualTo(String value) {
+            addCriterion("dataease_name <>", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameGreaterThan(String value) {
+            addCriterion("dataease_name >", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameGreaterThanOrEqualTo(String value) {
+            addCriterion("dataease_name >=", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLessThan(String value) {
+            addCriterion("dataease_name <", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLessThanOrEqualTo(String value) {
+            addCriterion("dataease_name <=", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameLike(String value) {
+            addCriterion("dataease_name like", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotLike(String value) {
+            addCriterion("dataease_name not like", value, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameIn(List<String> values) {
+            addCriterion("dataease_name in", values, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotIn(List<String> values) {
+            addCriterion("dataease_name not in", values, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameBetween(String value1, String value2) {
+            addCriterion("dataease_name between", value1, value2, "dataeaseName");
+            return (Criteria) this;
+        }
+
+        public Criteria andDataeaseNameNotBetween(String value1, String value2) {
+            addCriterion("dataease_name not between", value1, value2, "dataeaseName");
+            return (Criteria) this;
+        }
+
         public Criteria andTypeIsNull() {
             addCriterion("`type` is null");
             return (Criteria) this;
diff --git a/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml b/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml
index 6bea2c8c99..67053d2080 100644
--- a/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml
+++ b/backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml
@@ -6,6 +6,7 @@
     <result column="table_id" jdbcType="VARCHAR" property="tableId" />
     <result column="origin_name" jdbcType="VARCHAR" property="originName" />
     <result column="name" jdbcType="VARCHAR" property="name" />
+    <result column="dataease_name" jdbcType="VARCHAR" property="dataeaseName" />
     <result column="type" jdbcType="VARCHAR" property="type" />
     <result column="size" jdbcType="INTEGER" property="size" />
     <result column="de_type" jdbcType="INTEGER" property="deType" />
@@ -72,8 +73,8 @@
     </where>
   </sql>
   <sql id="Base_Column_List">
-    id, table_id, origin_name, `name`, `type`, `size`, de_type, `checked`, column_index, 
-    last_sync_time
+    id, table_id, origin_name, `name`, dataease_name, `type`, `size`, de_type, `checked`, 
+    column_index, last_sync_time
   </sql>
   <select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableFieldExample" resultMap="BaseResultMap">
     select
@@ -107,13 +108,13 @@
   </delete>
   <insert id="insert" parameterType="io.dataease.base.domain.DatasetTableField">
     insert into dataset_table_field (id, table_id, origin_name, 
-      `name`, `type`, `size`, de_type, 
-      `checked`, column_index, last_sync_time
-      )
+      `name`, dataease_name, `type`, 
+      `size`, de_type, `checked`, 
+      column_index, last_sync_time)
     values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{originName,jdbcType=VARCHAR}, 
-      #{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, 
-      #{checked,jdbcType=BIT}, #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT}
-      )
+      #{name,jdbcType=VARCHAR}, #{dataeaseName,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, 
+      #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, #{checked,jdbcType=BIT}, 
+      #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT})
   </insert>
   <insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableField">
     insert into dataset_table_field
@@ -130,6 +131,9 @@
       <if test="name != null">
         `name`,
       </if>
+      <if test="dataeaseName != null">
+        dataease_name,
+      </if>
       <if test="type != null">
         `type`,
       </if>
@@ -162,6 +166,9 @@
       <if test="name != null">
         #{name,jdbcType=VARCHAR},
       </if>
+      <if test="dataeaseName != null">
+        #{dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="type != null">
         #{type,jdbcType=VARCHAR},
       </if>
@@ -203,6 +210,9 @@
       <if test="record.name != null">
         `name` = #{record.name,jdbcType=VARCHAR},
       </if>
+      <if test="record.dataeaseName != null">
+        dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="record.type != null">
         `type` = #{record.type,jdbcType=VARCHAR},
       </if>
@@ -232,6 +242,7 @@
       table_id = #{record.tableId,jdbcType=VARCHAR},
       origin_name = #{record.originName,jdbcType=VARCHAR},
       `name` = #{record.name,jdbcType=VARCHAR},
+      dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
       `type` = #{record.type,jdbcType=VARCHAR},
       `size` = #{record.size,jdbcType=INTEGER},
       de_type = #{record.deType,jdbcType=INTEGER},
@@ -254,6 +265,9 @@
       <if test="name != null">
         `name` = #{name,jdbcType=VARCHAR},
       </if>
+      <if test="dataeaseName != null">
+        dataease_name = #{dataeaseName,jdbcType=VARCHAR},
+      </if>
       <if test="type != null">
         `type` = #{type,jdbcType=VARCHAR},
       </if>
@@ -280,6 +294,7 @@
     set table_id = #{tableId,jdbcType=VARCHAR},
       origin_name = #{originName,jdbcType=VARCHAR},
       `name` = #{name,jdbcType=VARCHAR},
+      dataease_name = #{dataeaseName,jdbcType=VARCHAR},
       `type` = #{type,jdbcType=VARCHAR},
       `size` = #{size,jdbcType=INTEGER},
       de_type = #{deType,jdbcType=INTEGER},
diff --git a/backend/src/main/java/io/dataease/commons/utils/Md5Utils.java b/backend/src/main/java/io/dataease/commons/utils/Md5Utils.java
new file mode 100644
index 0000000000..aeb2ab72a6
--- /dev/null
+++ b/backend/src/main/java/io/dataease/commons/utils/Md5Utils.java
@@ -0,0 +1,41 @@
+package io.dataease.commons.utils;
+
+import java.util.List;
+import java.io.File;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class Md5Utils {
+    private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+    private static final String UTF_8 = "UTF-8";
+
+    public static String md5(String src) {
+        return md5(src, UTF_8);
+    }
+
+    public static String md5(String src, String charset) {
+        try {
+            byte[] strTemp = charset == null || charset.equals("") ? src.getBytes() : src.getBytes(charset);
+            MessageDigest mdTemp = MessageDigest.getInstance("MD5");
+            mdTemp.update(strTemp);
+
+            byte[] md = mdTemp.digest();
+            int j = md.length;
+            char[] str = new char[j * 2];
+            int k = 0;
+
+            for (byte byte0 : md) {
+                str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
+                str[k++] = HEX_DIGITS[byte0 & 0xf];
+            }
+
+            return new String(str);
+        } catch (Exception e) {
+            throw new RuntimeException("MD5 encrypt error:", e);
+        }
+    }
+}
diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
index e2982eb4cd..5b704e2a5b 100644
--- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
+++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
@@ -1,5 +1,5 @@
 package io.dataease.datasource.constants;
 
 public enum DatasourceTypes {
-    mysql, sqlServer
+    mysql, sqlServer, excel
 }
diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
index d166a86462..4acac7aea5 100644
--- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
@@ -8,6 +8,8 @@ import io.dataease.base.mapper.DatasetTableMapper;
 import io.dataease.base.mapper.DatasourceMapper;
 import io.dataease.commons.utils.AuthUtils;
 import io.dataease.commons.utils.BeanUtils;
+import io.dataease.commons.utils.CommonThreadPool;
+import io.dataease.commons.utils.Md5Utils;
 import io.dataease.controller.request.dataset.DataSetTableRequest;
 import io.dataease.datasource.constants.DatasourceTypes;
 import io.dataease.datasource.dto.TableFiled;
@@ -54,6 +56,10 @@ public class DataSetTableService {
     @Resource
     private DataSetTableTaskService dataSetTableTaskService;
     @Resource
+    private CommonThreadPool commonThreadPool;
+    @Resource
+    private ExtractDataService extractDataService;
+    @Resource
     private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
     @Value("${upload.file.path}")
     private String path;
@@ -79,6 +85,9 @@ public class DataSetTableService {
             // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
             if (insert == 1) {
                 saveTableField(datasetTable);
+                commonThreadPool.addTask(()->{
+                    extractDataService.extractData(datasetTable.getId(), null, "all_scope");
+                });
             }
         } else {
             int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
@@ -330,6 +339,11 @@ public class DataSetTableService {
                 datasetTableField.setTableId(datasetTable.getId());
                 datasetTableField.setOriginName(filed.getFieldName());
                 datasetTableField.setName(filed.getRemarks());
+                if(StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")){
+                    datasetTableField.setDataeaseName("C_" + Md5Utils.md5(filed.getFieldName()));
+                }else {
+                    datasetTableField.setDataeaseName(filed.getFieldName());
+                }
                 datasetTableField.setType(filed.getFieldType());
                 if (ObjectUtils.isEmpty(ds)) {
                     datasetTableField.setDeType(transFieldType(filed.getFieldType()));
@@ -560,6 +574,7 @@ public class DataSetTableService {
                         tableFiled.setFieldName(readCell(row.getCell(j)));
                         tableFiled.setRemarks(readCell(row.getCell(j)));
                         tableFiled.setFieldType("TEXT");
+                        tableFiled.setFieldSize(1024);
                         fields.add(tableFiled);
                     } else {
                         r[j] = readCell(row.getCell(j));
diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
index 5b2a638017..28a4bd5fe5 100644
--- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
@@ -39,6 +39,9 @@ import org.pentaho.di.repository.filerep.KettleFileRepository;
 import org.pentaho.di.trans.TransHopMeta;
 import org.pentaho.di.trans.TransMeta;
 import org.pentaho.di.trans.step.StepMeta;
+import org.pentaho.di.trans.steps.excelinput.ExcelInputField;
+import org.pentaho.di.trans.steps.excelinput.ExcelInputMeta;
+import org.pentaho.di.trans.steps.excelinput.SpreadSheetType;
 import org.pentaho.di.trans.steps.sql.ExecSQLMeta;
 import org.pentaho.di.trans.steps.tableinput.TableInputMeta;
 import org.pentaho.di.trans.steps.textfileoutput.TextFileField;
@@ -87,7 +90,7 @@ public class ExtractDataService {
     @Value("${carte.passwd:cluster}")
     private String passwd;
 
-    private static String creatTableSql = "CREATE TABLE IF NOT EXISTS TABLE_NAME" +
+    private static String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
             "Column_Fields" +
             "UNIQUE KEY(dataease_uuid)\n" +
             "DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
@@ -97,28 +100,28 @@ public class ExtractDataService {
             "rm -rf %s\n";
 
     private String createDorisTablColumnSql( List<DatasetTableField> datasetTableFields){
-        String Column_Fields = "dataease_uuid  varchar(50),";
+        String Column_Fields = "dataease_uuid  varchar(50), `";
         for (DatasetTableField datasetTableField : datasetTableFields) {
-            Column_Fields = Column_Fields + datasetTableField.getOriginName() + " ";
+            Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
             switch (datasetTableField.getDeType()){
                 case 0:
-                    Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
+                    Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
                     break;
                 case 1:
-                    Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
+                    Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
                     break;
                 case 2:
-                    Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
+                    Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
                     break;
                 case 3:
-                    Column_Fields = Column_Fields + "DOUBLE" + ",";
+                    Column_Fields = Column_Fields + "DOUBLE" + ",`";
                     break;
                 default:
                     Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
                     break;
             }
         }
-        Column_Fields = Column_Fields.substring(0, Column_Fields.length() -1 );
+        Column_Fields = Column_Fields.substring(0, Column_Fields.length() -2);
         Column_Fields =  "(" + Column_Fields + ")\n";
         return Column_Fields;
     }
@@ -129,7 +132,6 @@ public class ExtractDataService {
         DatasourceRequest datasourceRequest = new DatasourceRequest();
         datasourceRequest.setDatasource(dorisDatasource);
         datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql));
-        System.out.println(datasourceRequest.getQuery());
         jdbcProvider.exec(datasourceRequest);
     }
 
@@ -145,21 +147,30 @@ public class ExtractDataService {
     public void extractData(String datasetTableId, String taskId, String type) {
         DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
         UpdateType updateType = UpdateType.valueOf(type);
+        DatasetTable datasetTable = null;
+        Datasource datasource = new Datasource();
         try {
-            DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
-            Datasource datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
+            datasetTable = dataSetTableService.get(datasetTableId);
+            if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
+                datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
+            }else {
+                datasource.setType(datasetTable.getType());
+            }
+
             List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
             datasetTableFields.sort((o1, o2) -> {
-                if (o1.getOriginName() == null) {
+                if (o1.getColumnIndex() == null) {
                     return -1;
                 }
-                if (o2.getOriginName() == null) {
+                if (o2.getColumnIndex() == null) {
                     return 1;
                 }
-                return o1.getOriginName().compareTo(o2.getOriginName());
+                return o1.getColumnIndex().compareTo(o2.getColumnIndex());
             });
+            for (DatasetTableField datasetTableField : datasetTableFields) {
+                System.out.println(new Gson().toJson(datasetTableField));
+            }
 
-            String tableName =   new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
             String dorisTablColumnSql = createDorisTablColumnSql(datasetTableFields);
             switch (updateType) {
                 // 全量更新
@@ -168,8 +179,8 @@ public class ExtractDataService {
                     // TODO  before: check doris table column type
                     createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
                     createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
-                    generateTransFile("all_scope", datasetTable, datasource, tableName, datasetTableFields, null);
-                    generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList())));
+                    generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
+                    generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
                     extractData(datasetTable, "all_scope");
                     replaceTable(DorisTableUtils.dorisName(datasetTableId));
                     datasetTableTaskLog.setStatus(JobStatus.Completed.name());
@@ -196,9 +207,7 @@ public class ExtractDataService {
                     if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {
                         String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
                                 .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
-                        //TODO sql column
-
-                        generateTransFile("incremental_add", datasetTable, datasource, tableName, datasetTableFields, sql);
+                        generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
                         generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource));
                         extractData(datasetTable, "incremental_add");
                     }
@@ -207,9 +216,7 @@ public class ExtractDataService {
                     if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())) {
                         String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
                                 .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
-                        //TODO sql column
-                        ;
-                        generateTransFile("incremental_delete", datasetTable, datasource, tableName, datasetTableFields, sql);
+                        generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
                         generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource));
                         extractData(datasetTable, "incremental_delete");
                     }
@@ -395,25 +402,11 @@ public class ExtractDataService {
         return String.join(",", datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).collect(Collectors.toList()));
     }
 
-    private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, String table, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
+    private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
         TransMeta transMeta = new TransMeta();
         String dorisOutputTable = null;
         DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
         DatabaseMeta dataMeta = null;
-        switch (datasourceType) {
-            case mysql:
-                MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
-                dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
-                transMeta.addDatabase(dataMeta);
-                break;
-            default:
-                break;
-
-        }
-        Datasource dorisDatasource =  (Datasource)CommonBeanFactory.getBean("DorisDatasource");
-        MysqlConfigration dorisConfigration = new Gson().fromJson(dorisDatasource.getConfiguration(), MysqlConfigration.class);
-        DatabaseMeta dorisDataMeta =  new DatabaseMeta("doris", "MYSQL", "Native", dorisConfigration.getHost(), dorisConfigration.getDataBase(), dorisConfigration.getPort().toString(), dorisConfigration.getUsername(), dorisConfigration.getPassword());
-        transMeta.addDatabase(dorisDataMeta);
         StepMeta inputStep = null;
         StepMeta outputStep = null;
         StepMeta udjcStep = null;
@@ -421,11 +414,28 @@ public class ExtractDataService {
         TransHopMeta hi2 = null;
         String transName = null;
 
+        switch (datasourceType) {
+            case mysql:
+                MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
+                dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
+                transMeta.addDatabase(dataMeta);
+                if(extractType.equalsIgnoreCase("all_scope")){
+                    String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
+                    selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), tableName, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).toArray(String[]::new));
+                }
+                inputStep = inputStep(transMeta, selectSQL);
+                break;
+            case excel:
+                String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData();
+                inputStep = excelInputStep(filePath, datasetTableFields);
+            default:
+                break;
+        }
+
         switch (extractType) {
             case "all_scope":
                 transName = "trans_" + datasetTable.getId();
                 dorisOutputTable = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
-                selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), table, datasetTableFields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new));
                 transMeta.setName(transName);
                 break;
             case "incremental_add":
@@ -442,7 +452,6 @@ public class ExtractDataService {
                 break;
         }
 
-        inputStep = inputStep(transMeta, selectSQL);
         udjcStep = udjc(datasetTableFields);
         outputStep = outputStep(dorisOutputTable);
         hi1 = new TransHopMeta(inputStep, udjcStep);
@@ -469,6 +478,27 @@ public class ExtractDataService {
         return fromStep;
     }
 
+    private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields){
+        ExcelInputMeta excelInputMeta = new ExcelInputMeta();
+        excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
+        excelInputMeta.setPassword("Encrypted");
+        excelInputMeta.setFileName(new String[]{filePath});
+        excelInputMeta.setStartsWithHeader(true);
+        ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];
+        for (int i=0; i<datasetTableFields.size();i++) {
+            ExcelInputField field = new ExcelInputField();
+            field.setName(datasetTableFields.get(i).getOriginName());
+            field.setType("String");
+            fields[i] = field;
+        }
+
+        excelInputMeta.setField(fields);
+        StepMeta fromStep = new StepMeta("ExcelInput", "Data Input", excelInputMeta);
+        fromStep.setDraw(true);
+        fromStep.setLocation(100, 100);
+        return fromStep;
+    }
+
     private StepMeta outputStep(String dorisOutputTable){
         TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
         textFileOutputMeta.setEncoding("UTF-8");
@@ -502,20 +532,6 @@ public class ExtractDataService {
         return  userDefinedJavaClassStep;
     }
 
-    private StepMeta execSqlStep(TransMeta transMeta, String dorisOutputTable, List<DatasetTableField>datasetTableFields){
-        ExecSQLMeta execSQLMeta = new ExecSQLMeta();
-        DatabaseMeta dorisDatabaseMeta = transMeta.findDatabase("doris");
-        execSQLMeta.setDatabaseMeta(dorisDatabaseMeta);
-        String deleteSql = "delete from DORIS_TABLE where dataease_uuid='?';".replace("DORIS_TABLE", dorisOutputTable);
-        execSQLMeta.setSql(deleteSql);
-        execSQLMeta.setExecutedEachInputRow(true);
-        execSQLMeta.setArguments(new String[]{"dataease_uuid"});
-        StepMeta execSQLStep = new StepMeta("ExecSQL", "ExecSQL", execSQLMeta);
-        execSQLStep.setLocation(600, 100);
-        execSQLStep.setDraw(true);
-        return execSQLStep;
-    }
-
     private static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" +
             "import java.util.List;\n" +
             "import java.io.File;\n" +
diff --git a/backend/src/main/resources/db/migration/V16__dataset_table_field.sql b/backend/src/main/resources/db/migration/V16__dataset_table_field.sql
new file mode 100644
index 0000000000..0db94c6dd4
--- /dev/null
+++ b/backend/src/main/resources/db/migration/V16__dataset_table_field.sql
@@ -0,0 +1,2 @@
+ALTER TABLE `dataset_table_field` ADD COLUMN `dataease_name`  varchar(255) NOT NULL COMMENT '字段名'  AFTER `name`;
+ALTER TABLE dataset_table_task_log CHANGE COLUMN `task_id` `task_id` VARCHAR(50) NULL COMMENT '任务ID' ;
\ No newline at end of file
diff --git a/backend/src/main/resources/generatorConfig.xml b/backend/src/main/resources/generatorConfig.xml
index ab97ce64ed..6aa551a59a 100644
--- a/backend/src/main/resources/generatorConfig.xml
+++ b/backend/src/main/resources/generatorConfig.xml
@@ -67,7 +67,7 @@
         <!--        <table tableName="datasource"/>-->
 <!--        <table tableName="sys_dict"/>-->
 <!--        <table tableName="sys_dict_item"/>-->
-        <table tableName="chart_view"/>
+        <table tableName="dataset_table_field"/>
 <!--        <table tableName="panel_design"/>-->