dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 抽取 excel 数据

Browse Source
This commit is contained in:
taojinlong 2021-05-07 16:01:04 +08:00
parent ac938c281b
commit 260d09a1a0
9 changed files with 225 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
View File

@ -16,6 +16,8 @@ public class DatasetTableField implements Serializable {
private String name;
private String dataeaseName;
private String type;
private Integer size;

70
backend/src/main/java/io/dataease/base/domain/DatasetTableFieldExample.java
View File

@ -384,6 +384,76 @@ public class DatasetTableFieldExample {
return (Criteria) this;
}
public Criteria andDataeaseNameIsNull() {
addCriterion("dataease_name is null");
return (Criteria) this;
}
public Criteria andDataeaseNameIsNotNull() {
addCriterion("dataease_name is not null");
return (Criteria) this;
}
public Criteria andDataeaseNameEqualTo(String value) {
addCriterion("dataease_name =", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotEqualTo(String value) {
addCriterion("dataease_name <>", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameGreaterThan(String value) {
addCriterion("dataease_name >", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameGreaterThanOrEqualTo(String value) {
addCriterion("dataease_name >=", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLessThan(String value) {
addCriterion("dataease_name <", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLessThanOrEqualTo(String value) {
addCriterion("dataease_name <=", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameLike(String value) {
addCriterion("dataease_name like", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotLike(String value) {
addCriterion("dataease_name not like", value, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameIn(List<String> values) {
addCriterion("dataease_name in", values, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotIn(List<String> values) {
addCriterion("dataease_name not in", values, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameBetween(String value1, String value2) {
addCriterion("dataease_name between", value1, value2, "dataeaseName");
return (Criteria) this;
}
public Criteria andDataeaseNameNotBetween(String value1, String value2) {
addCriterion("dataease_name not between", value1, value2, "dataeaseName");
return (Criteria) this;
}
public Criteria andTypeIsNull() {
addCriterion("`type` is null");
return (Criteria) this;

31
backend/src/main/java/io/dataease/base/mapper/DatasetTableFieldMapper.xml
View File

@ -6,6 +6,7 @@
<result column="table_id" jdbcType="VARCHAR" property="tableId" />
<result column="origin_name" jdbcType="VARCHAR" property="originName" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="dataease_name" jdbcType="VARCHAR" property="dataeaseName" />
<result column="type" jdbcType="VARCHAR" property="type" />
<result column="size" jdbcType="INTEGER" property="size" />
<result column="de_type" jdbcType="INTEGER" property="deType" />
@ -72,8 +73,8 @@
</where>
</sql>
<sql id="Base_Column_List">
id, table_id, origin_name, `name`, `type`, `size`, de_type, `checked`, column_index,
last_sync_time
id, table_id, origin_name, `name`, dataease_name, `type`, `size`, de_type, `checked`,
column_index, last_sync_time
</sql>
<select id="selectByExample" parameterType="io.dataease.base.domain.DatasetTableFieldExample" resultMap="BaseResultMap">
select
@ -107,13 +108,13 @@
</delete>
<insert id="insert" parameterType="io.dataease.base.domain.DatasetTableField">
insert into dataset_table_field (id, table_id, origin_name,
`name`, `type`, `size`, de_type,
`checked`, column_index, last_sync_time
)
`name`, dataease_name, `type`,
`size`, de_type, `checked`,
column_index, last_sync_time)
values (#{id,jdbcType=VARCHAR}, #{tableId,jdbcType=VARCHAR}, #{originName,jdbcType=VARCHAR},
#{name,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR}, #{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER},
#{checked,jdbcType=BIT}, #{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT}
)
#{name,jdbcType=VARCHAR}, #{dataeaseName,jdbcType=VARCHAR}, #{type,jdbcType=VARCHAR},
#{size,jdbcType=INTEGER}, #{deType,jdbcType=INTEGER}, #{checked,jdbcType=BIT},
#{columnIndex,jdbcType=INTEGER}, #{lastSyncTime,jdbcType=BIGINT})
</insert>
<insert id="insertSelective" parameterType="io.dataease.base.domain.DatasetTableField">
insert into dataset_table_field
@ -130,6 +131,9 @@
<if test="name != null">
`name`,
</if>
<if test="dataeaseName != null">
dataease_name,
</if>
<if test="type != null">
`type`,
</if>
@ -162,6 +166,9 @@
<if test="name != null">
#{name,jdbcType=VARCHAR},
</if>
<if test="dataeaseName != null">
#{dataeaseName,jdbcType=VARCHAR},
</if>
<if test="type != null">
#{type,jdbcType=VARCHAR},
</if>
@ -203,6 +210,9 @@
<if test="record.name != null">
`name` = #{record.name,jdbcType=VARCHAR},
</if>
<if test="record.dataeaseName != null">
dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
</if>
<if test="record.type != null">
`type` = #{record.type,jdbcType=VARCHAR},
</if>
@ -232,6 +242,7 @@
table_id = #{record.tableId,jdbcType=VARCHAR},
origin_name = #{record.originName,jdbcType=VARCHAR},
`name` = #{record.name,jdbcType=VARCHAR},
dataease_name = #{record.dataeaseName,jdbcType=VARCHAR},
`type` = #{record.type,jdbcType=VARCHAR},
`size` = #{record.size,jdbcType=INTEGER},
de_type = #{record.deType,jdbcType=INTEGER},
@ -254,6 +265,9 @@
<if test="name != null">
`name` = #{name,jdbcType=VARCHAR},
</if>
<if test="dataeaseName != null">
dataease_name = #{dataeaseName,jdbcType=VARCHAR},
</if>
<if test="type != null">
`type` = #{type,jdbcType=VARCHAR},
</if>
@ -280,6 +294,7 @@
set table_id = #{tableId,jdbcType=VARCHAR},
origin_name = #{originName,jdbcType=VARCHAR},
`name` = #{name,jdbcType=VARCHAR},
dataease_name = #{dataeaseName,jdbcType=VARCHAR},
`type` = #{type,jdbcType=VARCHAR},
`size` = #{size,jdbcType=INTEGER},
de_type = #{deType,jdbcType=INTEGER},

41
backend/src/main/java/io/dataease/commons/utils/Md5Utils.java Normal file
View File

@ -0,0 +1,41 @@
package io.dataease.commons.utils;
import java.util.List;
import java.io.File;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class Md5Utils {
private static final char[] HEX_DIGITS = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
private static final String UTF_8 = "UTF-8";
public static String md5(String src) {
return md5(src, UTF_8);
}
public static String md5(String src, String charset) {
try {
byte[] strTemp = charset == null || charset.equals("") ? src.getBytes() : src.getBytes(charset);
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(strTemp);
byte[] md = mdTemp.digest();
int j = md.length;
char[] str = new char[j * 2];
int k = 0;
for (byte byte0 : md) {
str[k++] = HEX_DIGITS[byte0 >>> 4 & 0xf];
str[k++] = HEX_DIGITS[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
throw new RuntimeException("MD5 encrypt error:", e);
}
}
}

2
backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java
View File

@ -1,5 +1,5 @@
package io.dataease.datasource.constants;
public enum DatasourceTypes {
mysql, sqlServer
mysql, sqlServer, excel
}

15
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -8,6 +8,8 @@ import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.commons.utils.Md5Utils;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.dto.TableFiled;
@ -54,6 +56,10 @@ public class DataSetTableService {
@Resource
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private CommonThreadPool commonThreadPool;
@Resource
private ExtractDataService extractDataService;
@Resource
private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper;
@Value("${upload.file.path}")
private String path;
@ -79,6 +85,9 @@ public class DataSetTableService {
// 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
if (insert == 1) {
saveTableField(datasetTable);
commonThreadPool.addTask(()->{
extractDataService.extractData(datasetTable.getId(), null, "all_scope");
});
}
} else {
int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
@ -330,6 +339,11 @@ public class DataSetTableService {
datasetTableField.setTableId(datasetTable.getId());
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
if(StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")){
datasetTableField.setDataeaseName("C_" + Md5Utils.md5(filed.getFieldName()));
}else {
datasetTableField.setDataeaseName(filed.getFieldName());
}
datasetTableField.setType(filed.getFieldType());
if (ObjectUtils.isEmpty(ds)) {
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
@ -560,6 +574,7 @@ public class DataSetTableService {
tableFiled.setFieldName(readCell(row.getCell(j)));
tableFiled.setRemarks(readCell(row.getCell(j)));
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
fields.add(tableFiled);
} else {
r[j] = readCell(row.getCell(j));

124
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -39,6 +39,9 @@ import org.pentaho.di.repository.filerep.KettleFileRepository;
import org.pentaho.di.trans.TransHopMeta;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.excelinput.ExcelInputField;
import org.pentaho.di.trans.steps.excelinput.ExcelInputMeta;
import org.pentaho.di.trans.steps.excelinput.SpreadSheetType;
import org.pentaho.di.trans.steps.sql.ExecSQLMeta;
import org.pentaho.di.trans.steps.tableinput.TableInputMeta;
import org.pentaho.di.trans.steps.textfileoutput.TextFileField;
@ -87,7 +90,7 @@ public class ExtractDataService {
@Value("${carte.passwd:cluster}")
private String passwd;
private static String creatTableSql = "CREATE TABLE IF NOT EXISTS TABLE_NAME" +
private static String creatTableSql = "CREATE TABLE IF NOT EXISTS `TABLE_NAME`" +
"Column_Fields" +
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
@ -97,28 +100,28 @@ public class ExtractDataService {
"rm -rf %s\n";
private String createDorisTablColumnSql( List<DatasetTableField> datasetTableFields){
String Column_Fields = "dataease_uuid varchar(50),";
String Column_Fields = "dataease_uuid varchar(50), `";
for (DatasetTableField datasetTableField : datasetTableFields) {
Column_Fields = Column_Fields + datasetTableField.getOriginName() + " ";
Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` ";
switch (datasetTableField.getDeType()){
case 0:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 1:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 2:
Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`";
break;
case 3:
Column_Fields = Column_Fields + "DOUBLE" + ",";
Column_Fields = Column_Fields + "DOUBLE" + ",`";
break;
default:
Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",";
break;
}
}
Column_Fields = Column_Fields.substring(0, Column_Fields.length() -1 );
Column_Fields = Column_Fields.substring(0, Column_Fields.length() -2);
Column_Fields = "(" + Column_Fields + ")\n";
return Column_Fields;
}
@ -129,7 +132,6 @@ public class ExtractDataService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource);
datasourceRequest.setQuery(creatTableSql.replace("TABLE_NAME", dorisTableName).replace("Column_Fields", dorisTablColumnSql));
System.out.println(datasourceRequest.getQuery());
jdbcProvider.exec(datasourceRequest);
}
@ -145,21 +147,30 @@ public class ExtractDataService {
public void extractData(String datasetTableId, String taskId, String type) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
UpdateType updateType = UpdateType.valueOf(type);
DatasetTable datasetTable = null;
Datasource datasource = new Datasource();
try {
DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
Datasource datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
datasetTable = dataSetTableService.get(datasetTableId);
if(StringUtils.isNotEmpty(datasetTable.getDataSourceId())){
datasource = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
}else {
datasource.setType(datasetTable.getType());
}
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
datasetTableFields.sort((o1, o2) -> {
if (o1.getOriginName() == null) {
if (o1.getColumnIndex() == null) {
return -1;
}
if (o2.getOriginName() == null) {
if (o2.getColumnIndex() == null) {
return 1;
}
return o1.getOriginName().compareTo(o2.getOriginName());
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
for (DatasetTableField datasetTableField : datasetTableFields) {
System.out.println(new Gson().toJson(datasetTableField));
}
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
String dorisTablColumnSql = createDorisTablColumnSql(datasetTableFields);
switch (updateType) {
// 全量更新
@ -168,8 +179,8 @@ public class ExtractDataService {
// TODO before: check doris table column type
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
generateTransFile("all_scope", datasetTable, datasource, tableName, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList())));
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
@ -196,9 +207,7 @@ public class ExtractDataService {
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
//TODO sql column
generateTransFile("incremental_add", datasetTable, datasource, tableName, datasetTableFields, sql);
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_add");
}
@ -207,9 +216,7 @@ public class ExtractDataService {
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())) {
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, dataSetTaskLogDTOS.get(0).getStartTime().toString()
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()));
//TODO sql column
;
generateTransFile("incremental_delete", datasetTable, datasource, tableName, datasetTableFields, sql);
generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource));
extractData(datasetTable, "incremental_delete");
}
@ -395,25 +402,11 @@ public class ExtractDataService {
return String.join(",", datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).collect(Collectors.toList()));
}
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, String table, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
private void generateTransFile(String extractType, DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String selectSQL) throws Exception {
TransMeta transMeta = new TransMeta();
String dorisOutputTable = null;
DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasource.getType());
DatabaseMeta dataMeta = null;
switch (datasourceType) {
case mysql:
MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
transMeta.addDatabase(dataMeta);
break;
default:
break;
}
Datasource dorisDatasource = (Datasource)CommonBeanFactory.getBean("DorisDatasource");
MysqlConfigration dorisConfigration = new Gson().fromJson(dorisDatasource.getConfiguration(), MysqlConfigration.class);
DatabaseMeta dorisDataMeta = new DatabaseMeta("doris", "MYSQL", "Native", dorisConfigration.getHost(), dorisConfigration.getDataBase(), dorisConfigration.getPort().toString(), dorisConfigration.getUsername(), dorisConfigration.getPassword());
transMeta.addDatabase(dorisDataMeta);
StepMeta inputStep = null;
StepMeta outputStep = null;
StepMeta udjcStep = null;
@ -421,11 +414,28 @@ public class ExtractDataService {
TransHopMeta hi2 = null;
String transName = null;
switch (datasourceType) {
case mysql:
MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class);
dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword());
transMeta.addDatabase(dataMeta);
if(extractType.equalsIgnoreCase("all_scope")){
String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), tableName, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).toArray(String[]::new));
}
inputStep = inputStep(transMeta, selectSQL);
break;
case excel:
String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData();
inputStep = excelInputStep(filePath, datasetTableFields);
default:
break;
}
switch (extractType) {
case "all_scope":
transName = "trans_" + datasetTable.getId();
dorisOutputTable = DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()));
selectSQL = dataSetTableService.createQuerySQL(datasource.getType(), table, datasetTableFields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new));
transMeta.setName(transName);
break;
case "incremental_add":
@ -442,7 +452,6 @@ public class ExtractDataService {
break;
}
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields);
outputStep = outputStep(dorisOutputTable);
hi1 = new TransHopMeta(inputStep, udjcStep);
@ -469,6 +478,27 @@ public class ExtractDataService {
return fromStep;
}
private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields){
ExcelInputMeta excelInputMeta = new ExcelInputMeta();
excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
excelInputMeta.setPassword("Encrypted");
excelInputMeta.setFileName(new String[]{filePath});
excelInputMeta.setStartsWithHeader(true);
ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];
for (int i=0; i<datasetTableFields.size();i++) {
ExcelInputField field = new ExcelInputField();
field.setName(datasetTableFields.get(i).getOriginName());
field.setType("String");
fields[i] = field;
}
excelInputMeta.setField(fields);
StepMeta fromStep = new StepMeta("ExcelInput", "Data Input", excelInputMeta);
fromStep.setDraw(true);
fromStep.setLocation(100, 100);
return fromStep;
}
private StepMeta outputStep(String dorisOutputTable){
TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
textFileOutputMeta.setEncoding("UTF-8");
@ -502,20 +532,6 @@ public class ExtractDataService {
return userDefinedJavaClassStep;
}
private StepMeta execSqlStep(TransMeta transMeta, String dorisOutputTable, List<DatasetTableField>datasetTableFields){
ExecSQLMeta execSQLMeta = new ExecSQLMeta();
DatabaseMeta dorisDatabaseMeta = transMeta.findDatabase("doris");
execSQLMeta.setDatabaseMeta(dorisDatabaseMeta);
String deleteSql = "delete from DORIS_TABLE where dataease_uuid='?';".replace("DORIS_TABLE", dorisOutputTable);
execSQLMeta.setSql(deleteSql);
execSQLMeta.setExecutedEachInputRow(true);
execSQLMeta.setArguments(new String[]{"dataease_uuid"});
StepMeta execSQLStep = new StepMeta("ExecSQL", "ExecSQL", execSQLMeta);
execSQLStep.setLocation(600, 100);
execSQLStep.setDraw(true);
return execSQLStep;
}
private static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" +
"import java.util.List;\n" +
"import java.io.File;\n" +

2
backend/src/main/resources/db/migration/V16__dataset_table_field.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE `dataset_table_field` ADD COLUMN `dataease_name` varchar(255) NOT NULL COMMENT '字段名' AFTER `name`;
ALTER TABLE dataset_table_task_log CHANGE COLUMN `task_id` `task_id` VARCHAR(50) NULL COMMENT '任务ID' ;

2
backend/src/main/resources/generatorConfig.xml
View File

@ -67,7 +67,7 @@
<!-- <table tableName="datasource"/>-->
<!-- <table tableName="sys_dict"/>-->
<!-- <table tableName="sys_dict_item"/>-->
<table tableName="chart_view"/>
<table tableName="dataset_table_field"/>
<!-- <table tableName="panel_design"/>-->