dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 上传excel支持列选择

Browse Source
This commit is contained in:
taojinlong 2024-11-26 17:37:28 +08:00
parent b73550652b
commit 718c0cd808
7 changed files with 236 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java
View File

@ -22,6 +22,7 @@ import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.job.schedule.ExtractDataJob;
import io.dataease.job.schedule.ScheduleManager;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import io.dataease.utils.LogUtil;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
@ -30,6 +31,7 @@ import org.quartz.JobKey;
import org.quartz.TriggerKey;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -245,17 +247,7 @@ public class DatasourceSyncManage {
private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
ExcelUtils excelUtils = new ExcelUtils();
List<String[]> dataList = excelUtils.fetchDataList(datasourceRequest);
String engineTableName;
switch (extractType) {
case all_scope:
engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable()));
break;
default:
engineTableName = TableUtils.tableName(datasourceRequest.getTable());
break;
}
CoreDeEngine engine = engineManage.info();
EngineRequest engineRequest = new EngineRequest();
engineRequest.setEngine(engine);
EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType());
@ -267,7 +259,7 @@ public class DatasourceSyncManage {
totalPage = dataList.size() / pageNumber;
}
for (int page = 1; page <= totalPage; page++) {
engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields));
engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields));
calciteProvider.exec(engineRequest);
}
}

1
core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java
View File

@ -440,6 +440,7 @@ public class ExcelUtils {
tableFiled.setFieldType(null);
tableFiled.setName(s);
tableFiled.setOriginName(s);
tableFiled.setChecked(true);
fields.add(tableFiled);
}
List<String[]> data = new ArrayList<>(noModelDataListener.getData());

17
core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java
View File

@ -40,12 +40,16 @@ public class H2EngineProvider extends EngineProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
String[] strings1 = new String[strings.length];
int length = 0;
String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()];
for (int i = 0; i < strings.length; i++) {
if (StringUtils.isEmpty(strings[i])) {
strings1[i] = null;
} else {
strings1[i] = strings[i].replace("'", "\\'");
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
strings1[length] = null;
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}
length++;
}
}
values.append("('").append(String.join("','", Arrays.asList(strings1)))
@ -81,6 +85,9 @@ public class H2EngineProvider extends EngineProvider {
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
if (!tableField.isChecked()) {
continue;
}
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}

21
core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java
View File

@ -47,19 +47,23 @@ public class MysqlEngineProvider extends EngineProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
String[] strings1 = new String[strings.length];
int length = 0;
String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()];
for (int i = 0; i < strings.length; i++) {
if (StringUtils.isEmpty(strings[i])) {
strings1[i] = null;
} else {
strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
strings1[length] = null;
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}
length++;
}
}
values.append("('").append(String.join("','", Arrays.asList(strings1)))
.append("'),");
}
List<TableField> keys = tableFields.stream().filter(TableField::isPrimaryKey).toList();
List<TableField> notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList();
List<TableField> keys = tableFields.stream().filter(tableField -> tableField.isPrimaryKey() && tableField.isChecked()).toList();
List<TableField> notKeys = tableFields.stream().filter(tableField -> tableField.isChecked() && !tableField.isPrimaryKey()).toList();
String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) {
insetSql = insetSql + " ON DUPLICATE KEY UPDATE ";
@ -101,6 +105,9 @@ public class MysqlEngineProvider extends EngineProvider {
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
if (!tableField.isChecked()) {
continue;
}
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}

80
core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
View File

@ -408,14 +408,13 @@ public class DatasourceServer implements DatasourceApi {
requestDatasource.setEnableDataFill(null);
List<String> sourceTables = ExcelUtils.getTables(sourceTableRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList());
List<String> tables = ExcelUtils.getTables(datasourceRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList());
if (dataSourceDTO.getEditType() == 0) {
if (Objects.equals(dataSourceDTO.getEditType(), replace)) {
toCreateTables = tables;
toDeleteTables = sourceTables.stream().filter(s -> tables.contains(s)).collect(Collectors.toList());
for (String deleteTable : toDeleteTables) {
try {
datasourceSyncManage.dropEngineTable(deleteTable);
} catch (Exception e) {
DEException.throwException("Failed to drop table " + deleteTable + ", " + e.getMessage());
} catch (Exception ignore) {
}
}
for (String toCreateTable : toCreateTables) {
@ -426,12 +425,16 @@ public class DatasourceServer implements DatasourceApi {
DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage());
}
}
datasourceSyncManage.extractExcelData(requestDatasource, "all_scope");
commonThreadPool.addTask(() -> {
datasourceSyncManage.extractExcelData(requestDatasource, "all_scope");
});
dataSourceManage.checkName(dataSourceDTO);
ExcelUtils.mergeSheets(requestDatasource, sourceData);
dataSourceManage.innerEdit(requestDatasource);
} else {
datasourceSyncManage.extractExcelData(requestDatasource, "add_scope");
commonThreadPool.addTask(() -> {
datasourceSyncManage.extractExcelData(requestDatasource, "add_scope");
});
dataSourceManage.checkName(dataSourceDTO);
dataSourceManage.innerEdit(requestDatasource);
}
@ -767,6 +770,7 @@ public class DatasourceServer implements DatasourceApi {
ExcelUtils excelUtils = new ExcelUtils();
ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file);
if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -776,15 +780,9 @@ public class DatasourceServer implements DatasourceApi {
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) {
if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) {
List<TableField> newTableFields = deepCopy(sheet.getFields());
newTableFields.sort((o1, o2) -> {
return o1.getName().compareTo(o2.getName());
});
List<TableField> newTableFields = sheet.getFields();
datasourceRequest.setTable(datasetTableDTO.getTableName());
List<TableField> oldTableFields = ExcelUtils.getTableFields(datasourceRequest);
oldTableFields.sort((o1, o2) -> {
return o1.getName().compareTo(o2.getName());
});
if (isEqual(newTableFields, oldTableFields)) {
sheet.setDeTableName(datasetTableDTO.getTableName());
excelSheetDataList.add(sheet);
@ -798,20 +796,29 @@ public class DatasourceServer implements DatasourceApi {
excelFileData.setSheets(excelSheetDataList);
}
} else {
// 替换
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
for (ExcelSheetData sheet : excelFileData.getSheets()) {
boolean find = false;
for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) {
if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) {
find = true;
sheet.setDeTableName(datasetTableDTO.getTableName());
datasourceRequest.setTable(datasetTableDTO.getTableName());
List<TableField> oldTableFields = ExcelUtils.getTableFields(datasourceRequest);
mergeFields(sheet.getFields(), oldTableFields);
}
}
if (!find) {
sheet.setNewSheet(true);
}
}
}
}
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (int i = 0; i < sheet.getFields().size() - 1; i++) {
for (int j = i + 1; j < sheet.getFields().size(); j++) {
@ -825,31 +832,42 @@ public class DatasourceServer implements DatasourceApi {
}
private boolean isEqual(List<TableField> newTableFields, List<TableField> oldTableFields) {
boolean isEqual = true;
if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) {
isEqual = false;
return false;
}
for (int i = 0; i < newTableFields.size(); i++) {
if (!newTableFields.get(i).getName().equals(oldTableFields.get(i).getName())) {
isEqual = false;
break;
newTableFields.forEach(tableField -> tableField.setChecked(false));
for (TableField oldField : oldTableFields) {
if (!oldField.isChecked()) {
continue;
}
if (!newTableFields.get(i).getFieldType().equals(oldTableFields.get(i).getFieldType())) {
if (oldTableFields.get(i).getFieldType().equals("TEXT")) {
continue;
boolean find = false;
for (TableField newField : newTableFields) {
if (oldField.getName().equals(newField.getName())) {
find = true;
newField.setChecked(oldField.isChecked());
newField.setPrimaryKey(oldField.isPrimaryKey());
newField.setLength(oldField.getLength());
break;
}
if (oldTableFields.get(i).getFieldType().equals("DOUBLE")) {
if (newTableFields.get(i).getFieldType().equals("LONG")) {
continue;
}
}
isEqual = false;
break;
}
if (!find) {
return find;
}
}
return true;
}
return isEqual;
private void mergeFields(List<TableField> oldFields, List<TableField> newFields) {
oldFields.forEach(tableField -> tableField.setChecked(false));
for (TableField newField : newFields) {
for (TableField oldField : oldFields) {
if (oldField.getName().equals(newField.getName())) {
newField.setChecked(oldField.isChecked());
newField.setPrimaryKey(oldField.isPrimaryKey());
newField.setLength(oldField.getLength());
}
}
}
}
private boolean isCsv(String fileName) {

163
core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue
View File

@ -73,6 +73,8 @@ const { emitter } = useEmitt()
const loading = ref(false)
const columns = shallowRef([])
const multipleSelection = shallowRef([])
const multipleTable = ref()
const defaultSheetObj = {
tableName: ' ',
@ -115,6 +117,9 @@ const generateColumns = (arr: Field[]) =>
fieldType: ele.fieldType,
dataKey: ele.originName,
title: ele.name,
checked: ele.checked,
primaryKey: ele.primaryKey,
length: ele.length,
width: 150,
headerCellRenderer: ({ column }) => (
<div class="flex-align-center icon">
@ -136,6 +141,8 @@ const handleNodeClick = data => {
if (data.sheet) {
Object.assign(sheetObj, data)
columns.value = generateColumns(data.fields)
multipleSelection.value = columns.value.filter(item => item.checked)
currentMode.value = 'preview'
}
}
@ -181,10 +188,11 @@ const uploadSuccess = response => {
param.value.name = response.data.excelLabel
}
tabList.value = response.data.sheets.map(ele => {
const { sheetId, tableName } = ele
const { sheetId, tableName, newSheet } = ele
return {
value: sheetId,
label: tableName
label: tableName,
newSheet: newSheet
}
})
state.excelData = [response.data]
@ -207,6 +215,23 @@ const saveExcelDs = (params, successCb, finallyCb) => {
if (selectNode[i].changeFiled) {
changeFiled = true
}
for (let j = 0; j < selectNode[i].fields.length; j++) {
if (
selectNode[i].fields[j].checked &&
selectNode[i].fields[j].primaryKey &&
!selectNode[i].fields[j].length
) {
ElMessage({
message:
t('datasource.primary_key_length') +
selectNode[i].excelLabel +
': ' +
selectNode[i].fields[j].name,
type: 'error'
})
return
}
}
selectedSheet.push(selectNode[i])
sheetFileMd5.push(selectNode[i].fieldsMd5)
}
@ -405,11 +430,90 @@ const appendReplaceExcel = response => {
}
const status = ref(false)
const initMultipleTable = ref(false)
const currentMode = ref('preview')
const refreshData = () => {
currentMode.value = 'preview'
}
const lengthChange = val => {
const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value)
sheet.fields.forEach(row => {
if (row.originName === val.dataKey) {
row.length = val.length
}
})
}
const primaryKeyChange = val => {
const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value)
sheet.fields.forEach(row => {
if (row.originName === val.dataKey) {
row.primaryKey = val.primaryKey
}
})
}
const handleSelectionChange = val => {
if (!initMultipleTable.value) {
multipleSelection.value = val
multipleSelection.value.forEach(row => {
row.checked = true
})
columns.value.forEach(row => {
let item
for (let i = 0; i < multipleSelection.value.length; i++) {
if (row.dataKey === multipleSelection.value[i].dataKey) {
item = multipleSelection.value[i]
}
}
if (item) {
row.checked = item.checked
} else {
row.checked = false
}
})
const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value)
sheet.fields.forEach(row => {
let item
for (let i = 0; i < multipleSelection.value.length; i++) {
if (row.originName === multipleSelection.value[i].dataKey) {
item = multipleSelection.value[i]
}
}
if (item) {
row.checked = item.checked
} else {
row.checked = false
}
})
}
}
const disabledFieldLength = item => {
if (!item.checked) {
return true
}
if (item.fieldType !== 'TEXT') {
return true
}
}
const changeCurrentMode = val => {
currentMode.value = val
if (val === 'select') {
nextTick(() => {
initMultipleTable.value = true
for (let i = 0; i < columns.value.length; i++) {
if (columns.value[i].checked) {
multipleTable?.value?.toggleRowSelection(columns.value[i], true)
}
}
initMultipleTable.value = false
})
}
}
const uploadStatus = val => {
status.value = val
}
@ -533,16 +637,16 @@ defineExpose({
></SheetTabs>
<div class="table-select_mode">
<div class="btn-select">
<div class="btn-select" v-if="param.id === '0' || sheetObj.newSheet">
<el-button
@click="currentMode = 'preview'"
@click="changeCurrentMode('preview')"
:class="[currentMode === 'preview' && 'is-active']"
text
>
{{ t('chart.data_preview') }}
</el-button>
<el-button
@click="currentMode = 'select'"
@click="changeCurrentMode('select')"
:class="[currentMode === 'select' && 'is-active']"
text
>
@ -562,7 +666,7 @@ defineExpose({
<el-auto-resizer v-if="currentMode === 'preview'">
<template #default="{ height, width }">
<el-table-v2
:columns="columns"
:columns="multipleSelection"
header-class="excel-header-cell"
:data="sheetObj.jsonArray"
:width="width"
@ -571,7 +675,14 @@ defineExpose({
/>
</template>
</el-auto-resizer>
<el-table header-class="header-cell" v-else :data="columns" style="width: 100%">
<el-table
header-class="header-cell"
v-else
ref="multipleTable"
:data="columns"
style="width: 100%"
@selection-change="handleSelectionChange"
>
<el-table-column type="selection" width="55" />
<el-table-column :label="t('data_set.field_name')">
<template #default="scope">{{ scope.row.title }}</template>
@ -592,6 +703,44 @@ defineExpose({
</div>
</template>
</el-table-column>
<el-table-column
prop="length"
:label="t('datasource.length')"
v-if="param.id === '0' || sheetObj.newSheet"
>
<template #default="scope">
<el-input-number
:disabled="disabledFieldLength(scope.row)"
v-model="scope.row.length"
autocomplete="off"
step-strictly
class="text-left edit-all-line"
:min="1"
:max="512"
:placeholder="t('common.inputText')"
controls-position="right"
type="number"
@change="lengthChange(scope.row)"
/>
</template>
</el-table-column>
<el-table-column
prop="primaryKey"
class-name="checkbox-table"
:label="t('datasource.set_key')"
width="100"
v-if="param.id === '0' || sheetObj.newSheet"
>
<template #default="scope">
<el-checkbox
:key="scope.row.dataKey"
v-model="scope.row.primaryKey"
:disabled="!scope.row.checked"
@change="primaryKeyChange(scope.row)"
>
</el-checkbox>
</template>
</el-table-column>
</el-table>
</div>
</template>

1
sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelSheetData.java
View File

@ -21,5 +21,6 @@ public class ExcelSheetData {
private String sheetId;
private String sheetExcelId;
private List<Map<String, Object>> jsonArray;
private boolean newSheet;
}