diff --git a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java index 9807a444bd..d1096a5cc7 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java @@ -22,6 +22,7 @@ import io.dataease.extensions.datasource.provider.Provider; import io.dataease.job.schedule.ExtractDataJob; import io.dataease.job.schedule.ScheduleManager; import io.dataease.utils.BeanUtils; +import io.dataease.utils.JsonUtil; import io.dataease.utils.LogUtil; import jakarta.annotation.Resource; import org.apache.commons.lang3.StringUtils; @@ -30,6 +31,7 @@ import org.quartz.JobKey; import org.quartz.TriggerKey; import org.springframework.stereotype.Component; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -245,17 +247,7 @@ public class DatasourceSyncManage { private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List tableFields) throws Exception { ExcelUtils excelUtils = new ExcelUtils(); List dataList = excelUtils.fetchDataList(datasourceRequest); - String engineTableName; - switch (extractType) { - case all_scope: - engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable())); - break; - default: - engineTableName = TableUtils.tableName(datasourceRequest.getTable()); - break; - } CoreDeEngine engine = engineManage.info(); - EngineRequest engineRequest = new EngineRequest(); engineRequest.setEngine(engine); EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType()); @@ -267,7 +259,7 @@ public class DatasourceSyncManage { totalPage = dataList.size() / pageNumber; } for (int page = 1; page <= totalPage; page++) { - engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields)); + engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields)); calciteProvider.exec(engineRequest); } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java index b8ae99d1ea..3ae742deb7 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java @@ -335,9 +335,6 @@ public class ExcelUtils { Double d = Double.valueOf(value); double eps = 1e-10; if (d - Math.floor(d) < eps) { - if (value.length() > 1 && value.startsWith("0")) { - return "TEXT"; - } return "LONG"; } else { return "DOUBLE"; @@ -440,6 +437,7 @@ public class ExcelUtils { tableFiled.setFieldType(null); tableFiled.setName(s); tableFiled.setOriginName(s); + tableFiled.setChecked(true); fields.add(tableFiled); } List data = new ArrayList<>(noModelDataListener.getData()); diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java index b0c9adedd1..e3e954bedb 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java @@ -40,12 +40,16 @@ public class H2EngineProvider extends EngineProvider { Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) { - String[] strings1 = new String[strings.length]; + int length = 0; + String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()]; for (int i = 0; i < strings.length; i++) { - if (StringUtils.isEmpty(strings[i])) { - strings1[i] = null; - } else { - strings1[i] = strings[i].replace("'", "\\'"); + if (tableFields.get(i).isChecked()) { + if (StringUtils.isEmpty(strings[i])) { + strings1[length] = null; + } else { + strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'"); + } + length++; } } values.append("('").append(String.join("','", Arrays.asList(strings1))) @@ -81,6 +85,9 @@ public class H2EngineProvider extends EngineProvider { StringBuilder columnFields = new StringBuilder("`"); StringBuilder key = new StringBuilder(); for (TableField tableField : tableFields) { + if (!tableField.isChecked()) { + continue; + } if (tableField.isPrimaryKey()) { key.append("`").append(tableField.getName()).append("`, "); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java index 5edc98e7d3..588b29a84f 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java @@ -47,19 +47,23 @@ public class MysqlEngineProvider extends EngineProvider { Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size(); for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) { - String[] strings1 = new String[strings.length]; + int length = 0; + String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()]; for (int i = 0; i < strings.length; i++) { - if (StringUtils.isEmpty(strings[i])) { - strings1[i] = null; - } else { - strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'"); + if (tableFields.get(i).isChecked()) { + if (StringUtils.isEmpty(strings[i])) { + strings1[length] = null; + } else { + strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'"); + } + length++; } } values.append("('").append(String.join("','", Arrays.asList(strings1))) .append("'),"); } - List keys = tableFields.stream().filter(TableField::isPrimaryKey).toList(); - List notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList(); + List keys = tableFields.stream().filter(tableField -> tableField.isPrimaryKey() && tableField.isChecked()).toList(); + List notKeys = tableFields.stream().filter(tableField -> tableField.isChecked() && !tableField.isPrimaryKey()).toList(); String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null"); if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) { insetSql = insetSql + " ON DUPLICATE KEY UPDATE "; @@ -101,6 +105,9 @@ public class MysqlEngineProvider extends EngineProvider { StringBuilder columnFields = new StringBuilder("`"); StringBuilder key = new StringBuilder(); for (TableField tableField : tableFields) { + if (!tableField.isChecked()) { + continue; + } if (tableField.isPrimaryKey()) { key.append("`").append(tableField.getName()).append("`, "); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java index 7f2d5cf47f..d1b4912d6f 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java @@ -408,14 +408,13 @@ public class DatasourceServer implements DatasourceApi { requestDatasource.setEnableDataFill(null); List sourceTables = ExcelUtils.getTables(sourceTableRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()); List tables = ExcelUtils.getTables(datasourceRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()); - if (dataSourceDTO.getEditType() == 0) { + if (Objects.equals(dataSourceDTO.getEditType(), replace)) { toCreateTables = tables; toDeleteTables = sourceTables.stream().filter(s -> tables.contains(s)).collect(Collectors.toList()); for (String deleteTable : toDeleteTables) { try { datasourceSyncManage.dropEngineTable(deleteTable); - } catch (Exception e) { - DEException.throwException("Failed to drop table " + deleteTable + ", " + e.getMessage()); + } catch (Exception ignore) { } } for (String toCreateTable : toCreateTables) { @@ -426,12 +425,16 @@ public class DatasourceServer implements DatasourceApi { DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage()); } } - datasourceSyncManage.extractExcelData(requestDatasource, "all_scope"); + commonThreadPool.addTask(() -> { + datasourceSyncManage.extractExcelData(requestDatasource, "all_scope"); + }); dataSourceManage.checkName(dataSourceDTO); ExcelUtils.mergeSheets(requestDatasource, sourceData); dataSourceManage.innerEdit(requestDatasource); } else { - datasourceSyncManage.extractExcelData(requestDatasource, "add_scope"); + commonThreadPool.addTask(() -> { + datasourceSyncManage.extractExcelData(requestDatasource, "add_scope"); + }); dataSourceManage.checkName(dataSourceDTO); dataSourceManage.innerEdit(requestDatasource); } @@ -767,6 +770,7 @@ public class DatasourceServer implements DatasourceApi { ExcelUtils excelUtils = new ExcelUtils(); ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file); + if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1 if (coreDatasource != null) { DatasourceRequest datasourceRequest = new DatasourceRequest(); @@ -776,15 +780,9 @@ public class DatasourceServer implements DatasourceApi { for (ExcelSheetData sheet : excelFileData.getSheets()) { for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) { if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) { - List newTableFields = deepCopy(sheet.getFields()); - newTableFields.sort((o1, o2) -> { - return o1.getName().compareTo(o2.getName()); - }); + List newTableFields = sheet.getFields(); datasourceRequest.setTable(datasetTableDTO.getTableName()); List oldTableFields = ExcelUtils.getTableFields(datasourceRequest); - oldTableFields.sort((o1, o2) -> { - return o1.getName().compareTo(o2.getName()); - }); if (isEqual(newTableFields, oldTableFields)) { sheet.setDeTableName(datasetTableDTO.getTableName()); excelSheetDataList.add(sheet); @@ -798,20 +796,29 @@ public class DatasourceServer implements DatasourceApi { excelFileData.setSheets(excelSheetDataList); } } else { + // 替换 if (coreDatasource != null) { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(transDTO(coreDatasource)); List datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); for (ExcelSheetData sheet : excelFileData.getSheets()) { + boolean find = false; for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) { if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) { + find = true; sheet.setDeTableName(datasetTableDTO.getTableName()); + datasourceRequest.setTable(datasetTableDTO.getTableName()); + List oldTableFields = ExcelUtils.getTableFields(datasourceRequest); + mergeFields(sheet.getFields(), oldTableFields); } } + if (!find) { + sheet.setNewSheet(true); + } } - } } + for (ExcelSheetData sheet : excelFileData.getSheets()) { for (int i = 0; i < sheet.getFields().size() - 1; i++) { for (int j = i + 1; j < sheet.getFields().size(); j++) { @@ -825,31 +832,42 @@ public class DatasourceServer implements DatasourceApi { } private boolean isEqual(List newTableFields, List oldTableFields) { - boolean isEqual = true; if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) { - isEqual = false; + return false; } - for (int i = 0; i < newTableFields.size(); i++) { - if (!newTableFields.get(i).getName().equals(oldTableFields.get(i).getName())) { - isEqual = false; - break; + newTableFields.forEach(tableField -> tableField.setChecked(false)); + for (TableField oldField : oldTableFields) { + if (!oldField.isChecked()) { + continue; } - if (!newTableFields.get(i).getFieldType().equals(oldTableFields.get(i).getFieldType())) { - if (oldTableFields.get(i).getFieldType().equals("TEXT")) { - continue; + boolean find = false; + for (TableField newField : newTableFields) { + if (oldField.getName().equals(newField.getName())) { + find = true; + newField.setChecked(oldField.isChecked()); + newField.setPrimaryKey(oldField.isPrimaryKey()); + newField.setLength(oldField.getLength()); + break; } - if (oldTableFields.get(i).getFieldType().equals("DOUBLE")) { - if (newTableFields.get(i).getFieldType().equals("LONG")) { - continue; - } - } - isEqual = false; - break; + } + if (!find) { + return find; } } + return true; + } - return isEqual; - + private void mergeFields(List oldFields, List newFields) { + oldFields.forEach(tableField -> tableField.setChecked(false)); + for (TableField newField : newFields) { + for (TableField oldField : oldFields) { + if (oldField.getName().equals(newField.getName())) { + newField.setChecked(oldField.isChecked()); + newField.setPrimaryKey(oldField.isPrimaryKey()); + newField.setLength(oldField.getLength()); + } + } + } } private boolean isCsv(String fileName) { diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue index f43363b95c..7dc1fba8c2 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestDraw.vue @@ -291,6 +291,7 @@ const saveItem = () => { for (let i = 0; i < apiItem.fields.length; i++) { if (apiItem.fields[i].primaryKey && !apiItem.fields[i].length) { ElMessage.error(t('datasource.primary_key_length') + apiItem.fields[i].name) + return } } } @@ -769,7 +770,7 @@ defineExpose({ step-strictly class="text-left edit-all-line" :min="1" - :max="4096" + :max="512" :placeholder="t('common.inputText')" controls-position="right" type="number" diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue b/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue index 11103feae7..b6858766e3 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue @@ -73,6 +73,8 @@ const { emitter } = useEmitt() const loading = ref(false) const columns = shallowRef([]) +const multipleSelection = shallowRef([]) +const multipleTable = ref() const defaultSheetObj = { tableName: ' ', @@ -115,6 +117,9 @@ const generateColumns = (arr: Field[]) => fieldType: ele.fieldType, dataKey: ele.originName, title: ele.name, + checked: ele.checked, + primaryKey: ele.primaryKey, + length: ele.length, width: 150, headerCellRenderer: ({ column }) => (
@@ -136,6 +141,8 @@ const handleNodeClick = data => { if (data.sheet) { Object.assign(sheetObj, data) columns.value = generateColumns(data.fields) + multipleSelection.value = columns.value.filter(item => item.checked) + currentMode.value = 'preview' } } @@ -181,10 +188,11 @@ const uploadSuccess = response => { param.value.name = response.data.excelLabel } tabList.value = response.data.sheets.map(ele => { - const { sheetId, tableName } = ele + const { sheetId, tableName, newSheet } = ele return { value: sheetId, - label: tableName + label: tableName, + newSheet: newSheet } }) state.excelData = [response.data] @@ -207,6 +215,23 @@ const saveExcelDs = (params, successCb, finallyCb) => { if (selectNode[i].changeFiled) { changeFiled = true } + for (let j = 0; j < selectNode[i].fields.length; j++) { + if ( + selectNode[i].fields[j].checked && + selectNode[i].fields[j].primaryKey && + !selectNode[i].fields[j].length + ) { + ElMessage({ + message: + t('datasource.primary_key_length') + + selectNode[i].excelLabel + + ': ' + + selectNode[i].fields[j].name, + type: 'error' + }) + return + } + } selectedSheet.push(selectNode[i]) sheetFileMd5.push(selectNode[i].fieldsMd5) } @@ -405,11 +430,90 @@ const appendReplaceExcel = response => { } const status = ref(false) +const initMultipleTable = ref(false) const currentMode = ref('preview') const refreshData = () => { currentMode.value = 'preview' } +const lengthChange = val => { + const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value) + sheet.fields.forEach(row => { + if (row.originName === val.dataKey) { + row.length = val.length + } + }) +} +const primaryKeyChange = val => { + const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value) + sheet.fields.forEach(row => { + if (row.originName === val.dataKey) { + row.primaryKey = val.primaryKey + } + }) +} + +const handleSelectionChange = val => { + if (!initMultipleTable.value) { + multipleSelection.value = val + multipleSelection.value.forEach(row => { + row.checked = true + }) + columns.value.forEach(row => { + let item + for (let i = 0; i < multipleSelection.value.length; i++) { + if (row.dataKey === multipleSelection.value[i].dataKey) { + item = multipleSelection.value[i] + } + } + if (item) { + row.checked = item.checked + } else { + row.checked = false + } + }) + + const sheet = state.excelData[0]?.sheets.find(ele => ele.sheetId === activeTab.value) + sheet.fields.forEach(row => { + let item + for (let i = 0; i < multipleSelection.value.length; i++) { + if (row.originName === multipleSelection.value[i].dataKey) { + item = multipleSelection.value[i] + } + } + if (item) { + row.checked = item.checked + } else { + row.checked = false + } + }) + } +} + +const disabledFieldLength = item => { + if (!item.checked) { + return true + } + if (item.fieldType !== 'TEXT') { + return true + } +} + +const changeCurrentMode = val => { + currentMode.value = val + if (val === 'select') { + nextTick(() => { + initMultipleTable.value = true + for (let i = 0; i < columns.value.length; i++) { + if (columns.value[i].checked) { + multipleTable?.value?.toggleRowSelection(columns.value[i], true) + } + } + initMultipleTable.value = false + }) + } +} + const uploadStatus = val => { status.value = val } @@ -533,16 +637,16 @@ defineExpose({ >
-
+
{{ t('chart.data_preview') }} @@ -562,7 +666,7 @@ defineExpose({ - + @@ -592,6 +703,44 @@ defineExpose({
+ + + + + +
diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelSheetData.java b/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelSheetData.java index ce68a39c98..370b0e77a0 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelSheetData.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelSheetData.java @@ -21,5 +21,6 @@ public class ExcelSheetData { private String sheetId; private String sheetExcelId; private List> jsonArray; + private boolean newSheet; }