From 4a9009dee11dbf10550b52fb68648081bf6695bd Mon Sep 17 00:00:00 2001 From: taojinlong Date: Thu, 12 Aug 2021 19:23:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E5=AF=BC=E5=85=A5excel=E7=9A=84=E5=A4=9Asheet=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/utils/ExcelReaderUtil.java | 10 +- .../commons/utils/ExcelXlsReader.java | 10 +- .../commons/utils/ExcelXlsxReader.java | 2 +- .../dataset/DataSetTableController.java | 13 +- .../request/dataset/DataSetTableRequest.java | 3 + .../dto/dataset/DataTableInfoDTO.java | 2 +- .../dataease/dto/dataset/ExcelSheetData.java | 10 +- .../service/dataset/DataSetTableService.java | 268 +++++++++---- .../service/dataset/ExtractDataService.java | 28 +- frontend/src/lang/en.js | 3 +- frontend/src/lang/tw.js | 3 +- frontend/src/lang/zh.js | 3 +- frontend/src/views/dataset/add/AddExcel.vue | 373 +++++++++++------- 13 files changed, 480 insertions(+), 248 deletions(-) diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java index 2bfddcf3de..0881531648 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java @@ -1,8 +1,6 @@ package io.dataease.commons.utils; import com.google.gson.Gson; import io.dataease.datasource.dto.TableFiled; - -import java.io.FileInputStream; import java.io.InputStream; import java.util.List; @@ -46,7 +44,7 @@ public class ExcelReaderUtil { ExcelXlsReader excelXls=new ExcelXlsReader(); excelXls.process(inputStream); System.out.println(excelXls.totalSheets.size()); - System.out.println(excelXls.totalSheets.get(0).getSheetName()); + System.out.println(excelXls.totalSheets.get(0).getExcelLable()); for (TableFiled field : excelXls.totalSheets.get(0).getFields()) { System.out.println(new Gson().toJson(field)); } @@ -56,7 +54,7 @@ public class ExcelReaderUtil { ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); excelXlsxReader.process(inputStream); System.out.println(excelXlsxReader.totalSheets.size()); - System.out.println(excelXlsxReader.totalSheets.get(0).getSheetName()); + System.out.println(excelXlsxReader.totalSheets.get(0).getExcelLable()); for (TableFiled field : excelXlsxReader.totalSheets.get(0).getFields()) { System.out.println(new Gson().toJson(field)); } @@ -68,7 +66,7 @@ public class ExcelReaderUtil { } public static void main(String[] args) throws Exception { - String file ="下单渠道.xlsx"; - ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file)); +// String file ="下单渠道.xlsx"; +// ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file)); } } diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java index 8d7d6050f3..c97c8d4c4c 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java @@ -300,9 +300,9 @@ public class ExcelXlsReader implements HSSFListener { } lastColumnNumber = -1; - if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){ ExcelSheetData excelSheetData = new ExcelSheetData(); - excelSheetData.setSheetName(sheetName); + excelSheetData.setExcelLable(sheetName); excelSheetData.setData(new ArrayList<>()); excelSheetData.setFields(new ArrayList<>()); totalSheets.add(excelSheetData); @@ -322,10 +322,10 @@ public class ExcelXlsReader implements HSSFListener { if (flag && curRow != 0) { //该行不为空行且该行不是第一行，发送（第一行为列名，不需要） - if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){ ExcelSheetData excelSheetData = new ExcelSheetData(); excelSheetData.setData(new ArrayList<>(data)); - excelSheetData.setSheetName(sheetName); + excelSheetData.setExcelLable(sheetName); excelSheetData.setFields(new ArrayList<>(fields)); List tmp = new ArrayList<>(cellList); excelSheetData.getData().add(tmp); @@ -333,7 +333,7 @@ public class ExcelXlsReader implements HSSFListener { totalSheets.add(excelSheetData); }else { List tmp = new ArrayList<>(cellList); - totalSheets.stream().filter(s->s.getSheetName().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp); + totalSheets.stream().filter(s->s.getExcelLable().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp); totalRows++; } } diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java index 63448318e1..8751a7c203 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java @@ -157,7 +157,7 @@ public class ExcelXlsxReader extends DefaultHandler { ExcelSheetData excelSheetData = new ExcelSheetData(); excelSheetData.setData(new ArrayList<>(data)); - excelSheetData.setSheetName(sheets.getSheetName()); + excelSheetData.setExcelLable(sheets.getSheetName()); excelSheetData.setFields(new ArrayList<>(fields)); totalSheets.add(excelSheetData); diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 0d952442be..ab6c300345 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -7,6 +7,7 @@ import io.dataease.base.domain.DatasetTableIncrementalConfig; import io.dataease.controller.request.dataset.DataSetTableRequest; import io.dataease.datasource.dto.TableFiled; import io.dataease.dto.dataset.DataSetTableDTO; +import io.dataease.dto.dataset.ExcelFileData; import io.dataease.service.dataset.DataSetTableService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -37,8 +38,12 @@ public class DataSetTableController { @ApiOperation("更新") @PostMapping("update") - public DatasetTable save(@RequestBody DataSetTableRequest datasetTable) throws Exception { - return dataSetTableService.save(datasetTable); + public void save(@RequestBody DataSetTableRequest datasetTable) throws Exception { + if(datasetTable.getType().equalsIgnoreCase("excel")){ + dataSetTableService.saveExcel(datasetTable); + }else { + dataSetTableService.save(datasetTable); + } } @ApiOperation("删除") @@ -121,8 +126,8 @@ public class DataSetTableController { @ApiOperation("excel上传") @PostMapping("excel/upload") - public Map excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId) throws Exception { - return dataSetTableService.excelSaveAndParse(file, tableId); + public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId, @RequestParam("editType") Integer editType ) throws Exception { + return dataSetTableService.excelSaveAndParse(file, tableId, editType); } @ApiOperation("检测doris") diff --git a/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java b/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java index df05e2cdc7..432969290b 100644 --- a/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java +++ b/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java @@ -2,6 +2,7 @@ package io.dataease.controller.request.dataset; import io.dataease.base.domain.DatasetTable; import io.dataease.datasource.dto.TableFiled; +import io.dataease.dto.dataset.ExcelSheetData; import lombok.Getter; import lombok.Setter; @@ -23,4 +24,6 @@ public class DataSetTableRequest extends DatasetTable { private Boolean isRename; private List typeFilter; private List fields; + private List sheets; + private boolean mergeSheet = false; } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java index a75f12dbba..1eed9391d1 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java @@ -14,7 +14,7 @@ import java.util.List; public class DataTableInfoDTO { private String table; private String sql; - private List sheets; + private List excelSheetDataList; private String data;// file path private List list; } diff --git a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java index 7e81d0fefd..10340dff3a 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java +++ b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java @@ -4,10 +4,18 @@ import io.dataease.datasource.dto.TableFiled; import lombok.Data; import java.util.List; +import java.util.Map; @Data public class ExcelSheetData { - private String sheetName; + private String excelLable; private List> data; private List fields; + private boolean isSheet = true; + private List> jsonArray; + private String datasetName; + private String sheetExcelId; + private String sheetId; + private String path; + private String fieldsMd5; } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 080882493f..18ae9a79c4 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -100,13 +100,7 @@ public class DataSetTableService { } private void extractData(DataSetTableRequest datasetTable) throws Exception { - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入"); - }); - return; - } - if (StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) { + if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) { DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest(); DatasetTableTask datasetTableTask = new DatasetTableTask(); datasetTableTask.setTableId(datasetTable.getId()); @@ -121,6 +115,113 @@ public class DataSetTableService { } } + public void saveExcel(DataSetTableRequest datasetTable)throws Exception { + if (StringUtils.isEmpty(datasetTable.getId())) { + if(datasetTable.isMergeSheet()){ + Map> map = datasetTable.getSheets().stream().collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5)); + for (String s : map.keySet()) { + DataSetTableRequest sheetTable = new DataSetTableRequest(); + BeanUtils.copyBean(sheetTable, datasetTable); + sheetTable.setId(UUID.randomUUID().toString()); + sheetTable.setCreateBy(AuthUtils.getUser().getUsername()); + sheetTable.setCreateTime(System.currentTimeMillis()); + List excelSheetDataList = map.get(s); + sheetTable.setName(excelSheetDataList.get(0).getDatasetName()); + checkName(sheetTable); + excelSheetDataList.forEach(excelSheetData -> { + String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + excelSheetData.setData(null); + excelSheetData.setJsonArray(null); + }); + DataTableInfoDTO info = new DataTableInfoDTO(); + info.setExcelSheetDataList(excelSheetDataList); + sheetTable.setInfo(new Gson().toJson(info)); + int insert = datasetTableMapper.insert(sheetTable); + if (insert == 1) { + saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields()); + commonThreadPool.addTask(() -> { + extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入"); + }); + } + } + }else { + for (ExcelSheetData sheet : datasetTable.getSheets()) { + String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + DataSetTableRequest sheetTable = new DataSetTableRequest(); + BeanUtils.copyBean(sheetTable, datasetTable); + sheetTable.setId(UUID.randomUUID().toString()); + sheetTable.setCreateBy(AuthUtils.getUser().getUsername()); + sheetTable.setCreateTime(System.currentTimeMillis()); + sheetTable.setName(sheet.getDatasetName()); + checkName(sheetTable); + sheet.setData(null); + sheet.setJsonArray(null); + List excelSheetDataList = new ArrayList<>(); + excelSheetDataList.add(sheet); + DataTableInfoDTO info = new DataTableInfoDTO(); + info.setExcelSheetDataList(excelSheetDataList); + sheetTable.setInfo(new Gson().toJson(info)); + int insert = datasetTableMapper.insert(sheetTable); + if (insert == 1) { + saveExcelTableField(sheetTable.getId(), sheet.getFields()); + commonThreadPool.addTask(() -> { + extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入"); + }); + } + } + } + return; + } + + List excelSheetDataList = new ArrayList<>(); + List oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList()); + for (ExcelSheetData sheet : datasetTable.getSheets()) { + //替换时， + if(datasetTable.getEditType() == 0){ + List newFields = sheet.getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList()); + if (!oldFields.equals(newFields)) { + DataEaseException.throwException(Translator.get("i18n_excel_colume_change")); + } + oldFields = newFields; + } + + String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + sheet.setData(null); + sheet.setJsonArray(null); + excelSheetDataList.add(sheet); + } + DataTableInfoDTO info = new DataTableInfoDTO(); + info.setExcelSheetDataList(excelSheetDataList); + datasetTable.setInfo(new Gson().toJson(info)); + int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable); + // 删除所有字段，重新抽象 + if(datasetTable.getEditType() == 0){ + dataSetTableFieldsService.deleteByTableId(datasetTable.getId()); + saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields()); + } + + if (update == 1) { + if (datasetTable.getEditType() == 0) { + commonThreadPool.addTask(() -> { + extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换"); + }); + } else if (datasetTable.getEditType() == 1) { + commonThreadPool.addTask(() -> { + extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加"); + }); + } + } + } + public DatasetTable save(DataSetTableRequest datasetTable) throws Exception { checkName(datasetTable); if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) { @@ -143,22 +244,11 @@ public class DataSetTableService { if (datasetTable.getIsRename() == null || !datasetTable.getIsRename()) { // 更新数据和字段 if (update == 1) { - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom") ) { // 删除所有字段，重新抽象 dataSetTableFieldsService.deleteByTableId(datasetTable.getId()); saveTableField(datasetTable); } - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - if (datasetTable.getEditType() == 0) { - commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换"); - }); - } else if (datasetTable.getEditType() == 1) { - commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加"); - }); - } - } } } } @@ -810,6 +900,29 @@ public class DataSetTableService { } } + public void saveExcelTableField(String datasetTableId, List fields) throws Exception { + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + TableFiled filed = fields.get(i); + DatasetTableField datasetTableField = DatasetTableField.builder().build(); + datasetTableField.setTableId(datasetTableId); + datasetTableField.setOriginName(filed.getFieldName()); + datasetTableField.setName(filed.getRemarks()); + datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName())); + datasetTableField.setType(filed.getFieldType()); + datasetTableField.setDeType(transFieldType(filed.getFieldType())); + datasetTableField.setDeExtractType(transFieldType(filed.getFieldType())); + datasetTableField.setSize(filed.getFieldSize()); + datasetTableField.setChecked(true); + datasetTableField.setColumnIndex(i); + datasetTableField.setLastSyncTime(System.currentTimeMillis()); + datasetTableField.setExtField(0); + datasetTableField.setGroupType(datasetTableField.getDeType() < 2 ? "d" : "q"); + dataSetTableFieldsService.save(datasetTableField); + } + } + } + public void saveTableField(DatasetTable datasetTable) throws Exception { Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); DataSetTableRequest dataSetTableRequest = new DataSetTableRequest(); @@ -828,11 +941,6 @@ public class DataSetTableService { datasourceRequest.setQuery(sqlAsTable); fields = datasourceProvider.fetchResultField(datasourceRequest); } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - /*DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); - String path = dataTableInfoDTO.getData(); - File file = new File(path); - Map map = parseExcel(path.substring(path.lastIndexOf("/") + 1), new FileInputStream(file), false); - fields = (List) map.get("fields");*/ fields = dataSetTableRequest.getFields(); } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { if (datasetTable.getMode() == 1) { @@ -1038,9 +1146,6 @@ public class DataSetTableService { } private void checkName(DatasetTable datasetTable) { -// if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) { -// return; -// } DatasetTableExample datasetTableExample = new DatasetTableExample(); DatasetTableExample.Criteria criteria = datasetTableExample.createCriteria(); if (StringUtils.isNotEmpty(datasetTable.getId())) { @@ -1069,11 +1174,13 @@ public class DataSetTableService { return map; } - public Map excelSaveAndParse(MultipartFile file, String tableId) throws Exception { + public ExcelFileData excelSaveAndParse(MultipartFile file, String tableId, Integer editType) throws Exception { String filename = file.getOriginalFilename(); // parse file - Map fileMap = parseExcel2(filename, file.getInputStream(), true); - if (StringUtils.isNotEmpty(tableId)) { + List excelSheetDataList = parseExcel2(filename, file.getInputStream(), true); + List retrunSheetDataList = new ArrayList<>(); + + if (StringUtils.isNotEmpty(tableId) && editType == 1 ) { List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId); datasetTableFields.sort((o1, o2) -> { if (o1.getColumnIndex() == null) { @@ -1084,64 +1191,80 @@ public class DataSetTableService { } return o1.getColumnIndex().compareTo(o2.getColumnIndex()); }); - List fields = (List) fileMap.get("fields"); - List newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList()); List oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); - if (!oldFields.equals(newFields)) { + for (ExcelSheetData excelSheetData : excelSheetDataList) { + List fields = excelSheetData.getFields(); + List newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList()); + if (oldFields.equals(newFields)) { + retrunSheetDataList.add(excelSheetData); + } + } + + if (retrunSheetDataList.size() == 0) { DataEaseException.throwException(Translator.get("i18n_excel_colume_change")); } + }else { + retrunSheetDataList = excelSheetDataList; } + // save file - String filePath = saveFile(file); - Map map = new HashMap<>(fileMap); - map.put("path", filePath); - return map; + String excelId = UUID.randomUUID().toString(); + String filePath = saveFile(file, excelId); + ExcelFileData excelFileData = new ExcelFileData(); + excelFileData.setExcelLable(filename); + excelFileData.setExcelId(excelId); + excelFileData.setPath(filePath); + + filename = filename.substring(0, filename.lastIndexOf('.')); + if(retrunSheetDataList.size() == 1){ + retrunSheetDataList.get(0).setDatasetName(filename); + retrunSheetDataList.get(0).setSheetExcelId(excelId); + retrunSheetDataList.get(0).setSheetId(UUID.randomUUID().toString()); + retrunSheetDataList.get(0).setPath(filePath); + }else { + for (ExcelSheetData excelSheetData : retrunSheetDataList) { + excelSheetData.setDatasetName(filename + "-" + excelSheetData.getExcelLable()); + excelSheetData.setSheetExcelId(excelId); + excelSheetData.setSheetId(UUID.randomUUID().toString()); + excelSheetData.setPath(filePath); + } + } + excelFileData.setSheets(retrunSheetDataList); + return excelFileData; } - private Map parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception { + private List parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception { + List excelSheetDataList = new ArrayList<>(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); - List fields = new ArrayList<>(); - List> data = new ArrayList<>(); - List> jsonArray = new ArrayList<>(); - List sheets = new ArrayList<>(); if (StringUtils.equalsIgnoreCase(suffix, "xls")) { ExcelXlsReader excelXlsReader = new ExcelXlsReader(); excelXlsReader.process(inputStream); - fields = excelXlsReader.totalSheets.get(0).getFields(); - data = excelXlsReader.totalSheets.get(0).getData(); - sheets = excelXlsReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); + excelSheetDataList = excelXlsReader.totalSheets; } if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); excelXlsxReader.process(inputStream); - fields = excelXlsxReader.totalSheets.get(0).getFields(); - data = excelXlsxReader.totalSheets.get(0).getData(); - sheets = excelXlsxReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); - } - - String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); - - // 校验excel字段是否重名 - if (checkIsRepeat(fieldArray)) { - DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); - } - - if (CollectionUtils.isNotEmpty(data)) { - jsonArray = data.stream().map(ele -> { - Map map = new HashMap<>(); - for (int i = 0; i < ele.size(); i++) { - map.put(fieldArray[i], ele.get(i)); - } - return map; - }).collect(Collectors.toList()); + excelSheetDataList = excelXlsxReader.totalSheets; } inputStream.close(); + excelSheetDataList.forEach(excelSheetData -> { + List> data = excelSheetData.getData(); + String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); + List> jsonArray = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.size(); i++) { + map.put(fieldArray[i], ele.get(i)); + } + return map; + }).collect(Collectors.toList()); + } + excelSheetData.setFieldsMd5(Md5Utils.md5(StringUtils.join(fieldArray, ","))); + excelSheetData.setJsonArray(jsonArray); + }); - Map map = new HashMap<>(); - map.put("fields", fields); - map.put("data", jsonArray); - map.put("sheets", sheets); - return map; + return excelSheetDataList; } private Map parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception { @@ -1387,16 +1510,15 @@ public class DataSetTableService { return ""; } - private String saveFile(MultipartFile file) throws Exception { + private String saveFile(MultipartFile file, String fileNameUUID) throws Exception { String filename = file.getOriginalFilename(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); - filename = Md5Utils.md5(filename.substring(0, filename.length() - suffix.length())); String dirPath = path + AuthUtils.getUser().getUsername() + "/"; File p = new File(dirPath); if (!p.exists()) { p.mkdirs(); } - String filePath = dirPath + filename + "." + suffix; + String filePath = dirPath + fileNameUUID + "." + suffix; File f = new File(filePath); FileOutputStream fileOutputStream = new FileOutputStream(f); fileOutputStream.write(file.getBytes()); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index c2bb860bb3..d332229b03 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -17,6 +17,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.datasource.service.DatasourceService; import io.dataease.dto.dataset.DataTableInfoDTO; +import io.dataease.dto.dataset.ExcelSheetData; import io.dataease.exception.DataEaseException; import io.dataease.listener.util.CacheUtils; import io.dataease.provider.QueryProvider; @@ -192,7 +193,9 @@ public class ExtractDataService { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); } finally { deleteFile("all_scope", datasetTableId); -// deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData()); + for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) { + deleteFile(excelSheetData.getPath()); + } } break; @@ -213,6 +216,9 @@ public class ExtractDataService { } finally { deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); + for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) { + deleteFile(excelSheetData.getPath()); + } } break; } @@ -831,18 +837,30 @@ public class ExtractDataService { private StepMeta excelInputStep(String Info, List datasetTableFields){ DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class); - String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1); + List excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList(); + String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1); ExcelInputMeta excelInputMeta = new ExcelInputMeta(); + + List sheetNames = new ArrayList<>(); + List files = new ArrayList<>(); + for (ExcelSheetData excelSheetData : excelSheetDataList) { + if(!sheetNames.contains(excelSheetData.getExcelLable())){ + sheetNames.add(excelSheetData.getExcelLable()); + } + if(!files.contains(excelSheetData.getPath())){ + files.add(excelSheetData.getPath()); + } + } if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI); - excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); + excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()])); } if (StringUtils.equalsIgnoreCase(suffix, "xls")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL); - excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); + excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()])); } excelInputMeta.setPassword("Encrypted"); - excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()}); + excelInputMeta.setFileName( files.toArray(new String[files.size()])); excelInputMeta.setStartsWithHeader(true); excelInputMeta.setIgnoreEmptyRows(true); ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()]; diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 2e66018e6b..cda88689fe 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1006,7 +1006,8 @@ export default { field_manage: 'Field Manage', edit_calc_field: 'Edit calc field', calc_field: 'Calc Field', - show_sql: 'Show SQL' + show_sql: 'Show SQL', + ple_select_excel: 'Please select excel file to import' }, datasource: { datasource: 'Data Source', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 12c3625fee..f2d612ca03 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1006,7 +1006,8 @@ export default { field_manage: '字段管理', edit_calc_field: '編輯計算字段', calc_field: '計算字段', - show_sql: '顯示SQL' + show_sql: '顯示SQL', + ple_select_excel: '請選擇要導入的 Excel' }, datasource: { datasource: '數據源', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 503a0c9c6c..255b9ff709 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1006,7 +1006,8 @@ export default { field_manage: '字段管理', edit_calc_field: '编辑计算字段', calc_field: '计算字段', - show_sql: '显示SQL' + show_sql: '显示SQL', + ple_select_excel: '请选择要导入的 Excel' }, datasource: { datasource: '数据源', diff --git a/frontend/src/views/dataset/add/AddExcel.vue b/frontend/src/views/dataset/add/AddExcel.vue index b777f543ba..12d16b8572 100644 --- a/frontend/src/views/dataset/add/AddExcel.vue +++ b/frontend/src/views/dataset/add/AddExcel.vue @@ -9,115 +9,132 @@ {{ $t('dataset.cancel') }} - + {{ $t('dataset.confirm') }} - - - - - - - - - - - {{ $t('dataset.upload_file') }} - {{ $t('dataset.uploading') }} - - - - - - - + - -
- {{ $t('dataset.data_preview') }} - （{{ $t('dataset.preview_100_data') }}） -
-
- - - - - -
- + + + + + + + +
+ + + + + +
+ + + @@ -140,12 +157,15 @@ export default { tableId: { type: String, default: null + }, + editType: { + type: Number, + default: 0 } }, data() { return { - name: '', - fields: [], + sheetObj: {datasetName: " ", fields: []}, sheets: [], data: [], mode: '1', @@ -160,13 +180,18 @@ export default { { label: this.$t('dataset.time'), value: 'DATETIME' }, { label: this.$t('dataset.value'), value: 'LONG' }, { label: this.$t('dataset.value') + '(' + this.$t('dataset.float') + ')', value: 'DOUBLE' } - ] + ], + props: { + label: 'excelLable', + children: 'sheets' + }, + count: 1, + excelData: [] } }, watch: { }, mounted() { - // this.initDataSource() window.onresize = () => { this.calHeight() } @@ -176,13 +201,33 @@ export default { if (!this.param.tableId) { this.param.tableId = '' } + if (!this.param.editType) { + this.param.editType = 0 + } }, methods: { - // initDataSource() { - // listDatasource().then(response => { - // this.options = response.data - // }) - // }, + handleCheckChange(data, checked, indeterminate) { + }, + handleNodeClick(data) { + if(data.sheet){ + this.sheetObj = data + this.fields = data.fields + this.jsonArray = data.jsonArray + const datas = this.jsonArray + this.$refs.plxTable.reloadData(datas) + } + }, + changeDatasetName(){ + for(var i=0;i 1) { - this.$warning(this.$t('dataset.sheet_warn')) - } - this.data = response.data.data - const datas = this.data - this.$refs.plxTable.reloadData(datas) - - if (file.name.lastIndexOf('.') > 0) { - this.name = file.name.substring(0, file.name.lastIndexOf('.')) - } + this.excelData.push(response.data) this.fileList = fileList this.uploading = false }, save() { - if (!this.name || this.name === '') { - this.$message({ - showClose: true, - message: this.$t('dataset.pls_input_name'), - type: 'error' - }) + + var validate = true; + var selectedSheet = [] + var sheetFileMd5 = [] + var selectNode = this.$refs.tree.getCheckedNodes() + for(var i=0;i 50) { + this.$message({ + showClose: true, + message: this.$t('dataset.char_can_not_more_50'), + type: 'error' + }) + return + } + + selectedSheet.push(selectNode[i]) + sheetFileMd5.push(selectNode[i].fieldsMd5) + } + } + if(selectedSheet.length == 0){ + this.$message.warning(this.$t('dataset.ple_select_excel')) return } - if (this.name.length > 50) { - this.$message({ - showClose: true, - message: this.$t('dataset.char_can_not_more_50'), - type: 'error' - }) - return + if(!validate){ + return; } + let table = {} if (!this.param.tableId) { table = { id: this.param.tableId, - name: this.name, sceneId: this.param.id, dataSourceId: null, type: 'excel', + sheets: selectedSheet, mode: parseInt(this.mode), - // info: '{"data":"' + this.path + '"}', - info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]] }), - fields: this.fields + editType: 0 } } else { table = { @@ -268,22 +318,37 @@ export default { sceneId: this.param.id, dataSourceId: null, type: 'excel', + sheets: selectedSheet, mode: parseInt(this.mode), - // info: '{"data":"' + this.path + '"}', - info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]] }), editType: this.param.editType ? this.param.editType : 0 } } - post('/dataset/table/update', table).then(response => { - // this.$store.dispatch('dataset/setSceneData', new Date().getTime()) - this.$emit('saveSuccess', table) - this.cancel() - }) + if(new Set(sheetFileMd5).size !== sheetFileMd5.length){ + this.$confirm('The fields in the data table are consistent, whether they are merged into one table?', 'Merge data table ', { + confirmButtonText: 'Merge', + cancelButtonText: 'Do not merge', + type: 'info' + }).then(() => { + table.mergeSheet = true + post('/dataset/table/update', table).then(response => { + this.$emit('saveSuccess', table) + this.cancel() + }) + }).catch(action => { + if(action == 'cancle'){ + return + } + table.mergeSheet = false + post('/dataset/table/update', table).then(response => { + this.$emit('saveSuccess', table) + this.cancel() + }) + }); + } }, cancel() { this.dataReset() - // this.$router.push('/dataset/home') if (this.param.tableId) { this.$emit('switchComponent', { name: 'ViewTable', param: this.param.table }) } else { @@ -337,4 +402,14 @@ export default { .dataPreview>>>.el-card__body{ padding:10px; } + + .el-header { + background-color: rgb(241, 243, 248); + color: #333; + line-height: 30px; + } + + .el-main { + padding: 0px + }