From 9354dc92eaf234a13d34237591f6e6a356a0356b Mon Sep 17 00:00:00 2001 From: "zhantiao@tiduyun.com" Date: Fri, 11 Nov 2022 10:11:21 +0800 Subject: [PATCH] =?UTF-8?q?perf:=E6=95=B0=E6=8D=AE=E9=9B=86=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E6=A8=A1=E5=9D=97=E7=9A=84=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/dataset/DataSetTableService.java | 161 +++++++----------- 1 file changed, 65 insertions(+), 96 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index d7ddee0749..5defcafe70 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -149,7 +149,7 @@ public class DataSetTableService { @Value("${upload.file.path}") private String path; - private static Logger logger = LoggerFactory.getLogger(ClassloaderResponsity.class); + private static final Logger logger = LoggerFactory.getLogger(ClassloaderResponsity.class); @DeCleaner(value = DePermissionType.DATASET, key = "sceneId") public List batchInsert(List datasetTable) throws Exception { @@ -185,7 +185,7 @@ public class DataSetTableService { @Transactional(propagation = Propagation.NOT_SUPPORTED) @DeCleaner(value = DePermissionType.DATASET, key = "sceneId") - public List saveExcel(DataSetTableRequest datasetTable) throws Exception { + public List saveExcel(DataSetTableRequest datasetTable) { List datasetIdList = new ArrayList<>(); if (StringUtils.isEmpty(datasetTable.getId())) { @@ -234,10 +234,9 @@ public class DataSetTableService { list.add(sheetTable); DeLogUtils.save(SysLogConstants.OPERATE_TYPE.CREATE, SysLogConstants.SOURCE_TYPE.DATASET, datasetTable.getId(), datasetTable.getSceneId(), null, null); } - datasetIdList.forEach(datasetId -> { - commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", - null, datasetIdList)); - }); + datasetIdList.forEach(datasetId -> commonThreadPool.addTask(() -> + extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", + null, datasetIdList))); } else { for (ExcelSheetData sheet : datasetTable.getSheets()) { String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName) @@ -269,10 +268,9 @@ public class DataSetTableService { list.add(sheetTable); DeLogUtils.save(SysLogConstants.OPERATE_TYPE.MODIFY, SysLogConstants.SOURCE_TYPE.DATASET, datasetTable.getId(), datasetTable.getSceneId(), null, null); } - datasetIdList.forEach(datasetId -> { - commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", - null, datasetIdList)); - }); + datasetIdList.forEach(datasetId -> commonThreadPool.addTask(() -> + extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", + null, datasetIdList))); } return list; @@ -303,15 +301,15 @@ public class DataSetTableService { DataTableInfoDTO info = new DataTableInfoDTO(); info.setExcelSheetDataList(excelSheetDataList); datasetTable.setInfo(new Gson().toJson(info)); - int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable); + // 替換時，先不刪除旧字段；同步成功后再删除 if (datasetTable.getEditType() == 0) { commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false), - Arrays.asList(datasetTable.getId()))); + Collections.singletonList(datasetTable.getId()))); } else if (datasetTable.getEditType() == 1) { commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", - null, Arrays.asList(datasetTable.getId()))); + null, Collections.singletonList(datasetTable.getId()))); } DeLogUtils.save(SysLogConstants.OPERATE_TYPE.MODIFY, SysLogConstants.SOURCE_TYPE.DATASET, datasetTable.getId(), datasetTable.getSceneId(), null, null); return Collections.singletonList(datasetTable); @@ -362,12 +360,12 @@ public class DataSetTableService { return datasetTable; } - public void alter(DataSetTableRequest request) throws Exception { + public void alter(DataSetTableRequest request) { checkName(request); datasetTableMapper.updateByPrimaryKeySelective(request); } - public void delete(String id) throws Exception { + public void delete(String id) { DatasetTable table = datasetTableMapper.selectByPrimaryKey(id); SysLogDTO sysLogDTO = DeLogUtils.buildLog(SysLogConstants.OPERATE_TYPE.DELETE, SysLogConstants.SOURCE_TYPE.DATASET, table.getId(), table.getSceneId(), null, null); datasetTableMapper.deleteByPrimaryKey(id); @@ -489,9 +487,8 @@ public class DataSetTableService { List res = new ArrayList<>(); Map map = new TreeMap<>(); group.forEach(ele -> map.put(ele.getId(), ele)); - Iterator> iterator = map.entrySet().iterator(); - while (iterator.hasNext()) { - res.add(iterator.next().getValue()); + for (Map.Entry stringDataSetTableDTOEntry : map.entrySet()) { + res.add(stringDataSetTableDTOEntry.getValue()); } res.sort(Comparator.comparing(DatasetTable::getName)); res.addAll(ds); @@ -521,8 +518,7 @@ public class DataSetTableService { return datasourceProvider.getTableFields(datasourceRequest); } - public Map> getFieldsFromDE(DataSetTableRequest dataSetTableRequest) - throws Exception { + public Map> getFieldsFromDE(DataSetTableRequest dataSetTableRequest) { DatasetTableField datasetTableField = DatasetTableField.builder().build(); datasetTableField.setTableId(dataSetTableRequest.getId()); datasetTableField.setChecked(Boolean.TRUE); @@ -1015,15 +1011,11 @@ public class DataSetTableService { break; case "TEXT": sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("TEXT")).collect(Collectors.toList()); - sqlVariableDetails.forEach(item -> { - item.setAlias(item.getVariableName()); - }); + sqlVariableDetails.forEach(item -> item.setAlias(item.getVariableName())); break; case "NUM": sqlVariableDetails = sqlVariableDetails.stream().filter(item -> item.getType().get(0).contains("LONG") || item.getType().get(0).contains("DOUBLE")).collect(Collectors.toList()); - sqlVariableDetails.forEach(item -> { - item.setAlias(item.getVariableName()); - }); + sqlVariableDetails.forEach(item -> item.setAlias(item.getVariableName())); break; } return sqlVariableDetails; @@ -1087,15 +1079,15 @@ public class DataSetTableService { if (select.getSelectBody() instanceof PlainSelect) { return handlePlainSelect((PlainSelect) select.getSelectBody(), select, dsType); } else { - String result = ""; + StringBuilder result = new StringBuilder(); SetOperationList setOperationList = (SetOperationList) select.getSelectBody(); for (int i = 0; i < setOperationList.getSelects().size(); i++) { - result = result + handlePlainSelect((PlainSelect) setOperationList.getSelects().get(i), null, dsType); + result.append(handlePlainSelect((PlainSelect) setOperationList.getSelects().get(i), null, dsType)); if (i < setOperationList.getSelects().size() - 1) { - result = result + " " + setOperationList.getOperations().get(i).toString() + " "; + result.append(" ").append(setOperationList.getOperations().get(i).toString()).append(" "); } } - return result; + return result.toString(); } } @@ -1140,14 +1132,14 @@ public class DataSetTableService { builder.append(" "); for (Iterator iter = select.getWithItemsList().iterator(); iter.hasNext(); ) { WithItem withItem = iter.next(); - builder.append(withItem.getName() + " AS ( " + removeVariables(withItem.getSubSelect().toString(), dsType) + " ) "); + builder.append(withItem.getName()).append(" AS ( ").append(removeVariables(withItem.getSubSelect().toString(), dsType)).append(" ) "); if (iter.hasNext()) { builder.append(","); } } } - builder.append(" " + plainSelect); + builder.append(" ").append(plainSelect); return builder.toString(); } @@ -1230,7 +1222,7 @@ public class DataSetTableService { String sqlAsTable = qp.createSQLPreview(sql, null); datasourceRequest.setQuery(sqlAsTable); - Map result = new HashMap<>(); + Map result; try { datasetSqlLog.setStartTime(System.currentTimeMillis()); result = datasourceProvider.fetchResultAndField(datasourceRequest); @@ -1275,7 +1267,7 @@ public class DataSetTableService { public Map getUnionPreview(DataSetTableRequest dataSetTableRequest) throws Exception { DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); - Map sqlMap = new HashMap<>(); + Map sqlMap; DatasourceRequest datasourceRequest = new DatasourceRequest(); Datasource ds; if (dataSetTableRequest.getMode() == 0) { @@ -1289,7 +1281,6 @@ public class DataSetTableService { } String sql = (String) sqlMap.get("sql"); List fieldList = (List) sqlMap.get("field"); - List join = (List) sqlMap.get("join"); Map res = new HashMap<>(); // 处理结果 @@ -1423,9 +1414,7 @@ public class DataSetTableService { DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0); if (CollectionUtils.isNotEmpty(list)) { StringBuilder field = new StringBuilder(); - Iterator> iterator = customInfo.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); + for (Map.Entry next : customInfo.entrySet()) { field.append(StringUtils.join(next.getValue(), ",")).append(","); } String f = field.substring(0, field.length() - 1); @@ -1493,9 +1482,7 @@ public class DataSetTableService { String tableName = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getTable(); if (CollectionUtils.isNotEmpty(list)) { StringBuilder field = new StringBuilder(); - Iterator> iterator = customInfo.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); + for (Map.Entry next : customInfo.entrySet()) { field.append(StringUtils.join(next.getValue(), ",")).append(","); } String f = field.substring(0, field.length() - 1); @@ -1601,9 +1588,7 @@ public class DataSetTableService { if (CollectionUtils.isNotEmpty(unionList)) { // field StringBuilder field = new StringBuilder(); - Iterator> iterator = checkedInfo.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); + for (Map.Entry next : checkedInfo.entrySet()) { field.append(StringUtils.join(next.getValue(), ",")).append(","); } String f = subPrefixSuffixChar(field.toString()); @@ -1664,8 +1649,7 @@ public class DataSetTableService { // 递归计算出所有子级的checkedFields和unionParam private void getUnionSQLDorisJoin(List childrenDs, Map checkedInfo, List unionList, List checkedFields) { - for (int i = 0; i < childrenDs.size(); i++) { - UnionDTO unionDTO = childrenDs.get(i); + for (UnionDTO unionDTO : childrenDs) { String tableId = unionDTO.getCurrentDs().getId(); String table = TableUtils.tableName(tableId); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); @@ -1752,9 +1736,7 @@ public class DataSetTableService { if (CollectionUtils.isNotEmpty(unionList)) { // field StringBuilder field = new StringBuilder(); - Iterator> iterator = checkedInfo.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry next = iterator.next(); + for (Map.Entry next : checkedInfo.entrySet()) { field.append(StringUtils.join(next.getValue(), ",")).append(","); } String f = subPrefixSuffixChar(field.toString()); @@ -1825,9 +1807,7 @@ public class DataSetTableService { // 递归计算出所有子级的checkedFields和unionParam private void getUnionSQLDatasourceJoin(List childrenDs, Map checkedInfo, List unionList, String keyword, List checkedFields) { - for (int i = 0; i < childrenDs.size(); i++) { - UnionDTO unionDTO = childrenDs.get(i); - + for (UnionDTO unionDTO : childrenDs) { DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId()); String tableId = unionDTO.getCurrentDs().getId(); if (ObjectUtils.isEmpty(datasetTable)) { @@ -1976,7 +1956,7 @@ public class DataSetTableService { Map sqlMap = getUnionSQLDoris(dataTableInfoDTO); String sql = (String) sqlMap.get("sql"); List fieldList = (List) sqlMap.get("field"); - List join = (List) sqlMap.get("join"); + // custom 创建doris视图 createDorisView(TableUtils.tableName(datasetTable.getId()), sql); @@ -2003,7 +1983,7 @@ public class DataSetTableService { Map sqlMap = getUnionSQLDatasource(dt, ds); String sql = (String) sqlMap.get("sql"); List fieldList = (List) sqlMap.get("field"); - List join = (List) sqlMap.get("join"); + datasourceRequest.setQuery(sql); fields = datasourceProvider.fetchResultField(datasourceRequest); @@ -2157,16 +2137,15 @@ public class DataSetTableService { private void checkColumes(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception { DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(datasetTableIncrementalConfig.getTableId()); List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(datasetTable.getId()) - .stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); - datasetTableFields.sort((o1, o2) -> { - if (o1.getColumnIndex() == null) { - return -1; - } - if (o2.getColumnIndex() == null) { - return 1; - } - return o1.getColumnIndex().compareTo(o2.getColumnIndex()); - }); + .stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).sorted((o1, o2) -> { + if (o1.getColumnIndex() == null) { + return -1; + } + if (o2.getColumnIndex() == null) { + return 1; + } + return o1.getColumnIndex().compareTo(o2.getColumnIndex()); + }).collect(Collectors.toList()); List originNameFields = datasetTableFields.stream().map(DatasetTableField::getOriginName) .collect(Collectors.toList()); @@ -2184,9 +2163,7 @@ public class DataSetTableService { List sqlFields = new ArrayList<>(); try { datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName) - .forEach(field -> { - sqlFields.add(field); - }); + .forEach(sqlFields::add); } catch (Exception e) { DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage()); } @@ -2204,7 +2181,7 @@ public class DataSetTableService { List sqlFields = new ArrayList<>(); try { datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName) - .forEach(field -> sqlFields.add(field)); + .forEach(sqlFields::add); } catch (Exception e) { DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage()); } @@ -2228,7 +2205,7 @@ public class DataSetTableService { criteria.andNameEqualTo(datasetTable.getName()); } List list = datasetTableMapper.selectByExample(datasetTableExample); - if (list.size() > 0) { + if (!list.isEmpty()) { throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group")); } } @@ -2253,7 +2230,7 @@ public class DataSetTableService { criteria.andNameIn(new ArrayList<>(nameSet)); } List list = datasetTableMapper.selectByExample(datasetTableExample); - if (list.size() > 0) { + if (!list.isEmpty()) { throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group")); } } @@ -2278,8 +2255,7 @@ public class DataSetTableService { if (StringUtils.isNotEmpty(tableId)) { List fields = dataSetTableFieldsService.getFieldsByTableId(tableId); - List datasetTableFields = fields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); - datasetTableFields.sort((o1, o2) -> { + List datasetTableFields = fields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).sorted((o1, o2) -> { if (o1.getColumnIndex() == null) { return -1; } @@ -2287,7 +2263,7 @@ public class DataSetTableService { return 1; } return o1.getColumnIndex().compareTo(o2.getColumnIndex()); - }); + }).collect(Collectors.toList()); List oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); @@ -2299,7 +2275,7 @@ public class DataSetTableService { returnSheetDataList.add(excelSheetData); } } - if (returnSheetDataList.size() == 0) { + if (returnSheetDataList.isEmpty()) { DataEaseException.throwException(Translator.get("i18n_excel_column_change")); } } else { @@ -2321,22 +2297,19 @@ public class DataSetTableService { for (ExcelSheetData excelSheetData : excelSheetDataList) { List tableFields = excelSheetData.getFields(); List newFields = tableFields.stream().map(TableField::getRemarks).collect(Collectors.toList()); - if (oldFields.equals(newFields)) { - excelSheetData.setChangeFiled(false); - } else { - excelSheetData.setChangeFiled(true); - } + excelSheetData.setChangeFiled(!oldFields.equals(newFields)); boolean effectExtField = false; for (String extFieldsRefName : extFieldsRefNames) { if (!newFields.contains(extFieldsRefName)) { effectExtField = true; + break; } } excelSheetData.setEffectExtField(effectExtField); returnSheetDataList.add(excelSheetData); } - if (returnSheetDataList.size() == 0) { + if (returnSheetDataList.isEmpty()) { DataEaseException.throwException(Translator.get("i18n_excel_column_change")); } } @@ -2403,7 +2376,7 @@ public class DataSetTableService { } List> data = new ArrayList<>(); int num = 1; - String line = null; + String line; while ((line = reader.readLine()) != null) { if (num > 100) { break; @@ -2450,7 +2423,7 @@ public class DataSetTableService { try { double d = cell.getNumericCellValue(); try { - Double value = new Double(d); + Double value = d; double eps = 1e-10; if (value - Math.floor(value) < eps) { if (cellType) { @@ -2469,7 +2442,7 @@ public class DataSetTableService { return nf.format(value); } } catch (Exception e) { - BigDecimal b = new BigDecimal(d); + BigDecimal b = BigDecimal.valueOf(d); return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + ""; } } catch (IllegalStateException e) { @@ -2502,7 +2475,7 @@ public class DataSetTableService { } else { double d = cell.getNumericCellValue(); try { - Double value = new Double(d); + Double value = d; double eps = 1e-10; if (value - Math.floor(value) < eps) { if (cellType) { @@ -2521,7 +2494,7 @@ public class DataSetTableService { return nf.format(value); } } catch (Exception e) { - BigDecimal b = new BigDecimal(d); + BigDecimal b = BigDecimal.valueOf(d); return b.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue() + ""; } } @@ -2655,18 +2628,14 @@ public class DataSetTableService { * 判断数组中是否有重复的值 */ public static boolean checkIsRepeat(String[] array) { - HashSet hashSet = new HashSet(); - for (int i = 0; i < array.length; i++) { - if (StringUtils.isEmpty(array[i])) { + HashSet hashSet = new HashSet<>(); + for (String s : array) { + if (StringUtils.isEmpty(s)) { throw new RuntimeException(Translator.get("i18n_excel_empty_column")); } - hashSet.add(array[i]); - } - if (hashSet.size() == array.length) { - return false; - } else { - return true; + hashSet.add(s); } + return hashSet.size() != array.length; } public DatasetTable syncDatasetTableField(String id) throws Exception { @@ -2857,7 +2826,7 @@ public class DataSetTableService { return expressionDeParser; } - static private boolean hasVariable(String sql) { + private static boolean hasVariable(String sql) { return sql.contains(SubstitutedParams); } @@ -2936,7 +2905,7 @@ public class DataSetTableService { //设置列的宽度 detailsSheet.setColumnWidth(j, 255 * 20); } else { - if ((fields.get(j).getDeType() == DeTypeConstants.DE_INT || fields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { + if ((fields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || fields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { try { cell.setCellValue(Double.valueOf(rowData.get(j))); } catch (Exception e) {