diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 2a90d706f9..d6c594439a 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -73,6 +73,11 @@ public class DataSetTableController { return dataSetTableService.getSQLPreview(dataSetTableRequest); } + @PostMapping("customPreview") + public Map customPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { + return dataSetTableService.getCustomPreview(dataSetTableRequest); + } + @PostMapping("incrementalConfig") public DatasetTableIncrementalConfig incrementalConfig(@RequestBody DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception { return dataSetTableService.incrementalConfig(datasetTableIncrementalConfig); diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoCustomUnion.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoCustomUnion.java new file mode 100644 index 0000000000..391643669e --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoCustomUnion.java @@ -0,0 +1,17 @@ +package io.dataease.dto.dataset; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * @Author gin + * @Date 2021/5/9 10:08 上午 + */ +@Getter +@Setter +public class DataTableInfoCustomUnion { + private String tableId; + private List checkedFields; +} diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java index 2214976123..ffa73619cd 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java @@ -3,6 +3,8 @@ package io.dataease.dto.dataset; import lombok.Getter; import lombok.Setter; +import java.util.List; + /** * @Author gin * @Date 2021/2/23 8:47 下午 @@ -13,4 +15,5 @@ public class DataTableInfoDTO { private String table; private String sql; private String data;// file path + private List list; } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index fc93757613..3bff8687e7 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -15,6 +15,8 @@ import io.dataease.datasource.provider.JdbcProvider; import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.dataset.DataSetPreviewPage; +import io.dataease.dto.dataset.DataSetTableUnionDTO; +import io.dataease.dto.dataset.DataTableInfoCustomUnion; import io.dataease.dto.dataset.DataTableInfoDTO; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -59,6 +61,8 @@ public class DataSetTableService { private ExtractDataService extractDataService; @Resource private DatasetTableIncrementalConfigMapper datasetTableIncrementalConfigMapper; + @Resource + private DataSetTableUnionService dataSetTableUnionService; @Value("${upload.file.path}") private String path; @@ -83,7 +87,7 @@ public class DataSetTableService { // 添加表成功后,获取当前表字段和类型,抽象到dataease数据库 if (insert == 1) { saveTableField(datasetTable); - commonThreadPool.addTask(()->{ + commonThreadPool.addTask(() -> { extractDataService.extractData(datasetTable.getId(), null, "all_scope"); }); } @@ -115,6 +119,9 @@ public class DataSetTableService { if (StringUtils.isNotEmpty(dataSetTableRequest.getSceneId())) { criteria.andSceneIdEqualTo(dataSetTableRequest.getSceneId()); } + if (ObjectUtils.isNotEmpty(dataSetTableRequest.getMode())) { + criteria.andModeEqualTo(dataSetTableRequest.getMode()); + } if (StringUtils.isNotEmpty(dataSetTableRequest.getSort())) { datasetTableExample.setOrderByClause(dataSetTableRequest.getSort()); } @@ -247,8 +254,8 @@ public class DataSetTableService { e.printStackTrace(); } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); - JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);; + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); @@ -267,7 +274,25 @@ public class DataSetTableService { e.printStackTrace(); } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT " + (page - 1) * pageSize + "," + realSize); + try { + data.addAll(jdbcProvider.getData(datasourceRequest)); + } catch (Exception e) { + e.printStackTrace(); + } + + try { + datasourceRequest.setQuery(createQueryCountSQL(ds.getType(), table)); + dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); + } catch (Exception e) { + e.printStackTrace(); + } } List> jsonArray = new ArrayList<>(); @@ -319,6 +344,93 @@ public class DataSetTableService { return map; } + public Map getCustomPreview(DataSetTableRequest dataSetTableRequest) throws Exception { + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); +// String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + List list = dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()); + + datasourceRequest.setQuery(getCustomSQL(dataTableInfoDTO, list)); + Map result = jdbcProvider.fetchResultAndField(datasourceRequest); + List data = result.get("dataList"); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + + List> jsonArray = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.length; i++) { + map.put(fieldArray[i], ele[i]); + } + return map; + }).collect(Collectors.toList()); + } + + Map map = new HashMap<>(); + map.put("fields", fields); + map.put("data", jsonArray); + + return map; + } + + private String getCustomSQL(DataTableInfoDTO dataTableInfoDTO, List list) { + Map customInfo = new HashMap<>(); + dataTableInfoDTO.getList().forEach(ele -> { + String table = DorisTableUtils.dorisName(ele.getTableId()); + List fields = dataSetTableFieldsService.getListByIds(ele.getCheckedFields()); + String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName()).toArray(String[]::new); + customInfo.put(table, array); + }); + DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0); + if (CollectionUtils.isNotEmpty(list)) { + StringBuilder field = new StringBuilder(); + Iterator> iterator = customInfo.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + field.append(StringUtils.join(next.getValue(), ",")).append(","); + } + String f = field.substring(0, field.length() - 1); + + StringBuilder join = new StringBuilder(); + for (DataTableInfoCustomUnion dataTableInfoCustomUnion : dataTableInfoDTO.getList()) { + for (DataSetTableUnionDTO dto : list) { + // 被关联表和自助数据集的表相等 + if (StringUtils.equals(dto.getTargetTableId(), dataTableInfoCustomUnion.getTableId())) { + DatasetTableField sourceField = dataSetTableFieldsService.get(dto.getSourceTableFieldId()); + DatasetTableField targetField = dataSetTableFieldsService.get(dto.getTargetTableFieldId()); + join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation())) + .append(DorisTableUtils.dorisName(dto.getTargetTableId())) + .append(" ON ") + .append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".").append(sourceField.getDataeaseName()) + .append(" = ") + .append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".").append(targetField.getDataeaseName()); + } + } + } + return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId())) + join.toString(); + } else { + return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","), DorisTableUtils.dorisName(first.getTableId())); + } + } + + private String convertUnionTypeToSQL(String unionType) { + switch (unionType) { + case "1:1": + return " INNER JOIN "; + case "1:N": + return " LEFT JOIN "; + case "N:1": + return " RIGHT JOIN "; + default: + return " INNER JOIN "; + } + } + public void saveTableField(DatasetTable datasetTable) throws Exception { Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); DataSetTableRequest dataSetTableRequest = new DataSetTableRequest(); @@ -341,12 +453,26 @@ public class DataSetTableService { // save field Map map = parseExcel(path.substring(path.lastIndexOf("/") + 1), new FileInputStream(file), false); fields = (List) map.get("fields"); - List> data = (List>) map.get("data"); - // save data +// List> data = (List>) map.get("data"); } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { // save field - - // save data + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + List list = dataTableInfoDTO.getList(); + List fieldList = new ArrayList<>(); + list.forEach(ele -> { + List listByIds = dataSetTableFieldsService.getListByIds(ele.getCheckedFields()); + fieldList.addAll(listByIds); + }); + for (int i = 0; i < fieldList.size(); i++) { + DatasetTableField datasetTableField = fieldList.get(i); + datasetTableField.setId(null); + datasetTableField.setTableId(datasetTable.getId()); + datasetTableField.setColumnIndex(i); + } + dataSetTableFieldsService.batchEdit(fieldList); + // custom 创建doris视图 + createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQL(dataTableInfoDTO, dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()))); + return; } if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { @@ -355,9 +481,9 @@ public class DataSetTableService { datasetTableField.setTableId(datasetTable.getId()); datasetTableField.setOriginName(filed.getFieldName()); datasetTableField.setName(filed.getRemarks()); - if(StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")){ + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { datasetTableField.setDataeaseName("C_" + Md5Utils.md5(filed.getFieldName())); - }else { + } else { datasetTableField.setDataeaseName(filed.getFieldName()); } datasetTableField.setType(filed.getFieldType()); @@ -375,6 +501,15 @@ public class DataSetTableService { } } + private void createDorisView(String dorisTableName, String customSql) throws Exception { + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(dorisDatasource); + datasourceRequest.setQuery("CREATE VIEW " + dorisTableName + " AS (" + customSql + ")"); + jdbcProvider.exec(datasourceRequest); + } + public String createQueryCountSQL(String type, String table) { DatasourceTypes datasourceType = DatasourceTypes.valueOf(type); switch (datasourceType) { @@ -480,7 +615,7 @@ public class DataSetTableService { public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) { - if(datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())){ + if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { return; } if (StringUtils.isEmpty(datasetTableIncrementalConfig.getId())) { diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 4b2fbe697e..51ed84a4a2 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -811,7 +811,9 @@ export default { target_table: '被关联表', target_field: '被关联字段', union_relation: '关联关系', - pls_setting_union_success: '请正确设置关联关系' + pls_setting_union_success: '请正确设置关联关系', + check_all: '全选', + can_not_union_self: '被关联表不能与关联表相同' }, datasource: { datasource: '数据源', diff --git a/frontend/src/views/dataset/add/AddCustom.vue b/frontend/src/views/dataset/add/AddCustom.vue index dd17397a4c..ce0ad58307 100644 --- a/frontend/src/views/dataset/add/AddCustom.vue +++ b/frontend/src/views/dataset/add/AddCustom.vue @@ -1,37 +1,48 @@ + + diff --git a/frontend/src/views/dataset/common/DatasetGroupSelector.vue b/frontend/src/views/dataset/common/DatasetGroupSelector.vue index 5ac1936be6..e82fda13e1 100644 --- a/frontend/src/views/dataset/common/DatasetGroupSelector.vue +++ b/frontend/src/views/dataset/common/DatasetGroupSelector.vue @@ -83,7 +83,7 @@ @node-click="sceneClick" > - + @@ -107,6 +107,23 @@ import { post } from '@/api/dataset/dataset' export default { name: 'DatasetGroupSelector', + props: { + mode: { + type: Number, + required: false, + default: -1 + }, + unionData: { + type: Array, + required: false, + default: null + }, + checkedList: { + type: Array, + required: false, + default: null + } + }, data() { return { sceneMode: false, @@ -132,10 +149,9 @@ export default { }, computed: {}, watch: { - // search(val){ - // this.groupForm.name = val; - // this.tree(this.groupForm); - // } + 'unionData': function() { + this.unionDataChange() + }, search(val) { if (val && val !== '') { this.tableData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.includes(val) }))) @@ -183,10 +199,15 @@ export default { if (this.currGroup) { post('/dataset/table/list', { sort: 'type asc,create_time desc,name asc', - sceneId: this.currGroup.id + sceneId: this.currGroup.id, + mode: this.mode < 0 ? null : this.mode }).then(response => { this.tables = response.data this.tableData = JSON.parse(JSON.stringify(this.tables)) + + this.$nextTick(function() { + this.unionDataChange() + }) }) } }, @@ -217,6 +238,40 @@ export default { sceneClick(data, node) { // console.log(data); this.$emit('getTable', data) + }, + + unionDataChange() { + if (!this.sceneMode) { + return + } + if (!this.checkedList || this.checkedList.length === 0) { + this.tableData.forEach(ele => { + const span = document.getElementById(ele.id).parentNode + const div1 = span.parentNode + const div2 = div1.parentNode + span.style.removeProperty('color') + div1.style.removeProperty('cursor') + div2.style.removeProperty('pointer-events') + }) + return + } + const tableList = this.tableData.map(ele => { + return ele.id + }) + const unionList = this.unionData.map(ele => { + return ele.targetTableId + }) + unionList.push(this.checkedList[0].tableId) + const notUnionList = tableList.concat(unionList).filter(v => tableList.includes(v) && !unionList.includes(v)) + + notUnionList.forEach(ele => { + const span = document.getElementById(ele).parentNode + const div1 = span.parentNode + const div2 = div1.parentNode + span.style.setProperty('color', '#c0c4cc') + div1.style.setProperty('cursor', 'not-allowed') + div2.style.setProperty('pointer-events', 'none') + }) } } } diff --git a/frontend/src/views/dataset/data/UnionView.vue b/frontend/src/views/dataset/data/UnionView.vue index ea3ab7eac6..3f5091768d 100644 --- a/frontend/src/views/dataset/data/UnionView.vue +++ b/frontend/src/views/dataset/data/UnionView.vue @@ -90,7 +90,7 @@ width="500" trigger="click" > - + {{ targetTable.name || $t('dataset.pls_slc_union_table') }} @@ -169,10 +169,12 @@ export default { }, methods: { initUnion() { - post('dataset/union/listByTableId/' + this.table.id, {}).then(response => { - // console.log(response) - this.unionData = response.data - }) + if (this.table.id) { + post('dataset/union/listByTableId/' + this.table.id, {}).then(response => { + // console.log(response) + this.unionData = response.data + }) + } }, showUnionEdit() { @@ -183,7 +185,7 @@ export default { this.editUnion = true }, saveUnion() { - console.log(this.union) + // console.log(this.union) if (!this.union.sourceTableFieldId || !this.union.sourceUnionRelation || !this.union.targetTableId || !this.union.targetTableFieldId) { this.$message({ type: 'error', @@ -245,6 +247,14 @@ export default { }, getTable(param) { // console.log(param) + if (param.id === this.table.id) { + this.$message({ + type: 'error', + message: this.$t('dataset.can_not_union_self'), + showClose: true + }) + return + } this.targetTable = param this.union.targetTableId = param.id this.union.targetTableFieldId = '' diff --git a/frontend/src/views/dataset/data/ViewTable.vue b/frontend/src/views/dataset/data/ViewTable.vue index b833d1777c..0f759300f5 100644 --- a/frontend/src/views/dataset/data/ViewTable.vue +++ b/frontend/src/views/dataset/data/ViewTable.vue @@ -31,7 +31,7 @@ - + diff --git a/frontend/src/views/dataset/index.vue b/frontend/src/views/dataset/index.vue index b172585f4d..7c21db7241 100644 --- a/frontend/src/views/dataset/index.vue +++ b/frontend/src/views/dataset/index.vue @@ -76,6 +76,7 @@ export default { .ms-main-container { height: calc(100vh - 56px); + padding: 15px 15px 0 15px; }