From 47476240404a7a55a2f1fd0e47bffc194460e1c5 Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 25 Feb 2022 10:32:36 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E8=8E=B7=E5=8F=96API=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=BA=90=E4=B8=8B=E8=A1=A8=E6=8E=A5=E5=8F=A3=E6=8A=A5?= =?UTF-8?q?=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/datasource/DatasourceService.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index f40eb5ef1b..b71b93638b 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -6,9 +6,11 @@ import com.google.gson.Gson; import com.jayway.jsonpath.JsonPath; import io.dataease.auth.annotation.DeCleaner; import io.dataease.base.domain.*; -import io.dataease.base.mapper.*; +import io.dataease.base.mapper.DatasetTableMapper; +import io.dataease.base.mapper.DatasourceMapper; import io.dataease.base.mapper.ext.ExtDataSourceMapper; import io.dataease.base.mapper.ext.query.GridExample; +import io.dataease.commons.constants.DatasourceTypes; import io.dataease.commons.constants.DePermissionType; import io.dataease.commons.exception.DEException; import io.dataease.commons.model.AuthURD; @@ -18,21 +20,19 @@ import io.dataease.commons.utils.LogUtil; import io.dataease.controller.ResultHolder; import io.dataease.controller.request.DatasourceUnionRequest; import io.dataease.controller.request.datasource.ApiDefinition; +import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.controller.sys.base.ConditionEntity; -import io.dataease.commons.constants.DatasourceTypes; -import io.dataease.provider.datasource.ApiProvider; -import io.dataease.provider.datasource.DatasourceProvider; -import io.dataease.provider.ProviderFactory; -import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.dto.DatasourceDTO; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.dto.datasource.*; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; +import io.dataease.provider.ProviderFactory; +import io.dataease.provider.datasource.ApiProvider; +import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.service.dataset.DataSetGroupService; import io.dataease.service.message.DeMsgutil; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -276,7 +276,7 @@ public class DatasourceService { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - if(!datasource.getType().equalsIgnoreCase("api")){ + if(!ds.getType().equalsIgnoreCase("api")){ datasourceProvider.checkStatus(datasourceRequest); } @@ -284,12 +284,12 @@ public class DatasourceService { // 获取当前数据源下的db、api类型数据集 DatasetTableExample datasetTableExample = new DatasetTableExample(); - datasetTableExample.createCriteria().andTypeIn(Arrays.asList("db","api")).andDataSourceIdEqualTo(datasource.getId()); + datasetTableExample.createCriteria().andTypeIn(Arrays.asList("db","api")).andDataSourceIdEqualTo(ds.getId()); List datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample); List list = new ArrayList<>(); for (TableDesc tableDesc : tables) { DBTableDTO dbTableDTO = new DBTableDTO(); - dbTableDTO.setDatasourceId(datasource.getId()); + dbTableDTO.setDatasourceId(ds.getId()); dbTableDTO.setName(tableDesc.getName()); dbTableDTO.setRemark(tableDesc.getRemark()); dbTableDTO.setEnableCheck(true); @@ -415,8 +415,8 @@ public class DatasourceService { if (!StringUtils.equals(temp.getStatus(), "Error")) { sendWebMsg(datasource); datasourceMapper.updateByPrimaryKeySelective(datasource); - } - + } + } } From 724cd0633f216863fde7de58e56a5aefee85c051 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Fri, 25 Feb 2022 13:33:44 +0800 Subject: [PATCH 2/4] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E9=95=9C=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a7dd1fd772..ae8731a389 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.cn-qingdao.aliyuncs.com/dataease/fabric8-java-alpine-openjdk8-jre:edge +FROM registry.cn-qingdao.aliyuncs.com/dataease/fabric8-java-alpine-openjdk8-jre:edge-chromium # RUN echo -e 'https://dl-cdn.alpinelinux.org/alpine/edge/main/\nhttps://dl-cdn.alpinelinux.org/alpine/edge/community/' > /etc/apk/repositories From 386cf68cd13b2f6d085c4d6c57422c16f38fa363 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Fri, 25 Feb 2022 13:33:44 +0800 Subject: [PATCH 3/4] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E9=95=9C=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a7dd1fd772..ae8731a389 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.cn-qingdao.aliyuncs.com/dataease/fabric8-java-alpine-openjdk8-jre:edge +FROM registry.cn-qingdao.aliyuncs.com/dataease/fabric8-java-alpine-openjdk8-jre:edge-chromium # RUN echo -e 'https://dl-cdn.alpinelinux.org/alpine/edge/main/\nhttps://dl-cdn.alpinelinux.org/alpine/edge/community/' > /etc/apk/repositories From 03a2e922544d6b466c9e50a25e75d7006a2a1a8d Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 25 Feb 2022 17:32:17 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20api=20=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E8=A1=A5=E5=85=A8=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/io/dataease/auth/api/demo.java | 11 +++----- .../commons/utils/HttpClientUtil.java | 21 ++++++++------- .../provider/datasource/ApiProvider.java | 16 +++++------- .../service/datasource/DatasourceService.java | 26 +++++++++++-------- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/backend/src/main/java/io/dataease/auth/api/demo.java b/backend/src/main/java/io/dataease/auth/api/demo.java index 1fcab1b099..04d7ce9c57 100644 --- a/backend/src/main/java/io/dataease/auth/api/demo.java +++ b/backend/src/main/java/io/dataease/auth/api/demo.java @@ -3,13 +3,10 @@ package io.dataease.auth.api; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; -import com.github.xiaoymin.knife4j.annotations.ApiSupport; -import io.dataease.base.domain.DatasetTableFunction; -import io.swagger.annotations.Api; + import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.*; -import java.util.List; @RestController @@ -19,10 +16,10 @@ public class demo { @GetMapping("demo") public Object listByTableId() { JSONArray jsonArray = new JSONArray(); - for(int i=0;i<100;i++){ + for(int i=0;i<10;i++){ JSONObject jsonObject = new JSONObject(); - for(int j=0;j<10;j++){ - jsonObject.set("column" + j, "value"+j); + for(int j=0;j<1;j++){ + jsonObject.set("column" +i + j, "value"+i+j); } jsonArray.put(jsonObject); } diff --git a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java index d00974d658..fb71455a64 100755 --- a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java +++ b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java @@ -1,8 +1,8 @@ package io.dataease.commons.utils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.UrlEncodedFormEntity; @@ -90,8 +90,7 @@ public class HttpClientUtil { httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset()); HttpResponse response = httpClient.execute(httpGet); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -136,8 +135,7 @@ public class HttpClientUtil { httpPost.setEntity(requestEntity); HttpResponse response = httpClient.execute(httpPost); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -198,8 +196,7 @@ public class HttpClientUtil { } HttpResponse response = httpClient.execute(httpPost); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -212,10 +209,14 @@ public class HttpClientUtil { } } - private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{ + private static String getResponseStr(HttpResponse response, HttpClientConfig config) throws Exception{ if(response.getStatusLine().getStatusCode() >= 400){ - throw new Exception(EntityUtils.toString(entity, config.getCharset())); + String msg = EntityUtils.toString(response.getEntity(), config.getCharset()); + if(StringUtils.isEmpty(msg)){ + msg = "StatusCode: " + response.getStatusLine().getStatusCode(); + } + throw new Exception(msg); } - return EntityUtils.toString(entity, config.getCharset()); + return EntityUtils.toString(response.getEntity(), config.getCharset()); } } diff --git a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java index 42a800f1c5..56ebf17f63 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java @@ -28,7 +28,7 @@ public class ApiProvider extends DatasourceProvider{ public List getData(DatasourceRequest datasourceRequest) throws Exception { ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); String response = execHttpRequest(apiDefinition); - return fetchResult(response, apiDefinition.getDataPath()); + return fetchResult(response, apiDefinition); } @Override @@ -64,7 +64,7 @@ public class ApiProvider extends DatasourceProvider{ fieldList = getTableFileds(datasourceRequest); result.put("fieldList", fieldList); - dataList = fetchResult(response, apiDefinition.getDataPath()); + dataList = fetchResult(response, apiDefinition); result.put("dataList", dataList); return result; } @@ -170,16 +170,14 @@ public class ApiProvider extends DatasourceProvider{ return response; } - private List fetchResult(String result, String path){ + private List fetchResult(String result, ApiDefinition apiDefinition){ List dataList = new LinkedList<>(); - List datas = JsonPath.read(result, path); + List datas = JsonPath.read(result, apiDefinition.getDataPath()); for (LinkedHashMap data : datas) { - String[] row = new String[data.entrySet().size()]; - Iterator it = data.entrySet().iterator(); + String[] row = new String[apiDefinition.getFields().size()]; int i = 0; - while (it.hasNext()){ - Map.Entry entry = (Map.Entry)it.next(); - row[i] = Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); + for (DatasetTableField field : apiDefinition.getFields()) { + row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); i++; } dataList.add(row); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index b71b93638b..7a20a5bd39 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -362,18 +362,16 @@ public class DatasourceService { List dataList = new ArrayList<>(); List fields = new ArrayList<>(); - Boolean getFileds = true; - + Set fieldKeys = new HashSet<>(); + //第一遍获取 field for (LinkedHashMap data : datas) { - JSONObject jsonObject = new JSONObject(); - Iterator it = data.entrySet().iterator(); - while (it.hasNext()){ - Map.Entry entry = (Map.Entry)it.next(); - jsonObject.put((String) entry.getKey(), Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); - if(getFileds) { + Set keys = data.keySet(); + for (String key : keys) { + if(!fieldKeys.contains(key)){ + fieldKeys.add(key); DatasetTableField tableField = new DatasetTableField(); - tableField.setOriginName((String) entry.getKey()); - tableField.setName((String) entry.getKey()); + tableField.setOriginName(key); + tableField.setName(key); tableField.setSize(65535); tableField.setDeExtractType(0); tableField.setDeType(0); @@ -381,7 +379,13 @@ public class DatasourceService { fields.add(tableField); } } - getFileds = false; + } + //第二遍获取 data + for (LinkedHashMap data : datas) { + JSONObject jsonObject = new JSONObject(); + for (String key : fieldKeys) { + jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); + } dataList.add(jsonObject); } apiDefinition.setDatas(dataList);