diff --git a/backend/src/main/java/io/dataease/auth/api/demo.java b/backend/src/main/java/io/dataease/auth/api/demo.java index 1fcab1b099..04d7ce9c57 100644 --- a/backend/src/main/java/io/dataease/auth/api/demo.java +++ b/backend/src/main/java/io/dataease/auth/api/demo.java @@ -3,13 +3,10 @@ package io.dataease.auth.api; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; -import com.github.xiaoymin.knife4j.annotations.ApiSupport; -import io.dataease.base.domain.DatasetTableFunction; -import io.swagger.annotations.Api; + import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.*; -import java.util.List; @RestController @@ -19,10 +16,10 @@ public class demo { @GetMapping("demo") public Object listByTableId() { JSONArray jsonArray = new JSONArray(); - for(int i=0;i<100;i++){ + for(int i=0;i<10;i++){ JSONObject jsonObject = new JSONObject(); - for(int j=0;j<10;j++){ - jsonObject.set("column" + j, "value"+j); + for(int j=0;j<1;j++){ + jsonObject.set("column" +i + j, "value"+i+j); } jsonArray.put(jsonObject); } diff --git a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java index d00974d658..fb71455a64 100755 --- a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java +++ b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java @@ -1,8 +1,8 @@ package io.dataease.commons.utils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.UrlEncodedFormEntity; @@ -90,8 +90,7 @@ public class HttpClientUtil { httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset()); HttpResponse response = httpClient.execute(httpGet); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -136,8 +135,7 @@ public class HttpClientUtil { httpPost.setEntity(requestEntity); HttpResponse response = httpClient.execute(httpPost); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -198,8 +196,7 @@ public class HttpClientUtil { } HttpResponse response = httpClient.execute(httpPost); - HttpEntity entity = response.getEntity(); - return getResponseStr(response, entity, config); + return getResponseStr(response, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); @@ -212,10 +209,14 @@ public class HttpClientUtil { } } - private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{ + private static String getResponseStr(HttpResponse response, HttpClientConfig config) throws Exception{ if(response.getStatusLine().getStatusCode() >= 400){ - throw new Exception(EntityUtils.toString(entity, config.getCharset())); + String msg = EntityUtils.toString(response.getEntity(), config.getCharset()); + if(StringUtils.isEmpty(msg)){ + msg = "StatusCode: " + response.getStatusLine().getStatusCode(); + } + throw new Exception(msg); } - return EntityUtils.toString(entity, config.getCharset()); + return EntityUtils.toString(response.getEntity(), config.getCharset()); } } diff --git a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java index 42a800f1c5..56ebf17f63 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java @@ -28,7 +28,7 @@ public class ApiProvider extends DatasourceProvider{ public List getData(DatasourceRequest datasourceRequest) throws Exception { ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); String response = execHttpRequest(apiDefinition); - return fetchResult(response, apiDefinition.getDataPath()); + return fetchResult(response, apiDefinition); } @Override @@ -64,7 +64,7 @@ public class ApiProvider extends DatasourceProvider{ fieldList = getTableFileds(datasourceRequest); result.put("fieldList", fieldList); - dataList = fetchResult(response, apiDefinition.getDataPath()); + dataList = fetchResult(response, apiDefinition); result.put("dataList", dataList); return result; } @@ -170,16 +170,14 @@ public class ApiProvider extends DatasourceProvider{ return response; } - private List fetchResult(String result, String path){ + private List fetchResult(String result, ApiDefinition apiDefinition){ List dataList = new LinkedList<>(); - List datas = JsonPath.read(result, path); + List datas = JsonPath.read(result, apiDefinition.getDataPath()); for (LinkedHashMap data : datas) { - String[] row = new String[data.entrySet().size()]; - Iterator it = data.entrySet().iterator(); + String[] row = new String[apiDefinition.getFields().size()]; int i = 0; - while (it.hasNext()){ - Map.Entry entry = (Map.Entry)it.next(); - row[i] = Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); + for (DatasetTableField field : apiDefinition.getFields()) { + row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); i++; } dataList.add(row); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index b71b93638b..7a20a5bd39 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -362,18 +362,16 @@ public class DatasourceService { List dataList = new ArrayList<>(); List fields = new ArrayList<>(); - Boolean getFileds = true; - + Set fieldKeys = new HashSet<>(); + //第一遍获取 field for (LinkedHashMap data : datas) { - JSONObject jsonObject = new JSONObject(); - Iterator it = data.entrySet().iterator(); - while (it.hasNext()){ - Map.Entry entry = (Map.Entry)it.next(); - jsonObject.put((String) entry.getKey(), Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); - if(getFileds) { + Set keys = data.keySet(); + for (String key : keys) { + if(!fieldKeys.contains(key)){ + fieldKeys.add(key); DatasetTableField tableField = new DatasetTableField(); - tableField.setOriginName((String) entry.getKey()); - tableField.setName((String) entry.getKey()); + tableField.setOriginName(key); + tableField.setName(key); tableField.setSize(65535); tableField.setDeExtractType(0); tableField.setDeType(0); @@ -381,7 +379,13 @@ public class DatasourceService { fields.add(tableField); } } - getFileds = false; + } + //第二遍获取 data + for (LinkedHashMap data : datas) { + JSONObject jsonObject = new JSONObject(); + for (String key : fieldKeys) { + jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); + } dataList.add(jsonObject); } apiDefinition.setDatas(dataList);