package io.dataease.provider.datasource; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.JsonObject; import io.dataease.controller.sys.response.BasicInfo; import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.plugins.common.dto.datasource.TableDesc; import io.dataease.plugins.common.dto.datasource.TableField; import io.dataease.plugins.common.request.datasource.DatasourceRequest; import io.dataease.plugins.datasource.provider.Provider; import com.jayway.jsonpath.JsonPath; import io.dataease.commons.utils.HttpClientConfig; import io.dataease.commons.utils.HttpClientUtil; import io.dataease.controller.request.datasource.ApiDefinition; import io.dataease.controller.request.datasource.ApiDefinitionRequest; import io.dataease.service.system.SystemParameterService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.*; import java.util.stream.Collectors; @Service("apiProvider") public class ApiProvider extends Provider { @Resource private SystemParameterService systemParameterService; @Override public List getData(DatasourceRequest datasourceRequest) throws Exception { BasicInfo basicInfo = systemParameterService.basicInfo(); ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); String response = execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10); return fetchResult(response, apiDefinition); } @Override public List getTables(DatasourceRequest datasourceRequest) throws Exception { List tableDescs = new ArrayList<>(); List lists = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken>() { }.getType()); for (ApiDefinition apiDefinition : lists) { TableDesc tableDesc = new TableDesc(); tableDesc.setName(apiDefinition.getName()); tableDesc.setRemark(apiDefinition.getDesc()); tableDescs.add(tableDesc); } return tableDescs; } @Override public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { return null; } public Map fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception { BasicInfo basicInfo = systemParameterService.basicInfo(); Map result = new HashMap<>(); List dataList = new ArrayList<>(); List fieldList = new ArrayList<>(); ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); String response = execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10); fieldList = getTableFileds(apiDefinition); result.put("fieldList", fieldList); dataList = fetchResult(response, apiDefinition); result.put("dataList", dataList); return result; } private List getTableFileds(ApiDefinition apiDefinition) throws Exception { List tableFields = new ArrayList<>(); for (DatasetTableFieldDTO field : apiDefinition.getFields()) { TableField tableField = new TableField(); tableField.setFieldName(field.getName()); tableField.setRemarks(field.getName()); tableField.setFieldSize(field.getSize()); tableField.setFieldType(field.getDeExtractType().toString()); tableFields.add(tableField); } return tableFields; } public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { List lists = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken>() { }.getType()); List tableFields = new ArrayList<>(); for (ApiDefinition apiDefinition : lists) { if (datasourceRequest.getTable().equalsIgnoreCase(apiDefinition.getName())) { tableFields = getTableFileds(apiDefinition); } } return tableFields; } public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { Gson gson = new Gson(); List apiDefinitionList = gson.fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken>() { }.getType()); JsonObject apiItemStatuses = new JsonObject(); for (ApiDefinition apiDefinition : apiDefinitionList) { datasourceRequest.setTable(apiDefinition.getName()); try { getData(datasourceRequest); apiItemStatuses.addProperty(apiDefinition.getName(), "Success"); } catch (Exception ignore) { apiItemStatuses.addProperty(apiDefinition.getName(), "Error"); } } return gson.toJson(apiItemStatuses); } static public String execHttpRequest(ApiDefinition apiDefinition, int socketTimeout) throws Exception { String response = ""; HttpClientConfig httpClientConfig = new HttpClientConfig(); httpClientConfig.setSocketTimeout(socketTimeout * 1000); ApiDefinitionRequest apiDefinitionRequest = apiDefinition.getRequest(); for (Map header : apiDefinitionRequest.getHeaders()) { if (header.get("name") != null && StringUtils.isNotEmpty(header.get("name").toString()) && header.get("value") != null && StringUtils.isNotEmpty(header.get("value").toString())) { httpClientConfig.addHeader(header.get("name").toString(), header.get("value").toString()); } } if (apiDefinitionRequest.getAuthManager() != null && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getUsername()) && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getPassword()) && apiDefinitionRequest.getAuthManager().getVerification().equals("Basic Auth")) { String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((apiDefinitionRequest.getAuthManager().getUsername() + ":" + apiDefinitionRequest.getAuthManager().getPassword()).getBytes()); httpClientConfig.addHeader("Authorization", authValue); } switch (apiDefinition.getMethod()) { case "GET": response = HttpClientUtil.get(apiDefinition.getUrl().trim(), httpClientConfig); break; case "POST": if (apiDefinitionRequest.getBody().get("type") == null) { throw new Exception("请求类型不能为空"); } String type = apiDefinitionRequest.getBody().get("type").toString(); if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) { String raw = null; if (apiDefinitionRequest.getBody().get("raw") != null) { raw = apiDefinitionRequest.getBody().get("raw").toString(); response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig); } } if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) { if (apiDefinitionRequest.getBody().get("kvs") != null) { Map body = new HashMap<>(); JSONObject bodyObj = JSONObject.parseObject(apiDefinitionRequest.getBody().toString()); JSONArray kvsArr = bodyObj.getJSONArray("kvs"); for (int i = 0; i < kvsArr.size(); i++) { JSONObject kv = kvsArr.getJSONObject(i); if (kv.containsKey("name")) { body.put(kv.getString("name"), kv.getString("value")); } } response = HttpClientUtil.post(apiDefinition.getUrl(), body, httpClientConfig); } } break; default: break; } return response; } static public ApiDefinition checkApiDefinition(ApiDefinition apiDefinition, String response) throws Exception { if (StringUtils.isEmpty(response)) { throw new Exception("该请求返回数据为空"); } List fields = new ArrayList<>(); String rootPath; if (response.startsWith("[")) { rootPath = "$[*]"; JSONArray jsonArray = JSONObject.parseArray(response); for (Object o : jsonArray) { handleStr(apiDefinition, o.toString(), fields, rootPath); } } else { rootPath = "$"; handleStr(apiDefinition, response, fields, rootPath); } for (JSONObject field : fields) { if(field.containsKey("children") && CollectionUtils.isNotEmpty(field.getJSONArray("children"))){ field.put("disabled", false); } if(field.containsKey("children") && CollectionUtils.isEmpty(field.getJSONArray("children"))){ field.put("disabled", true); } } apiDefinition.setJsonFields(fields); return apiDefinition; } static private void handleStr(ApiDefinition apiDefinition, String jsonStr, List fields, String rootPath) { if (jsonStr.startsWith("[")) { JSONArray jsonArray = JSONObject.parseArray(jsonStr); for (Object o : jsonArray) { handleStr(apiDefinition, o.toString(), fields, rootPath); } } else { JSONObject jsonObject = JSONObject.parseObject(jsonStr); for (String s : jsonObject.keySet()) { String value = jsonObject.getString(s); if (StringUtils.isNotEmpty(value) && value.startsWith("[")) { JSONObject o = new JSONObject(); try { JSONArray jsonArray = jsonObject.getJSONArray(s); List childrenField = new ArrayList<>(); for (Object object : jsonArray) { JSONObject.parseObject(object.toString()); handleStr(apiDefinition, JSON.toJSONString(object, SerializerFeature.WriteMapNullValue), childrenField, rootPath + "." + s + "[*]"); } o.put("children", childrenField); o.put("childrenDataType", "LIST"); } catch (Exception e) { JSONArray array = new JSONArray(); array.add(StringUtils.isNotEmpty(jsonObject.getString(s)) ? jsonObject.getString(s) : ""); o.put("value", array); } o.put("jsonPath", rootPath + "." + s); setProperty(apiDefinition, o, s); if (!hasItem(apiDefinition, fields, o)) { fields.add(o); } } else if (StringUtils.isNotEmpty(value) && value.startsWith("{")) { List children = new ArrayList<>(); handleStr(apiDefinition, jsonObject.getString(s), children, rootPath + "." + s); JSONObject o = new JSONObject(); o.put("children", children); o.put("childrenDataType", "OBJECT"); o.put("jsonPath", rootPath + "." + s); setProperty(apiDefinition, o, s); if (!hasItem(apiDefinition, fields, o)) { fields.add(o); } } else { JSONObject o = new JSONObject(); o.put("jsonPath", rootPath + "." + s); setProperty(apiDefinition, o, s); JSONArray array = new JSONArray(); array.add(StringUtils.isNotEmpty(jsonObject.getString(s)) ? jsonObject.getString(s) : ""); o.put("value", array); if (!hasItem(apiDefinition, fields, o)) { fields.add(o); } } } } } static private void setProperty(ApiDefinition apiDefinition, JSONObject o, String s) { o.put("originName", s); o.put("name", s); o.put("type", "STRING"); o.put("checked", false); o.put("size", 65535); o.put("deExtractType", 0); o.put("deType", 0); o.put("extField", 0); o.put("checked", false); for (DatasetTableFieldDTO fieldDTO : apiDefinition.getFields()) { if (StringUtils.isNotEmpty(o.getString("jsonPath")) && StringUtils.isNotEmpty(fieldDTO.getJsonPath()) && fieldDTO.getJsonPath().equals(o.getString("jsonPath"))) { o.put("checked", true); o.put("deExtractType", fieldDTO.getDeExtractType()); o.put("name", fieldDTO.getName()); } } } static private boolean hasItem(ApiDefinition apiDefinition, List fields, JSONObject item) { boolean has = false; for (JSONObject field : fields) { if (field.getString("jsonPath").equals(item.getString("jsonPath"))) { has = true; mergeField(field, item); mergeValue(field, apiDefinition, item); break; } } return has; } static void mergeField(JSONObject field, JSONObject item) { if (item.getJSONArray("children") != null) { JSONArray itemChildren = item.getJSONArray("children"); JSONArray fieldChildren = field.getJSONArray("children"); if (fieldChildren == null) { fieldChildren = new JSONArray(); } for (Object itemChild : itemChildren) { boolean hasKey = false; JSONObject itemChildObject = JSONObject.parseObject(itemChild.toString()); for (Object fieldChild : fieldChildren) { JSONObject fieldChildObject = JSONObject.parseObject(fieldChild.toString()); if (itemChildObject.getString("jsonPath").equals(fieldChildObject.getString("jsonPath"))) { mergeField(fieldChildObject, itemChildObject); hasKey = true; } } if (!hasKey) { fieldChildren.add(itemChild); } } } } static void mergeValue(JSONObject field, ApiDefinition apiDefinition, JSONObject item) { JSONArray array = field.getJSONArray("value"); if (array != null && item.getString("value") != null && array.size() < apiDefinition.getPreviewNum()) { array.add(item.getJSONArray("value").get(0).toString()); field.put("value", array); } if (CollectionUtils.isNotEmpty(field.getJSONArray("children")) && CollectionUtils.isNotEmpty(item.getJSONArray("children"))) { JSONArray fieldChildren = field.getJSONArray("children"); JSONArray itemChildren = item.getJSONArray("children"); JSONArray fieldArrayChildren = new JSONArray(); for (Object fieldChild : fieldChildren) { JSONObject jsonObject = JSONObject.parseObject(fieldChild.toString()); JSONObject find = null; for (Object itemChild : itemChildren) { JSONObject itemObject = JSONObject.parseObject(itemChild.toString()); if (jsonObject.getString("jsonPath").equals(itemObject.getString("jsonPath"))) { find = itemObject; } } if (find != null) { mergeValue(jsonObject, apiDefinition, find); } fieldArrayChildren.add(jsonObject); } field.put("children", fieldArrayChildren); } } private List fetchResult(String result, ApiDefinition apiDefinition) { List dataList = new LinkedList<>(); if (StringUtils.isNotEmpty(apiDefinition.getDataPath()) && CollectionUtils.isEmpty(apiDefinition.getJsonFields())) { List datas = new ArrayList<>(); Object object = JsonPath.read(result, apiDefinition.getDataPath()); if (object instanceof List) { datas = (List) object; } else { datas.add((LinkedHashMap) object); } for (LinkedHashMap data : datas) { String[] row = new String[apiDefinition.getFields().size()]; int i = 0; for (DatasetTableFieldDTO field : apiDefinition.getFields()) { row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); i++; } dataList.add(row); } } else { List jsonPaths = apiDefinition.getFields().stream().map(DatasetTableFieldDTO::getJsonPath).collect(Collectors.toList()); Long maxLenth = 0l; List> columnDataList = new ArrayList<>(); for (int i = 0; i < jsonPaths.size(); i++) { List datas = new ArrayList<>(); Object object = JsonPath.read(result, jsonPaths.get(i)); if (object instanceof List && jsonPaths.get(i).contains("[*]")) { datas = (List) object; } else { if (object != null) { datas.add(object.toString()); } } maxLenth = maxLenth > datas.size() ? maxLenth : datas.size(); columnDataList.add(datas); } for (int i = 0; i < maxLenth; i++) { String[] row = new String[apiDefinition.getFields().size()]; dataList.add(row); } for (int i = 0; i < columnDataList.size(); i++) { for (int j = 0; j < columnDataList.get(i).size(); j++) { dataList.get(j)[i] = Optional.ofNullable(String.valueOf(columnDataList.get(i).get(j))).orElse("").replaceAll("\n", " ").replaceAll("\r", " "); } } } return dataList; } private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws Exception { List apiDefinitionList = new ArrayList<>(); List apiDefinitionListTemp = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken>() { }.getType()); if (CollectionUtils.isNotEmpty(apiDefinitionListTemp)) { for (ApiDefinition apiDefinition : apiDefinitionListTemp) { if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) { apiDefinitionList.add(apiDefinition); } } } if (CollectionUtils.isEmpty(apiDefinitionList)) { throw new Exception("未找到API数据表"); } if (apiDefinitionList.size() > 1) { throw new Exception("存在重名的API数据表"); } for (ApiDefinition apiDefinition : apiDefinitionList) { if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) { return apiDefinition; } } throw new Exception("未找到API数据表"); } }