dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 定时同步任务支持在历史数据变动后增量更新到doris中

Browse Source
This commit is contained in:
taojinlong 2024-01-24 16:51:45 +08:00
parent 42c85bdc82
commit 3df5e3e2e9
3 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java
View File

@ -370,6 +370,9 @@ public class ExcelXlsReader implements HSSFListener {
private String checkType(String str, int thisColumn) { private String checkType(String str, int thisColumn) {
if (str.length() > 19) {
return "TEXT";
}
String type = null; String type = null;
try { try {
double d = Double.valueOf(str); double d = Double.valueOf(str);

3
core/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2949,6 +2949,9 @@ public class DataSetTableService {
} }
private String cellType(String value) { private String cellType(String value) {
if (value.length() > 19) {
return "TEXT";
}
try { try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sdf.parse(value); sdf.parse(value);

16
core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -479,9 +479,10 @@ public class ExtractDataService {
} }
private void extractApiData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType) throws Exception { private void extractApiData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<ApiDefinition> lists = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() { List<ApiDefinition> lists = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() {
}.getType()); }.getType());
lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList()); lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(dataTableInfoDTO.getTable())).collect(Collectors.toList());
if (CollectionUtils.isEmpty(lists)) { if (CollectionUtils.isEmpty(lists)) {
throw new Exception("未找到API数据表"); throw new Exception("未找到API数据表");
} }
@ -521,15 +522,16 @@ public class ExtractDataService {
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), dataFile); script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), dataFile);
break; break;
} }
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile)); BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) { for (String[] strings : dataList) {
String content = ""; String content = "";
for (int i = 0; i < strings.length; i++) { for (int i = 0; i < strings.length; i++) {
content = content + strings[i] + separator; content = i != strings.length - 1 ? content + strings[i] + separator : content + strings[i];
}
boolean isSetKey = dataTableInfoDTO.isSetKey() && CollectionUtils.isNotEmpty(dataTableInfoDTO.getKeys());
if (!isSetKey) {
content = Md5Utils.md5(content) + separator + content;
} }
content = content + Md5Utils.md5(content);
bw.write(content); bw.write(content);
bw.newLine(); bw.newLine();
} }
@ -565,8 +567,8 @@ public class ExtractDataService {
} catch (Exception e) { } catch (Exception e) {
throw e; throw e;
} finally { } finally {
File deleteFile = new File(root_path + datasetTable.getId() + ".sh"); // File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
FileUtils.forceDelete(deleteFile); // FileUtils.forceDelete(deleteFile);
} }
} }