dongjunchuan/de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9011 from dataease/pr@dev@fixapi

Browse Source 
fix(数据集)：修复同步API数据集丢失数据
This commit is contained in:
taojinlong 2024-04-09 15:14:12 +08:00 committed by GitHub
commit 5a83fb568e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
core/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -522,31 +522,35 @@ public class ExtractDataService {
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), dataFile);
break;
}
BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
for (String[] strings : dataList) {
String content = "";
for (int i = 0; i < strings.length; i++) {
content = i != strings.length - 1 ? content + strings[i] + separator : content + strings[i];
try (BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile))) {
for (String[] strings : dataList) {
String content = "";
for (int i = 0; i < strings.length; i++) {
content = i != strings.length - 1 ? content + strings[i] + separator : content + strings[i];
}
boolean isSetKey = dataTableInfoDTO.isSetKey() && CollectionUtils.isNotEmpty(dataTableInfoDTO.getKeys());
if (!isSetKey) {
content = Md5Utils.md5(content) + separator + content;
}
bw.write(content);
bw.newLine();
}
boolean isSetKey = dataTableInfoDTO.isSetKey() && CollectionUtils.isNotEmpty(dataTableInfoDTO.getKeys());
if (!isSetKey) {
content = Md5Utils.md5(content) + separator + content;
}
bw.write(content);
bw.newLine();
} catch (Exception e) {
throw e;
}
bw.close();
File scriptFile = new File(root_path + datasetTable.getId() + ".sh");
scriptFile.createNewFile();
scriptFile.setExecutable(true);
try (BufferedWriter scriptFileBw = new BufferedWriter(new FileWriter(root_path + datasetTable.getId() + ".sh"))) {
scriptFileBw.write("#!/bin/sh");
scriptFileBw.newLine();
scriptFileBw.write(script);
scriptFileBw.newLine();
} catch (Exception e) {
throw e;
}
BufferedWriter scriptFileBw = new BufferedWriter(new FileWriter(root_path + datasetTable.getId() + ".sh"));
scriptFileBw.write("#!/bin/sh");
scriptFileBw.newLine();
scriptFileBw.write(script);
scriptFileBw.newLine();
scriptFileBw.close();
try {
Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh");
@ -794,7 +798,7 @@ public class ExtractDataService {
csvSheetData.setExcelLabel(excelSheetData.getExcelLabel());
csvSheetData.setFieldsMd5(Md5Utils.md5(StringUtils.join(fieldArray, ",")));
totalSheets = Arrays.asList(csvSheetData);
}else {
} else {
totalSheets = dataSetTableService.excelSheetDataList(new FileInputStream(excelSheetData.getPath()), false);
}
@ -1239,7 +1243,7 @@ public class ExtractDataService {
textFileField.setType("String");
outputFields[i] = textFileField;
}
if(!isSetKey){
if (!isSetKey) {
TextFileField textFileField = new TextFileField();
textFileField.setName("dataease_uuid");
textFileField.setType("String");
@ -1260,7 +1264,7 @@ public class ExtractDataService {
outputFields[i] = textFileField;
}
if(!isSetKey){
if (!isSetKey) {
TextFileField textFileField = new TextFileField();
textFileField.setName("dataease_uuid");
textFileField.setType("String");
@ -1281,7 +1285,7 @@ public class ExtractDataService {
outputFields[i] = textFileField;
}
if(!isSetKey){
if (!isSetKey) {
TextFileField textFileField = new TextFileField();
textFileField.setName("dataease_uuid");
textFileField.setType("String");