diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 524b467ff2..dc24ed3a8f 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -120,6 +120,19 @@ public class ExtractDataService { " exit 1\n" + "fi\n"; + private static final String shellScriptForDeleteFile = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" + + "rm -rf %s \n" + + "if [ $? -eq 0 ] ; then\n" + + " failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" + + " if [ \"x${failstatus}\" != \"x\" ];then" + + " echo $result\n" + + " exit 1\n" + + " fi\n" + + "else\n" + + " echo $result\n" + + " exit 1\n" + + "fi\n"; + public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) { datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); @@ -445,14 +458,20 @@ public class ExtractDataService { String dataFile = null; String script = null; + String streamLoadScript = ""; + if(kettleFilesKeep){ + streamLoadScript = shellScript; + }else { + streamLoadScript = shellScriptForDeleteFile; + } switch (extractType) { case "all_scope": dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()))); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()))); break; default: dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId())); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId())); break; } @@ -744,7 +763,6 @@ public class ExtractDataService { DataEaseException.throwException(transStatus.getLoggingString()); return; } - executing = true; String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null); SlaveServerJobStatus jobStatus = null; @@ -759,7 +777,7 @@ public class ExtractDataService { if (jobStatus.getStatusDescription().equals("Finished")) { return; } else { - DataEaseException.throwException((jobStatus.getLoggingString())); + DataEaseException.throwException(jobStatus.getLoggingString()); } } @@ -773,20 +791,26 @@ public class ExtractDataService { Datasource dorisDatasource = engineService.getDeEngine(); DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); String columns = columnFields + ",dataease_uuid"; + String streamLoadScript = ""; + if(kettleFilesKeep){ + streamLoadScript = shellScript; + }else { + streamLoadScript = shellScriptForDeleteFile; + } switch (extractType) { case "all_scope": outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())); jobName = "job_" + TableUtils.tableName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention); break; case "incremental_add": outFile = TableUtils.addName(datasetTable.getId()); jobName = "job_add_" + TableUtils.tableName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); break; case "incremental_delete": outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId())); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId()); break; default: diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index c557158fd2..70afbfe964 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -277,6 +277,7 @@ export default { this.form = JSON.parse(JSON.stringify(row)) this.originConfiguration = this.form.configuration if (row.type === 'api') { + } else { this.form.configuration = JSON.parse(this.form.configuration) } @@ -494,7 +495,9 @@ export default { changeType() { for (let i = 0; i < this.dsTypes.length; i++) { if (this.dsTypes[i].type === this.form.type) { - this.form.configuration.extraParams = this.dsTypes[i].extraParams + if(row.type !== 'api'){ + this.form.configuration.extraParams = this.dsTypes[i].extraParams + } this.datasourceType = this.dsTypes[i] } }