diff --git a/backend/src/main/java/io/dataease/provider/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java index 0e057c1285..f8ea8f174e 100644 --- a/backend/src/main/java/io/dataease/provider/QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java @@ -43,4 +43,8 @@ public abstract class QueryProvider { public abstract String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList); public abstract String wrapSql(String sql); + + public abstract String createRawQuerySQL(String table, List fields); + + public abstract String createRawQuerySQLAsTmp(String sql, List fields); } diff --git a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java index d9faed7c4d..ee6b7a02a4 100644 --- a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java @@ -348,6 +348,28 @@ public class DorisQueryProvider extends QueryProvider { return tmpSql; } + @Override + public String createRawQuerySQL(String table, List fields){ + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + public String transMysqlFilterTerm(String term) { switch (term) { case "eq": diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java index 00133dade4..e2cbb61fd7 100644 --- a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java @@ -102,8 +102,6 @@ public class MysqlQueryProvider extends QueryProvider { } return stringBuilder.toString(); }).toArray(String[]::new); - -// return MessageFormat.format("SELECT {0} FROM {1} ORDER BY " + (fields.size() > 0 ? fields.get(0).getOriginName() : "null"), StringUtils.join(array, ","), table); return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); } @@ -356,6 +354,21 @@ public class MysqlQueryProvider extends QueryProvider { return tmpSql; } + @Override + public String createRawQuerySQL(String table, List fields){ + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + } + public String transMysqlFilterTerm(String term) { switch (term) { case "eq": diff --git a/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java index 88c1454cc8..6cc986cb0c 100644 --- a/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java @@ -380,6 +380,31 @@ public class OracleQueryProvider extends QueryProvider { return getSQLSummary(" (" + sqlFix(sql) + ") tmp ", yAxis, customFilter, extFilterRequestList); } + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0"; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields){ + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" ").append(f.getOriginName()).append(" AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") tmp ", fields); + } + public String transMysqlFilterTerm(String term) { switch (term) { case "eq": @@ -490,16 +515,6 @@ public class OracleQueryProvider extends QueryProvider { return sql; } - @Override - public String wrapSql(String sql) { - sql = sql.trim(); - if (sql.lastIndexOf(";") == (sql.length() - 1)) { - sql = sql.substring(0, sql.length() - 1); - } - String tmpSql = "SELECT * FROM (" + sql + ") tmp " + " where rownum <= 0"; - return tmpSql; - } - private String transDateFormat(String dateStyle, String datePattern) { String split = "-"; if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { diff --git a/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java index 0e3d09bd96..a0005fefc5 100644 --- a/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java @@ -270,6 +270,21 @@ public class SqlserverQueryProvider extends QueryProvider { return tmpSql; } + @Override + public String createRawQuerySQL(String table, List fields){ + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + } + public String transMysqlFilterTerm(String term) { switch (term) { case "eq": diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 0a2bd2f8ca..2a856f21b9 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -206,13 +206,13 @@ public class ExtractDataService { extractData(datasetTable, "all_scope"); replaceTable(DorisTableUtils.dorisName(datasetTableId)); saveSucessLog(datasetTableTaskLog); - deleteFile("all_scope", datasetTableId); +// deleteFile("all_scope", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); - deleteFile("all_scope", datasetTableId); +// deleteFile("all_scope", datasetTableId); }finally { if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); @@ -267,15 +267,15 @@ public class ExtractDataService { extractData(datasetTable, "incremental_delete"); } saveSucessLog(datasetTableTaskLog); - deleteFile("incremental_add", datasetTableId); - deleteFile("incremental_delete", datasetTableId); +// deleteFile("incremental_add", datasetTableId); +// deleteFile("incremental_delete", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } }catch (Exception e){ saveErrorLog(datasetTableId, taskId, e); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); - deleteFile("incremental_add", datasetTableId); - deleteFile("incremental_delete", datasetTableId); +// deleteFile("incremental_add", datasetTableId); +// deleteFile("incremental_delete", datasetTableId); }finally { if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTask.setRate(ScheduleType.SIMPLE_COMPLETE.toString()); @@ -610,15 +610,7 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword()); dataMeta.addExtraOption("MYSQL","characterEncoding", "UTF-8"); transMeta.addDatabase(dataMeta); - if (extractType.equalsIgnoreCase("all_scope")) { - if(datasetTable.getType().equalsIgnoreCase("sql")){ - selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); - }else { - String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); - QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createQuerySQL(tableName, datasetTableFields); - } - } + selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputStep = inputStep(transMeta, selectSQL); udjcStep = udjc(datasetTableFields, false); break; @@ -626,15 +618,7 @@ public class ExtractDataService { SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfigration.class); dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfigration.getHost(), sqlServerConfigration.getDataBase(), sqlServerConfigration.getPort().toString(), sqlServerConfigration.getUsername(), sqlServerConfigration.getPassword()); transMeta.addDatabase(dataMeta); - if (extractType.equalsIgnoreCase("all_scope")) { - if(datasetTable.getType().equalsIgnoreCase("sql")){ - selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); - }else { - String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); - QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createQuerySQL(tableName, datasetTableFields); - } - } + selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputStep = inputStep(transMeta, selectSQL); udjcStep = udjc(datasetTableFields, false); break; @@ -647,15 +631,8 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword()); } transMeta.addDatabase(dataMeta); - if (extractType.equalsIgnoreCase("all_scope")) { - if(datasetTable.getType().equalsIgnoreCase("sql")){ - selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); - }else { - String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); - QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createQuerySQL(tableName, datasetTableFields); - } - } + + selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputStep = inputStep(transMeta, selectSQL); udjcStep = udjc(datasetTableFields, false); break; @@ -702,6 +679,25 @@ public class ExtractDataService { FileUtils.writeStringToFile(file, transXml, "UTF-8"); } + private String getSelectSQL(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) { + if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("db")) { + String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields); + } + + if(extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")){ + selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); + } + if(!extractType.equalsIgnoreCase("all_scope")){ + QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); + selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); + } + return selectSQL; + } + private StepMeta inputStep(TransMeta transMeta, String selectSQL) { TableInputMeta tableInput = new TableInputMeta(); DatabaseMeta database = transMeta.findDatabase("db");