From b6798c6fee4a47329d24fcc558ee08dbd7e8f40a Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 21 Dec 2022 20:01:15 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E9=87=87=E7=94=A8=E5=88=86=E9=A1=B5=E7=9A=84=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=E6=8B=89=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../provider/query/ck/CKQueryProvider.java | 12 ++- .../provider/query/db2/Db2QueryProvider.java | 17 +++- .../query/hive/HiveQueryProvider.java | 12 ++- .../query/impala/ImpalaQueryProvider.java | 12 ++- .../query/mysql/MysqlQueryProvider.java | 5 +- .../query/oracle/OracleQueryProvider.java | 95 ++++++++++++++++--- .../provider/query/pg/PgQueryProvider.java | 16 +++- .../query/redshift/RedshiftQueryProvider.java | 16 +++- .../service/dataset/ExtractDataService.java | 44 ++++++--- 9 files changed, 187 insertions(+), 42 deletions(-) diff --git a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java index 2b8e0b1008..29681e692f 100644 --- a/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/ck/CKQueryProvider.java @@ -820,6 +820,14 @@ public class CKQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + return "SELECT COUNT(*) from " + String.format(CKConstants.KEYWORD_TABLE, sql); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -831,12 +839,12 @@ public class CKQueryProvider extends QueryProvider { } return stringBuilder.toString(); }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), table); } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_OFFSET, DE_PAGE_SIZE ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java index e1cdf780f5..64d7ca3f5f 100644 --- a/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/db2/Db2QueryProvider.java @@ -808,6 +808,17 @@ public class Db2QueryProvider extends QueryProvider { return tmpSql; } + + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); + schema = String.format(Db2Constants.KEYWORD_TABLE, schema); + return "SELECT COUNT(*) from " + schema + "." + String.format(Db2Constants.KEYWORD_TABLE, sql); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -817,15 +828,15 @@ public class Db2QueryProvider extends QueryProvider { }).toArray(String[]::new); if (ds != null) { Db2Configuration db2Configuration = new Gson().fromJson(ds.getConfiguration(), Db2Configuration.class); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), db2Configuration.getSchema() + ".\"" + table + "\""); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), db2Configuration.getSchema() + String.format(Db2Constants.KEYWORD_TABLE, table)); } else { - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), table); } } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS de_tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS de_tmp LIMIT DE_OFFSET, DE_PAGE_SIZE ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java index 2329aaf5c1..023e9e32b0 100644 --- a/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/hive/HiveQueryProvider.java @@ -770,6 +770,14 @@ public class HiveQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + return "SELECT COUNT(*) from " + String.format(HiveConstants.KEYWORD_TABLE, sql); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -781,12 +789,12 @@ public class HiveQueryProvider extends QueryProvider { } return stringBuilder.toString(); }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), table); } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_OFFSET, DE_PAGE_SIZE ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java index 2e33fc8d93..a5e9c49d74 100644 --- a/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/impala/ImpalaQueryProvider.java @@ -768,6 +768,14 @@ public class ImpalaQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + return "SELECT COUNT(*) from " + String.format(ImpalaConstants.KEYWORD_TABLE, sql); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -779,12 +787,12 @@ public class ImpalaQueryProvider extends QueryProvider { } return stringBuilder.toString(); }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), table); } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java index be8b8414c5..3a4bd7d406 100644 --- a/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/mysql/MysqlQueryProvider.java @@ -808,12 +808,12 @@ public class MysqlQueryProvider extends QueryProvider { } return stringBuilder.toString(); }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_OFFSET, DE_PAGE_SIZE ", StringUtils.join(array, ","), table); } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_OFFSET, DE_PAGE_SIZE ", fields, null); } public String transTreeItem(SQLObj tableObj, DatasetRowPermissionsTreeItem item) { @@ -1346,4 +1346,5 @@ public class MysqlQueryProvider extends QueryProvider { "{\"dateformat\": \"%Y%m%d %H:%i:%S\"}\n" + "]", Dateformat.class); } + } diff --git a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java index 6cdf6d874c..11c5fd238b 100644 --- a/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/oracle/OracleQueryProvider.java @@ -860,25 +860,94 @@ public class OracleQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if (isTable) { + String schema = new Gson().fromJson(ds.getConfiguration(), OracleConfiguration.class).getSchema(); + schema = String.format(OracleConstants.KEYWORD_TABLE, schema); + return "SELECT COUNT(*) from " + schema + "." + String.format(OracleConstants.KEYWORD_TABLE, sql); + } else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { - String[] array = fields.stream().map(f -> { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(" \"").append(f.getOriginName()).append("\""); - return stringBuilder.toString(); - }).toArray(String[]::new); - OracleConfiguration oracleConfiguration = new Gson().fromJson(ds.getConfiguration(), OracleConfiguration.class); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), oracleConfiguration.getSchema() + ".\"" + table + "\""); + List xAxis = new ArrayList<>(); + ChartViewFieldDTO chartViewFieldDTO = new ChartViewFieldDTO(); + chartViewFieldDTO.setOriginName("ROWNUM"); + xAxis.add(chartViewFieldDTO); + fields.forEach(datasetTableField -> { + ChartViewFieldDTO f = new ChartViewFieldDTO(); + f.setOriginName(datasetTableField.getOriginName()); + f.setDeType(0); + xAxis.add(f); + }); + + List fieldCustomFilter = new ArrayList<>(); + ChartFieldCustomFilterDTO chartFieldCustomFilterDTO = new ChartFieldCustomFilterDTO(); + DatasetTableField datasetTableField = new DatasetTableField(); + datasetTableField.setOriginName("ROWNUM"); + datasetTableField.setDeType(0); + chartFieldCustomFilterDTO.setField(datasetTableField); + + List filterItemDTOS = new ArrayList<>(); + ChartCustomFilterItemDTO itemDTO = new ChartCustomFilterItemDTO(); + itemDTO.setTerm("le"); + itemDTO.setValue("DE_ALL"); + filterItemDTOS.add(itemDTO); + chartFieldCustomFilterDTO.setFilter(filterItemDTOS); + fieldCustomFilter.add(chartFieldCustomFilterDTO); + + + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(OracleConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(OracleConstants.ALIAS_FIX, String.format(TABLE_ALIAS_PREFIX, 0))) + .build(); + setSchema(tableObj, ds); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + if (x.getOriginName().equalsIgnoreCase("ROWNUM")) { + xFields.add(SQLObj.builder() + .fieldName(x.getOriginName()) + .fieldAlias("DE_ROWNUM") + .build()); + continue; + } + String originField = String.format(OracleConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + String fieldAlias = String.format(OracleConstants.KEYWORD_TABLE, x.getOriginName()); + xFields.add(getXFields(x, originField, fieldAlias)); + } + } + + String customWheres = transCustomFilterList(tableObj, fieldCustomFilter); + + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.add(customWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", false); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("previewSql"); + st.add("isGroup", false); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(OracleConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + + return "SELECT * FROM (" + sqlFix(st.render()) + ") DE_RESULT_TMP " + " WHERE DE_ROWNUM >= DE_OFFSET"; } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - String[] array = fields.stream().map(f -> { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(" \"").append(f.getOriginName()).append("\""); - return stringBuilder.toString(); - }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), " (" + sqlFix(sql) + ") DE_TMP "); + return createRawQuerySQL("(" + sqlFix(sql) + ")", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java index 469f9a1dc4..4c7320ca93 100644 --- a/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/pg/PgQueryProvider.java @@ -800,6 +800,16 @@ public class PgQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); + String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, sql); + return "SELECT COUNT(*) from " + String.format(PgConstants.KEYWORD_TABLE, tableWithSchema); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -810,15 +820,15 @@ public class PgQueryProvider extends QueryProvider { if (ds != null) { String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, table); - return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), tableWithSchema); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), tableWithSchema); } else { - return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), table); } } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java index 3b447866d4..a0a5e7cd21 100644 --- a/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/query/redshift/RedshiftQueryProvider.java @@ -807,6 +807,16 @@ public class RedshiftQueryProvider extends QueryProvider { return tmpSql; } + public String getTotalCount(boolean isTable, String sql, Datasource ds) { + if(isTable){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); + String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, sql); + return "SELECT COUNT(*) from " + String.format(ImpalaConstants.KEYWORD_TABLE, tableWithSchema); + }else { + return "SELECT COUNT(*) from ( " + sql + " ) DE_COUNT_TEMP"; + } + } + @Override public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { @@ -817,15 +827,15 @@ public class RedshiftQueryProvider extends QueryProvider { if (ds != null) { String schema = new Gson().fromJson(ds.getConfiguration(), JdbcConfiguration.class).getSchema(); String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, table); - return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), tableWithSchema); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), tableWithSchema); } else { - return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table); + return MessageFormat.format("SELECT {0} FROM {1} LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", StringUtils.join(array, ","), table); } } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS DE_TEMP LIMIT DE_PAGE_SIZE OFFSET DE_OFFSET ", fields, null); } @Override diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index adff409cb7..73f1c37050 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -96,6 +96,7 @@ public class ExtractDataService { @Resource private KettleService kettleService; + private static final String lastUpdateTime = "${__last_update_time__}"; private static final String currentUpdateTime = "${__current_update_time__}"; private static final String separator = "|DE|"; @@ -104,6 +105,9 @@ public class ExtractDataService { @Value("${kettle.files.keep:false}") private boolean kettleFilesKeep; + @Value("${extract.page.size:50000}") + private Long extractPageSize; + private static final String shellScript = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" + "if [ $? -eq 0 ] ; then\n" + @@ -433,7 +437,23 @@ public class ExtractDataService { extractApiData(datasetTable, datasource, datasetTableFields, extractType); return; } - extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL); + Map sql = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); + if (StringUtils.isNotEmpty(sql.get("totalSql"))) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + datasourceRequest.setQuery(sql.get("totalSql")); + List tmpData = datasourceProvider.getData(datasourceRequest); + Long totalItems = CollectionUtils.isEmpty(tmpData) ? 0 : Long.valueOf(tmpData.get(0)[0]); + Long totalPage = (totalItems / extractPageSize) + (totalItems % extractPageSize > 0 ? 1 : 0); + for (Long i = 0L; i < totalPage; i++) { + Long offset = i * extractPageSize; + Long all = offset + extractPageSize; + extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, sql.get("selectSQL").replace("DE_OFFSET", offset.toString()).replace("DE_PAGE_SIZE", extractPageSize.toString()).replace("DE_ALL", all.toString())); + } + } else { + extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL); + } } private void extractApiData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType) throws Exception { @@ -923,7 +943,6 @@ public class ExtractDataService { } } transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, mysqlConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.mysql, mysqlConfiguration); break; @@ -931,7 +950,6 @@ public class ExtractDataService { SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfiguration.class); dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfiguration.getHost().trim(), sqlServerConfiguration.getDataBase(), sqlServerConfiguration.getPort().toString(), sqlServerConfiguration.getUsername(), sqlServerConfiguration.getPassword()); transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, sqlServerConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.sqlServer, sqlServerConfiguration); break; @@ -939,7 +957,6 @@ public class ExtractDataService { PgConfiguration pgConfiguration = new Gson().fromJson(datasource.getConfiguration(), PgConfiguration.class); dataMeta = new DatabaseMeta("db", "POSTGRESQL", "Native", pgConfiguration.getHost().trim(), pgConfiguration.getDataBase(), pgConfiguration.getPort().toString(), pgConfiguration.getUsername(), pgConfiguration.getPassword()); transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, pgConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.pg, pgConfiguration); break; @@ -952,7 +969,6 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfiguration.getHost().trim(), oracleConfiguration.getDataBase(), oracleConfiguration.getPort().toString(), oracleConfiguration.getUsername(), oracleConfiguration.getPassword()); } transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, oracleConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle, oracleConfiguration); break; @@ -961,7 +977,6 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "ORACLE", "Native", chConfiguration.getHost().trim(), chConfiguration.getDataBase().trim(), chConfiguration.getPort().toString(), chConfiguration.getUsername(), chConfiguration.getPassword()); dataMeta.setDatabaseType("Clickhouse"); transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, chConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.ck, chConfiguration); break; @@ -970,7 +985,6 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "DB2", "Native", db2Configuration.getHost().trim(), db2Configuration.getDataBase().trim(), db2Configuration.getPort().toString(), db2Configuration.getUsername(), db2Configuration.getPassword()); dataMeta.setDatabaseType("DB2"); transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputSteps = inputStep(transMeta, selectSQL, db2Configuration); udjcStep = udjc(datasetTableFields, DatasourceTypes.db2, db2Configuration); break; @@ -1019,11 +1033,13 @@ public class ExtractDataService { FileUtils.writeStringToFile(file, transXml, "UTF-8"); } - private String getSelectSQL(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) { + private Map getSelectSQL(String extractType, DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String selectSQL) { + Map sql = new HashMap<>(); if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase(DatasetType.DB.name())) { String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields, datasource); + sql.put("selectSQL", qp.createRawQuerySQL(tableName, datasetTableFields, datasource)); + sql.put("totalSql", qp.getTotalCount(true, tableName, datasource)); } if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase(DatasetType.SQL.name())) { @@ -1033,13 +1049,17 @@ public class ExtractDataService { selectSQL = new String(java.util.Base64.getDecoder().decode(selectSQL)); } QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); + sql.put("totalSql", qp.getTotalCount(false, selectSQL, datasource)); + sql.put("selectSQL", qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields)); } + if (!extractType.equalsIgnoreCase("all_scope")) { QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); + sql.put("totalSql", qp.getTotalCount(false, selectSQL, datasource)); + sql.put("selectSQL", qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields)); } - return selectSQL; + + return sql; } private List inputStep(TransMeta transMeta, String selectSQL, JdbcConfiguration jdbcConfiguration) {