From 64adee61f5b9b09d872c53995fb8d7c6240bd047 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Sat, 25 Jun 2022 17:24:23 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=B0=9D=E8=AF=95=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/dataease/provider/ProviderFactory.java | 1 + .../provider/datasource/JdbcProvider.java | 24 +++++++++++------ .../service/dataset/ExtractDataService.java | 26 ++++++++++++------- frontend/src/lang/en.js | 4 ++- frontend/src/lang/tw.js | 2 ++ frontend/src/lang/zh.js | 2 ++ .../system/datasource/DsConfiguration.vue | 8 ++++++ 7 files changed, 49 insertions(+), 18 deletions(-) diff --git a/backend/src/main/java/io/dataease/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/provider/ProviderFactory.java index e5dcaf4ff9..8bc2cbd64f 100644 --- a/backend/src/main/java/io/dataease/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/provider/ProviderFactory.java @@ -27,6 +27,7 @@ public class ProviderFactory implements ApplicationContextAware { DataSourceType dataSourceType = new DataSourceType(d.getType(), d.getName(), false, d.getExtraParams(), d.getCalculationMode(), d.isJdbc()); if(dataSourceType.getType().equalsIgnoreCase("oracle")){ dataSourceType.setCharset(d.getCharset()); + dataSourceType.setTargetCharset(d.getTargetCharset()); } beanFactory.registerSingleton(d.getType(), dataSourceType); } diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index f460504f02..7417f10708 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -216,10 +216,12 @@ public class JdbcProvider extends DefaultJdbcProvider { private List getDataResult(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { String charset = null; + String targetCharset = "UTF-8"; if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) { - JdbcConfiguration JdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); - if (StringUtils.isNotEmpty(JdbcConfiguration.getCharset()) && !JdbcConfiguration.getCharset().equalsIgnoreCase("Default")) { - charset = JdbcConfiguration.getCharset(); + JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), JdbcConfiguration.class); + if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) { + charset = jdbcConfiguration.getCharset(); + targetCharset = jdbcConfiguration.getTargetCharset(); } } List list = new LinkedList<>(); @@ -239,11 +241,17 @@ public class JdbcProvider extends DefaultJdbcProvider { row[j] = rs.getBoolean(j + 1) ? "1" : "0"; break; default: - if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) { - row[j] = new String(rs.getString(j + 1).getBytes(charset), "UTF-8"); + if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) { + row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString(); } else { - row[j] = rs.getString(j + 1); + if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) { + String orginStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset); + row[j] = new String(orginStr.getBytes("UTF-8"), "UTF-8"); + } else { + row[j] = rs.getString(j + 1); + } } + break; } } @@ -308,7 +316,7 @@ public class JdbcProvider extends DefaultJdbcProvider { String queryStr = getTablesSql(datasourceRequest); try (Connection con = getConnection(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) { } catch (Exception e) { - LogUtil.error("Datasource is invalid: " + datasourceRequest.getDatasource().getName() , e); + LogUtil.error("Datasource is invalid: " + datasourceRequest.getDatasource().getName(), e); io.dataease.plugins.common.exception.DataEaseException.throwException(e.getMessage()); } return "Success"; @@ -452,7 +460,7 @@ public class JdbcProvider extends DefaultJdbcProvider { driverClassName = defaultDriver; jdbcClassLoader = extendedJdbcClassLoader; } else { - if(deDriver == null){ + if (deDriver == null) { deDriver = deDriverMapper.selectByPrimaryKey(customDriver); } driverClassName = deDriver.getDriverClass(); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 79bc9c6476..b63f71824f 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -12,6 +12,7 @@ import io.dataease.plugins.common.constants.DatasourceTypes; import io.dataease.plugins.common.constants.DeTypeConstants; import io.dataease.plugins.common.dto.datasource.TableField; import io.dataease.plugins.common.request.datasource.DatasourceRequest; +import io.dataease.plugins.datasource.entity.JdbcConfiguration; import io.dataease.plugins.datasource.provider.Provider; import io.dataease.plugins.datasource.query.QueryProvider; import io.dataease.provider.DDLProvider; @@ -918,7 +919,7 @@ public class ExtractDataService { } transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, mysqlConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.mysql); break; case sqlServer: @@ -926,7 +927,7 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "MSSQLNATIVE", "Native", sqlServerConfiguration.getHost().trim(), sqlServerConfiguration.getDataBase(), sqlServerConfiguration.getPort().toString(), sqlServerConfiguration.getUsername(), sqlServerConfiguration.getPassword()); transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, sqlServerConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.sqlServer); break; case pg: @@ -934,7 +935,7 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "POSTGRESQL", "Native", pgConfiguration.getHost().trim(), pgConfiguration.getDataBase(), pgConfiguration.getPort().toString(), pgConfiguration.getUsername(), pgConfiguration.getPassword()); transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, pgConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.pg); break; case oracle: @@ -946,9 +947,8 @@ public class ExtractDataService { dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfiguration.getHost().trim(), oracleConfiguration.getDataBase(), oracleConfiguration.getPort().toString(), oracleConfiguration.getUsername(), oracleConfiguration.getPassword()); } transMeta.addDatabase(dataMeta); - selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, oracleConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle); break; case ck: @@ -957,7 +957,7 @@ public class ExtractDataService { dataMeta.setDatabaseType("Clickhouse"); transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, chConfiguration); udjcStep = udjc(datasetTableFields, DatasourceTypes.ck); break; case db2: @@ -966,7 +966,7 @@ public class ExtractDataService { dataMeta.setDatabaseType("DB2"); transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); - inputStep = inputStep(transMeta, selectSQL); + inputStep = inputStep(transMeta, selectSQL, db2Configuration); udjcStep = udjc(datasetTableFields, DatasourceTypes.db2); break; case excel: @@ -1030,11 +1030,14 @@ public class ExtractDataService { return selectSQL; } - private StepMeta inputStep(TransMeta transMeta, String selectSQL) { + private StepMeta inputStep(TransMeta transMeta, String selectSQL, JdbcConfiguration jdbcConfiguration) { TableInputMeta tableInput = new TableInputMeta(); DatabaseMeta database = transMeta.findDatabase("db"); tableInput.setDatabaseMeta(database); tableInput.setSQL(selectSQL); + if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) { + tableInput.setLazyConversionActive(true); + } StepMeta fromStep = new StepMeta("TableInput", "Data Input", tableInput); fromStep.setDraw(true); fromStep.setLocation(100, 100); @@ -1094,7 +1097,12 @@ public class ExtractDataService { private StepMeta outputStep(String dorisOutputTable, List datasetTableFields, Datasource datasource) { TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta(); - textFileOutputMeta.setEncoding("UTF-8"); + JdbcConfiguration jdbcConfiguration = new Gson().fromJson(datasource.getConfiguration(), JdbcConfiguration.class); + if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) { + textFileOutputMeta.setEncoding(jdbcConfiguration.getTargetCharset()); + }else { + textFileOutputMeta.setEncoding("UTF-8"); + } textFileOutputMeta.setHeaderEnabled(false); textFileOutputMeta.setFilename(root_path + dorisOutputTable); textFileOutputMeta.setSeparator(separator); diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 9398ba7623..e4df43259d 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1383,7 +1383,9 @@ export default { schema: 'Database Schema', please_choose_schema: 'Please select Schema', charset: 'Charset', - please_choose_charset: 'Please select Charset', + targetCharset: 'Target Charset', + please_choose_targetCharset: 'Please select target charset', + please_choose_charset: 'Please select charset', edit_datasource_msg: 'Modifying the data source information may make the data set under the modified data source unavailable. Confirm the modification？', repeat_datasource_msg: 'Data source information with the same configuration already exists, ', confirm_save: 'Confirm save?', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 98289a8e2e..f9a74eeeb0 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1384,6 +1384,8 @@ export default { schema: '數據庫 Schema', please_choose_schema: '請選擇數據庫 Schema', charset: '字符集', + targetCharset: '目標字符集', + please_choose_targetCharset: '請選擇目標字符集', please_choose_charset: '請選擇數據庫字符集', edit_datasource_msg: '修改數據源信息，可能會導致改數據源下的數據集不可用，確認修改？', repeat_datasource_msg: '已經存在相同配置的數據源信息，', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 2d9678c76b..4c8e99cc8b 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1389,8 +1389,10 @@ export default { get_schema: '获取 Schema', schema: '数据库 Schema', charset: '字符集', + targetCharset: '目标字符集', please_choose_schema: '请选择数据库 Schema', please_choose_charset: '请选择数据库字符集', + please_choose_targetCharset: '请选择目标字符集', edit_datasource_msg: '修改数据源信息，可能会导致该数据源下的数据集不可用，确认修改？', repeat_datasource_msg: '已经存在相同配置的数据源信息, ', confirm_save: '确认保存?', diff --git a/frontend/src/views/system/datasource/DsConfiguration.vue b/frontend/src/views/system/datasource/DsConfiguration.vue index f2b7a8478d..c7e3f1e824 100644 --- a/frontend/src/views/system/datasource/DsConfiguration.vue +++ b/frontend/src/views/system/datasource/DsConfiguration.vue @@ -228,6 +228,14 @@ + + + + + +