From 97d822796b5f056753fe93f07f9350201e05fbec Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 11:27:44 +0800 Subject: [PATCH 01/18] =?UTF-8?q?feat(=E8=A7=86=E5=9B=BE):spark=20?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=A8=A1=E5=BC=8F=E4=BF=AE=E6=94=B9=EF=BC=9B?= =?UTF-8?q?=E8=A7=86=E5=9B=BE=E5=88=9B=E5=BB=BA=E6=94=AF=E6=8C=81=E5=90=8D?= =?UTF-8?q?=E7=A7=B0=E7=BC=96=E8=BE=91=EF=BC=8C=E5=90=8C=E6=97=B6=E5=B0=86?= =?UTF-8?q?=E8=A7=86=E5=9B=BE=E5=90=8D=E7=A7=B0=E9=BB=98=E8=AE=A4=E4=BD=9C?= =?UTF-8?q?=E4=B8=BAchart=20title?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/dataease/service/spark/SparkCalc.java | 3 +- frontend/src/lang/zh.js | 4 ++- frontend/src/views/chart/group/Group.vue | 33 +++++++++++++++--- frontend/src/views/chart/view/ChartEdit.vue | 34 +++++++++---------- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java index 936ddb8d57..9c42468c9b 100644 --- a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java +++ b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java @@ -22,6 +22,7 @@ import org.apache.spark.sql.*; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; +import org.apache.spark.storage.StorageLevel; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; import scala.Tuple2; @@ -144,7 +145,7 @@ public class SparkCalc { }); StructType structType = DataTypes.createStructType(structFields); - Dataset dataFrame = sqlContext.createDataFrame(rdd, structType).persist(); + Dataset dataFrame = sqlContext.createDataFrame(rdd, structType).persist(StorageLevel.MEMORY_AND_DISK_SER()); CacheUtil.getInstance().addCacheData(hTable, dataFrame); dataFrame.count(); return dataFrame; diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index f71ea7c723..2d30594db3 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -697,7 +697,9 @@ export default { rose_type: '玫瑰图模式', radius_mode: '半径', area_mode: '面积', - rose_radius: '圆角' + rose_radius: '圆角', + view_name: '视图名称', + name_can_not_empty: '名称不能为空' }, dataset: { datalist: '数据集', diff --git a/frontend/src/views/chart/group/Group.vue b/frontend/src/views/chart/group/Group.vue index 1e40a3f378..e15470f6fd 100644 --- a/frontend/src/views/chart/group/Group.vue +++ b/frontend/src/views/chart/group/Group.vue @@ -199,10 +199,17 @@ width="70%" class="dialog-css" > + + + + + + +
- {{ $t('chart.cancel') }} - {{ $t('chart.confirm') }} + {{ $t('chart.cancel') }} + {{ $t('chart.confirm') }}
@@ -518,10 +525,24 @@ export default { this.selectTableFlag = true }, + closeCreateChart() { + this.selectTableFlag = false + this.table = {} + }, + createChart() { console.log(this.table) + if (!this.table.name) { + this.$message({ + message: this.$t('chart.name_can_not_empty'), + type: 'error', + showClose: true + }) + return + } const view = {} view.name = this.table.name + view.title = this.table.name view.sceneId = this.currGroup.id view.tableId = this.table.id view.type = 'bar' @@ -540,7 +561,7 @@ export default { }) view.customFilter = JSON.stringify([]) post('/chart/view/save', view).then(response => { - this.selectTableFlag = false + this.closeCreateChart() this.$store.dispatch('chart/setTableId', null) this.$store.dispatch('chart/setTableId', this.table.id) // this.$router.push('/chart/chart-edit') @@ -551,7 +572,7 @@ export default { }, getTable(table) { - this.table = table + this.table = JSON.parse(JSON.stringify(table)) }, refresh() { @@ -630,4 +651,8 @@ export default { .dialog-css >>> .el-dialog__body { padding: 10px 20px 20px; } + + .form-item>>>.el-form-item__label{ + font-size: 12px; + } diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 2dddd7cb81..bd8f4876ef 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -132,23 +132,23 @@ -
- {{ $t('chart.result_filter') }} -
- - - - - -
-
+ + + + + + + + + + + + + + + + + From 74ccd7d30775d53cb40d0f6407110554291874a8 Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 11:30:04 +0800 Subject: [PATCH 02/18] =?UTF-8?q?feat(=E8=A7=86=E5=9B=BE):=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=BF=87=E6=BB=A4=E5=99=A8=E6=9A=82=E6=97=B6=E9=9A=90?= =?UTF-8?q?=E8=97=8F=E4=B8=8D=E5=81=9A...?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/chart/view/ChartEdit.vue | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index bd8f4876ef..5c866a932a 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -132,23 +132,23 @@ - - - - - - - - - - - - - - - - - +
+ {{ $t('chart.result_filter') }} +
+ + + + + +
+
From 6607bb56c5d442d2f2befb0f2d3b0237fc29b52d Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 12:06:52 +0800 Subject: [PATCH 03/18] =?UTF-8?q?feat(test):=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/listener/AppStartReadHBaseListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java b/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java index 2fc2f731d7..d73e298f78 100644 --- a/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java +++ b/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java @@ -45,14 +45,14 @@ public class AppStartReadHBaseListener implements ApplicationListener datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample); for (DatasetTable table : datasetTables) { - commonThreadPool.addTask(() -> { +// commonThreadPool.addTask(() -> { try { List fields = dataSetTableFieldsService.getFieldsByTableId(table.getId()); sparkCalc.getHBaseDataAndCache(table.getId(), fields); } catch (Exception e) { e.printStackTrace(); } - }); +// }); } } } From e5226cfab23f09c3372a8990a9bbc292acff450d Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 14 Apr 2021 14:15:11 +0800 Subject: [PATCH 04/18] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../provider/DatasourceProvider.java | 5 +- .../datasource/provider/JdbcProvider.java | 104 ++++++++++++++---- .../datasource/service/DatasourceService.java | 30 ++++- 3 files changed, 116 insertions(+), 23 deletions(-) diff --git a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java index 71b513471f..05bb3c86e6 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java @@ -19,7 +19,7 @@ public abstract class DatasourceProvider { abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; - public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception{ + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { return new ArrayList<>(); }; @@ -27,7 +27,7 @@ public abstract class DatasourceProvider { getData(datasourceRequest); } - abstract public Long count(DatasourceRequest datasourceRequest)throws Exception; + abstract public Long count(DatasourceRequest datasourceRequest) throws Exception; abstract public List getPageData(DatasourceRequest datasourceRequest) throws Exception; @@ -35,4 +35,5 @@ public abstract class DatasourceProvider { abstract public List fetchResultField(ResultSet rs) throws Exception; + abstract public void initConnectionPool(DatasourceRequest datasourceRequest) throws Exception; } diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 6ef5ee0b1d..6517233c45 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -13,24 +13,28 @@ import org.springframework.stereotype.Service; import java.sql.*; import java.text.MessageFormat; import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; @Service("jdbc") public class JdbcProvider extends DatasourceProvider { - + private static Map> jdbcConnection = new HashMap<>(); + private static int poolSize = 20; @Override public List getData(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); - try ( - Connection connection = getConnection(datasourceRequest); - Statement stat = connection.createStatement(); - ResultSet rs = stat.executeQuery(datasourceRequest.getQuery()) - ) { + Connection connection = null; + try { + connection = getConnectionFromPool(datasourceRequest); + Statement stat = connection.createStatement(); + ResultSet rs = stat.executeQuery(datasourceRequest.getQuery()); list = fetchResult(rs); } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); + }finally { + returnSource(connection, datasourceRequest.getDatasource().getId()); } return list; } @@ -38,14 +42,18 @@ public class JdbcProvider extends DatasourceProvider { @Override public ResultSet getDataResultSet(DatasourceRequest datasourceRequest) throws Exception { ResultSet rs; + Connection connection = null; try { - Connection connection = getConnection(datasourceRequest); + connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); rs = stat.executeQuery(datasourceRequest.getQuery()); + returnSource(connection, datasourceRequest.getDatasource().getId()); } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); + }finally { + returnSource(connection, datasourceRequest.getDatasource().getId()); } return rs; } @@ -53,16 +61,19 @@ public class JdbcProvider extends DatasourceProvider { @Override public List getPageData(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); - try ( - Connection connection = getConnection(datasourceRequest); - Statement stat = connection.createStatement(); - ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() - 1) * datasourceRequest.getPageSize(), datasourceRequest.getPageSize())) - ) { + Connection connection = null; + try { + connection = getConnectionFromPool(datasourceRequest); + Statement stat = connection.createStatement(); + ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() - 1) * datasourceRequest.getPageSize(), datasourceRequest.getPageSize())); + returnSource(connection, datasourceRequest.getDatasource().getId()); list = fetchResult(rs); } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); + }finally { + returnSource(connection, datasourceRequest.getDatasource().getId()); } return list; } @@ -112,23 +123,28 @@ public class JdbcProvider extends DatasourceProvider { public List getTables(DatasourceRequest datasourceRequest) throws Exception { List tables = new ArrayList<>(); String queryStr = getTablesSql(datasourceRequest); - try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) { + Connection con = null; + try { + con = getConnectionFromPool(datasourceRequest); + Statement ps = con.createStatement(); ResultSet resultSet = ps.executeQuery(queryStr); while (resultSet.next()) { tables.add(resultSet.getString(1)); } + return tables; } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); + }finally { + returnSource(con, datasourceRequest.getDatasource().getId()); } - return tables; } @Override public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); - try ( - Connection connection = getConnection(datasourceRequest); - ) { + Connection connection = null; + try { + connection = getConnectionFromPool(datasourceRequest); DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet resultSet = databaseMetaData.getColumns(null, "%", datasourceRequest.getTable().toUpperCase(), "%"); while (resultSet.next()) { @@ -152,6 +168,8 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception("ERROR:" + e.getMessage(), e); } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); + }finally { + returnSource(connection, datasourceRequest.getDatasource().getId()); } return list; } @@ -161,27 +179,73 @@ public class JdbcProvider extends DatasourceProvider { @Override public void test(DatasourceRequest datasourceRequest) throws Exception { String queryStr = getTablesSql(datasourceRequest); - try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) { + Connection con = null; + try { + con = getConnection(datasourceRequest); + Statement ps = con.createStatement(); ResultSet resultSet = ps.executeQuery(queryStr); } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); + }finally { + con.close(); } } public Long count(DatasourceRequest datasourceRequest) throws Exception { - try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) { + Connection con = null; + try { + con = getConnectionFromPool(datasourceRequest); Statement ps = con.createStatement(); ResultSet resultSet = ps.executeQuery(datasourceRequest.getQuery()); while (resultSet.next()) { return resultSet.getLong(1); } } catch (Exception e) { throw new Exception("ERROR: " + e.getMessage(), e); + }finally { + returnSource(con, datasourceRequest.getDatasource().getId()); } return 0L; } - private Connection getConnection(DatasourceRequest datasourceRequest) throws Exception { + private void returnSource(Connection connection, String dataSourceId) throws Exception{ + if(connection != null && !connection.isClosed()){ + ArrayBlockingQueue connections = jdbcConnection.get(dataSourceId); + connections.put(connection); + } + } + + private Connection getConnectionFromPool(DatasourceRequest datasourceRequest)throws Exception { + ArrayBlockingQueue connections = jdbcConnection.get(datasourceRequest.getDatasource().getId()); + if (connections == null) { + initConnectionPool(datasourceRequest); + } + connections = jdbcConnection.get(datasourceRequest.getDatasource().getId()); + Connection co = connections.take(); + return co; + } + + @Override + public void initConnectionPool(DatasourceRequest datasourceRequest)throws Exception{ + ArrayBlockingQueue connections = jdbcConnection.get(datasourceRequest.getDatasource().getId()); + if (connections == null) { + connections = new ArrayBlockingQueue<>(poolSize); + for (int i = 0; i < poolSize ; i++) { + Connection connection = getConnection(datasourceRequest); + connections.add(connection); + } + jdbcConnection.put(datasourceRequest.getDatasource().getId(), connections); + }else { + for (int i = 0; i < poolSize ; i++) { + Connection connection = connections.take(); + connection.close(); + connection = getConnection(datasourceRequest); + connections.add(connection); + } + } + } + + private static Connection getConnection(DatasourceRequest datasourceRequest) throws Exception { String username = null; String password = null; String driver = null; diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index 24f6b2a4c0..2780916961 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -5,6 +5,7 @@ import io.dataease.base.mapper.*; import io.dataease.base.mapper.ext.ExtDataSourceMapper; import io.dataease.base.mapper.ext.query.GridExample; import io.dataease.commons.exception.DEException; +import io.dataease.commons.utils.CommonThreadPool; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.ProviderFactory; @@ -24,7 +25,8 @@ public class DatasourceService { @Resource private DatasourceMapper datasourceMapper; - + @Resource + private CommonThreadPool commonThreadPool; @Resource private ExtDataSourceMapper extDataSourceMapper; @@ -39,6 +41,7 @@ public class DatasourceService { datasource.setUpdateTime(currentTimeMillis); datasource.setCreateTime(currentTimeMillis); datasourceMapper.insertSelective(datasource); + initConnectionPool(datasource); return datasource; } @@ -68,6 +71,7 @@ public class DatasourceService { datasource.setCreateTime(null); datasource.setUpdateTime(System.currentTimeMillis()); datasourceMapper.updateByPrimaryKeySelective(datasource); + initConnectionPool(datasource); } public void validate(Datasource datasource) throws Exception { @@ -89,4 +93,28 @@ public class DatasourceService { return datasourceMapper.selectByPrimaryKey(id); } + private void initConnectionPool(Datasource datasource){ + commonThreadPool.addTask(() ->{ + try { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + datasourceProvider.initConnectionPool(datasourceRequest); + }catch (Exception e){} + }); + } + + public void initAllDataSourceConnectionPool(){ + List datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample()); + datasources.forEach(datasource -> { + try { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + datasourceProvider.initConnectionPool(datasourceRequest); + }catch (Exception e){ + e.printStackTrace(); + } + }); + } } From 1276109f47792c7b4bd497095e6c361105158f0c Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 15:18:53 +0800 Subject: [PATCH 05/18] =?UTF-8?q?feat(frontend):=E4=BB=AA=E8=A1=A8?= =?UTF-8?q?=E6=9D=BF=E6=A0=B7=E5=BC=8F=E5=BE=AE=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/canvas/components/Toolbar.vue | 4 ++-- frontend/src/views/dataset/add/AddExcel.vue | 6 +++++- frontend/src/views/panel/list/PanelViewShow.vue | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/canvas/components/Toolbar.vue b/frontend/src/components/canvas/components/Toolbar.vue index a55f26dc89..9342e441e7 100644 --- a/frontend/src/components/canvas/components/Toolbar.vue +++ b/frontend/src/components/canvas/components/Toolbar.vue @@ -228,8 +228,8 @@ export default { float: right; height: 35px; line-height: 35px; - background: #fff; - border-bottom: 1px solid #ddd; + /*background: #fff;*/ + /*border-bottom: 1px solid #ddd;*/ .canvas-config { display: inline-block; diff --git a/frontend/src/views/dataset/add/AddExcel.vue b/frontend/src/views/dataset/add/AddExcel.vue index 21a6e6e8a6..5f98af56c3 100644 --- a/frontend/src/views/dataset/add/AddExcel.vue +++ b/frontend/src/views/dataset/add/AddExcel.vue @@ -24,10 +24,11 @@ {{ $t('dataset.upload_file') }} @@ -117,6 +118,9 @@ export default { // this.options = response.data // }) // }, + uploadSuccess(response, file, fileList) { + console.log(response) + }, save() { // console.log(this.checkTableList); diff --git a/frontend/src/views/panel/list/PanelViewShow.vue b/frontend/src/views/panel/list/PanelViewShow.vue index 635c796b5d..98b41efe0f 100644 --- a/frontend/src/views/panel/list/PanelViewShow.vue +++ b/frontend/src/views/panel/list/PanelViewShow.vue @@ -118,10 +118,10 @@ export default { } .panel-design-head { - height: 40px; - background-color: white; + height: 35px; + /*background-color: white;*/ padding: 0 6px; - line-height: 40px; + line-height: 35px; } .panel-design-preview { From daf784f2dfecc891ca7e0acd8cac82a2037e30d9 Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 16:00:09 +0800 Subject: [PATCH 06/18] feat(backend):spark --- .../java/io/dataease/config/CommonConfig.java | 17 ++++++++++++++++- .../io/dataease/service/spark/SparkCalc.java | 5 ++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java index f22c749e62..0728b2a831 100644 --- a/backend/src/main/java/io/dataease/config/CommonConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -40,7 +40,22 @@ public class CommonConfig { SparkSession spark = SparkSession.builder() .appName(env.getProperty("spark.appName", "DataeaseJob")) .master(env.getProperty("spark.master", "local[*]")) - .config("spark.scheduler.mode", "FAIR") + .config("spark.scheduler.mode", env.getProperty("spark.scheduler.mode", "FAIR")) + .config("spark.serializer", env.getProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer")) + .config("spark.executor.cores", env.getProperty("spark.executor.cores", "8")) + .config("spark.executor.memory", env.getProperty("spark.executor.memory", "6442450944b")) + .config("spark.locality.wait", env.getProperty("spark.locality.wait", "600000")) + .config("spark.maxRemoteBlockSizeFetchToMem", env.getProperty("spark.maxRemoteBlockSizeFetchToMem", "2000m")) + .config("spark.shuffle.detectCorrupt", env.getProperty("spark.shuffle.detectCorrupt", "false")) + .config("spark.shuffle.service.enabled", env.getProperty("spark.shuffle.service.enabled", "true")) + .config("spark.sql.adaptive.enabled", env.getProperty("spark.sql.adaptive.enabled", "true")) + .config("spark.sql.adaptive.shuffle.targetPostShuffleInputSize", env.getProperty("spark.sql.adaptive.shuffle.targetPostShuffleInputSize", "200M")) + .config("spark.sql.broadcastTimeout", env.getProperty("spark.sql.broadcastTimeout", "12000")) + .config("spark.sql.retainGroupColumns", env.getProperty("spark.sql.retainGroupColumns", "false")) + .config("spark.sql.sortMergeJoinExec.buffer.in.memory.threshold", env.getProperty("spark.sql.sortMergeJoinExec.buffer.in.memory.threshold", "100000")) + .config("spark.sql.sortMergeJoinExec.buffer.spill.threshold", env.getProperty("spark.sql.sortMergeJoinExec.buffer.spill.threshold", "100000")) + .config("spark.sql.variable.substitute", env.getProperty("spark.sql.variable.substitute", "false")) + .config("spark.temp.expired.time", env.getProperty("spark.temp.expired.time", "3600")) .getOrCreate(); return spark; } diff --git a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java index 9c42468c9b..b3c8cf71a2 100644 --- a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java +++ b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java @@ -91,7 +91,10 @@ public class SparkCalc { public Dataset getHBaseDataAndCache(JavaSparkContext sparkContext, SQLContext sqlContext, String hTable, List fields) throws Exception { Scan scan = new Scan(); - scan.addFamily(column_family.getBytes()); + scan.addFamily(Bytes.toBytes(column_family)); + for (DatasetTableField field : fields) { + scan.addColumn(Bytes.toBytes(column_family), Bytes.toBytes(field.getOriginName())); + } ClientProtos.Scan proto = ProtobufUtil.toScan(scan); String scanToString = new String(Base64.getEncoder().encode(proto.toByteArray())); From eaf2b13882ce7cccfcd255cb03bfbb064eef717b Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 16:06:08 +0800 Subject: [PATCH 07/18] =?UTF-8?q?feat(backend):=E6=B3=A8=E9=87=8A=E3=80=81?= =?UTF-8?q?=E5=AF=BC=E5=8C=85=20=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/main/java/io/dataease/config/CommonConfig.java | 2 -- .../main/java/io/dataease/listener/AppStartListener.java | 1 - .../io/dataease/listener/AppStartReadHBaseListener.java | 6 ------ .../java/io/dataease/service/chart/ChartViewService.java | 1 - .../src/main/java/io/dataease/service/spark/SparkCalc.java | 5 ----- 5 files changed, 15 deletions(-) diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java index 0728b2a831..6e98754712 100644 --- a/backend/src/main/java/io/dataease/config/CommonConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -2,8 +2,6 @@ package io.dataease.config; import com.fit2cloud.autoconfigure.QuartzAutoConfiguration; import io.dataease.commons.utils.CommonThreadPool; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SparkSession; import org.pentaho.di.core.KettleEnvironment; import org.pentaho.di.repository.filerep.KettleFileRepository; diff --git a/backend/src/main/java/io/dataease/listener/AppStartListener.java b/backend/src/main/java/io/dataease/listener/AppStartListener.java index 6193506241..a2224eaaea 100644 --- a/backend/src/main/java/io/dataease/listener/AppStartListener.java +++ b/backend/src/main/java/io/dataease/listener/AppStartListener.java @@ -1,7 +1,6 @@ package io.dataease.listener; import io.dataease.base.domain.DatasetTableTask; -import io.dataease.job.sechedule.ScheduleManager; import io.dataease.service.ScheduleService; import io.dataease.service.dataset.DataSetTableTaskService; import org.springframework.boot.context.event.ApplicationReadyEvent; diff --git a/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java b/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java index d73e298f78..d8dcc2e3e6 100644 --- a/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java +++ b/backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java @@ -3,16 +3,10 @@ package io.dataease.listener; import io.dataease.base.domain.DatasetTable; import io.dataease.base.domain.DatasetTableExample; import io.dataease.base.domain.DatasetTableField; -import io.dataease.base.domain.DatasetTableFieldExample; -import io.dataease.base.mapper.DatasetTableFieldMapper; import io.dataease.base.mapper.DatasetTableMapper; -import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.CommonThreadPool; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.spark.SparkCalc; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SQLContext; -import org.apache.spark.sql.SparkSession; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.core.annotation.Order; diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 2100b7ada1..28f0e7b4d6 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -4,7 +4,6 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import io.dataease.base.domain.*; import io.dataease.base.mapper.ChartViewMapper; -import io.dataease.base.mapper.DatasetTableFieldMapper; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; import io.dataease.controller.request.chart.ChartViewRequest; diff --git a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java index b3c8cf71a2..b3e2010513 100644 --- a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java +++ b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java @@ -6,7 +6,6 @@ import io.dataease.dto.chart.ChartViewFieldDTO; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -28,7 +27,6 @@ import org.springframework.stereotype.Service; import scala.Tuple2; import javax.annotation.Resource; -import java.math.BigDecimal; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Base64; @@ -51,7 +49,6 @@ public class SparkCalc { JavaSparkContext sparkContext = new JavaSparkContext(spark.sparkContext()); // Spark SQL Context -// SQLContext sqlContext = CommonBeanFactory.getBean(SQLContext.class); SQLContext sqlContext = new SQLContext(sparkContext); sqlContext.setConf("spark.sql.shuffle.partitions", env.getProperty("spark.sql.shuffle.partitions", "1")); sqlContext.setConf("spark.default.parallelism", env.getProperty("spark.default.parallelism", "1")); @@ -82,7 +79,6 @@ public class SparkCalc { JavaSparkContext sparkContext = new JavaSparkContext(spark.sparkContext()); // Spark SQL Context -// SQLContext sqlContext = CommonBeanFactory.getBean(SQLContext.class); SQLContext sqlContext = new SQLContext(sparkContext); sqlContext.setConf("spark.sql.shuffle.partitions", env.getProperty("spark.sql.shuffle.partitions", "1")); sqlContext.setConf("spark.default.parallelism", env.getProperty("spark.default.parallelism", "1")); @@ -99,7 +95,6 @@ public class SparkCalc { String scanToString = new String(Base64.getEncoder().encode(proto.toByteArray())); // HBase config -// Configuration conf = CommonBeanFactory.getBean(Configuration.class); org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); conf.set("hbase.zookeeper.quorum", env.getProperty("hbase.zookeeper.quorum")); conf.set("hbase.zookeeper.property.clientPort", env.getProperty("hbase.zookeeper.property.clientPort")); From 924f639661819914c801335011843332fc4d9f79 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 14 Apr 2021 18:31:13 +0800 Subject: [PATCH 08/18] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasource/service/DatasourceService.java | 18 ++++++----- .../AppStartInitDataSourceListener.java | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 backend/src/main/java/io/dataease/listener/AppStartInitDataSourceListener.java diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index 2780916961..7fb24d6fae 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -107,14 +107,16 @@ public class DatasourceService { public void initAllDataSourceConnectionPool(){ List datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample()); datasources.forEach(datasource -> { - try { - DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(datasource); - datasourceProvider.initConnectionPool(datasourceRequest); - }catch (Exception e){ - e.printStackTrace(); - } + commonThreadPool.addTask(() ->{ + try { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + datasourceProvider.initConnectionPool(datasourceRequest); + }catch (Exception e){ + e.printStackTrace(); + } + }); }); } } diff --git a/backend/src/main/java/io/dataease/listener/AppStartInitDataSourceListener.java b/backend/src/main/java/io/dataease/listener/AppStartInitDataSourceListener.java new file mode 100644 index 0000000000..632cb2e767 --- /dev/null +++ b/backend/src/main/java/io/dataease/listener/AppStartInitDataSourceListener.java @@ -0,0 +1,31 @@ +package io.dataease.listener; + +import io.dataease.base.domain.DatasetTable; +import io.dataease.base.domain.DatasetTableExample; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.mapper.DatasetTableMapper; +import io.dataease.commons.utils.CommonThreadPool; +import io.dataease.datasource.service.DatasourceService; +import io.dataease.service.dataset.DataSetTableFieldsService; +import io.dataease.service.spark.SparkCalc; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.core.annotation.Order; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.List; + +@Component +@Order(value = 2) +public class AppStartInitDataSourceListener implements ApplicationListener { + @Resource + private DatasourceService datasourceService; + @Override + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + System.out.println("================= Init datasource connection pool ================="); + // 项目启动，从数据集中找到定时抽取的表，从HBase中读取放入缓存 + datasourceService.initAllDataSourceConnectionPool(); + } +} From 4d3e20126a3f2c15e0b85f71300a7f4c83fe159f Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 14 Apr 2021 18:46:13 +0800 Subject: [PATCH 09/18] =?UTF-8?q?feat(backend):=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E3=80=81=E8=A7=86=E5=9B=BE=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/chart/ChartViewController.java | 5 +++++ .../controller/dataset/DataSetTableController.java | 4 ++++ .../io/dataease/service/chart/ChartViewService.java | 12 ++++++++++++ .../service/dataset/DataSetTableService.java | 12 ++++++++++++ 4 files changed, 33 insertions(+) diff --git a/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java b/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java index 9c2cfcd498..e584a53c2c 100644 --- a/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java +++ b/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java @@ -44,4 +44,9 @@ public class ChartViewController { public ChartViewDTO getData(@PathVariable String id) throws Exception { return chartViewService.getData(id); } + + @PostMapping("chartDetail/{id}") + public Map chartDetail(@PathVariable String id) { + return chartViewService.getChartDetail(id); + } } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 7a902aa505..91e8d5a3dc 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -82,4 +82,8 @@ public class DataSetTableController { dataSetTableService.saveIncrementalConfig(datasetTableIncrementalConfig); } + @PostMapping("datasetDetail/{id}") + public Map datasetDetail(@PathVariable String id) { + return dataSetTableService.getDatasetDetail(id); + } } diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 28f0e7b4d6..5612977965 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -20,6 +20,7 @@ import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.spark.SparkCalc; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -249,4 +250,15 @@ public class ChartViewService { throw new RuntimeException("Name can't repeat in same group."); } } + + public Map getChartDetail(String id) { + Map map = new HashMap<>(); + ChartViewWithBLOBs chartViewWithBLOBs = chartViewMapper.selectByPrimaryKey(id); + map.put("chart", chartViewWithBLOBs); + if (ObjectUtils.isNotEmpty(chartViewWithBLOBs)) { + Map datasetDetail = dataSetTableService.getDatasetDetail(chartViewWithBLOBs.getTableId()); + map.putAll(datasetDetail); + } + return map; + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index f40e017de8..0ca76754fe 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -16,6 +16,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.dataset.DataTableInfoDTO; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -422,4 +423,15 @@ public class DataSetTableService { throw new RuntimeException("Name can't repeat in same group."); } } + + public Map getDatasetDetail(String id) { + Map map = new HashMap<>(); + DatasetTable table = datasetTableMapper.selectByPrimaryKey(id); + map.put("table", table); + if (ObjectUtils.isNotEmpty(table)) { + Datasource datasource = datasourceMapper.selectByPrimaryKey(table.getDataSourceId()); + map.put("datasource", datasource); + } + return map; + } } From fea7ceec9b1c127ab95ee721eac0d2d722fbd2f9 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 15 Apr 2021 10:33:22 +0800 Subject: [PATCH 10/18] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canvas/components/Editor/MarkLine.vue | 422 +++++++++--------- .../canvas/components/Editor/Shape.vue | 18 +- .../canvas/components/Editor/index.vue | 22 +- .../widget/DeWidget/DeDrawingWidget.vue | 21 +- .../widget/drawServiceImpl/MySelectImpl.js | 87 ++++ frontend/src/components/widget/index.js | 3 +- .../widget/service/DrawWidgetService.js | 83 ++++ frontend/src/store/getters.js | 5 +- frontend/src/store/modules/application.js | 30 +- frontend/src/views/panel/edit/index.vue | 37 +- .../src/views/panel/filter/filterDialog.vue | 23 +- frontend/src/views/panel/filter/index.vue | 42 +- 12 files changed, 512 insertions(+), 281 deletions(-) create mode 100644 frontend/src/components/widget/drawServiceImpl/MySelectImpl.js diff --git a/frontend/src/components/canvas/components/Editor/MarkLine.vue b/frontend/src/components/canvas/components/Editor/MarkLine.vue index 2ba5233d75..a07ccedc80 100644 --- a/frontend/src/components/canvas/components/Editor/MarkLine.vue +++ b/frontend/src/components/canvas/components/Editor/MarkLine.vue @@ -1,14 +1,14 @@ diff --git a/frontend/src/components/canvas/components/Editor/Shape.vue b/frontend/src/components/canvas/components/Editor/Shape.vue index c2ebc6bc61..9db1a349d0 100644 --- a/frontend/src/components/canvas/components/Editor/Shape.vue +++ b/frontend/src/components/canvas/components/Editor/Shape.vue @@ -3,13 +3,13 @@ - - - - - - - + + + + + + +
{ diff --git a/frontend/src/components/canvas/components/Editor/index.vue b/frontend/src/components/canvas/components/Editor/index.vue index 06efece521..277591c938 100644 --- a/frontend/src/components/canvas/components/Editor/index.vue +++ b/frontend/src/components/canvas/components/Editor/index.vue @@ -30,8 +30,8 @@ :id="'component' + item.id" class="component" :style="getComponentStyle(item.style)" - :element="item" - :item="item" + :service-name="item.widgetService.name" + :panel-id="panelInfo.id" @filter-value-change="filterValueChange" /> @@ -97,12 +97,18 @@ export default { isShowArea: false } }, - computed: mapState([ - 'componentData', - 'curComponent', - 'canvasStyleData', - 'editor' - ]), + computed: { + panelInfo() { + return this.$store.state.panel.panelInfo + }, + ...mapState([ + 'componentData', + 'curComponent', + 'canvasStyleData', + 'editor' + ]) + }, + mounted() { // 获取编辑器元素 this.$store.commit('getEditor') diff --git a/frontend/src/components/widget/DeWidget/DeDrawingWidget.vue b/frontend/src/components/widget/DeWidget/DeDrawingWidget.vue index ab37b9d57d..03bf5480d8 100644 --- a/frontend/src/components/widget/DeWidget/DeDrawingWidget.vue +++ b/frontend/src/components/widget/DeWidget/DeDrawingWidget.vue @@ -1,19 +1,30 @@