diff --git a/backend/src/main/java/io/dataease/config/CommonConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java index 6cc3ed2bf5..e3a73048ed 100644 --- a/backend/src/main/java/io/dataease/config/CommonConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -37,6 +37,7 @@ public class CommonConfig { SparkSession spark = SparkSession.builder() .appName(env.getProperty("spark.appName", "DataeaseJob")) .master(env.getProperty("spark.master", "local[*]")) + .config("spark.scheduler.mode", "FAIR") .getOrCreate(); JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); return sc; diff --git a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java index 96e76f5497..ef905453ab 100644 --- a/backend/src/main/java/io/dataease/service/spark/SparkCalc.java +++ b/backend/src/main/java/io/dataease/service/spark/SparkCalc.java @@ -15,6 +15,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -29,6 +30,7 @@ import scala.Tuple2; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Base64; +import java.util.Iterator; import java.util.List; /** @@ -51,33 +53,37 @@ public class SparkCalc { conf.set(TableInputFormat.SCAN, scanToString); JavaPairRDD pairRDD = sparkContext.newAPIHadoopRDD(conf, TableInputFormat.class, ImmutableBytesWritable.class, Result.class); - JavaRDD rdd = pairRDD.map((Function, Row>) immutableBytesWritableResultTuple2 -> - { - Result result = immutableBytesWritableResultTuple2._2; - List list = new ArrayList<>(); - xAxis.forEach(x -> { - if (x.getDeType() == 0 || x.getDeType() == 1) { - list.add(Bytes.toString(result.getValue(column_family.getBytes(), x.getOriginName().getBytes()))); - } else if (x.getDeType() == 2) { - String l = Bytes.toString(result.getValue(column_family.getBytes(), x.getOriginName().getBytes())); - if (StringUtils.isEmpty(l)) { - l = "0"; + + JavaRDD rdd = pairRDD.mapPartitions((FlatMapFunction>, Row>) tuple2Iterator -> { + List iterator = new ArrayList<>(); + while (tuple2Iterator.hasNext()) { + Result result = tuple2Iterator.next()._2; + List list = new ArrayList<>(); + xAxis.forEach(x -> { + if (x.getDeType() == 0 || x.getDeType() == 1) { + list.add(Bytes.toString(result.getValue(column_family.getBytes(), x.getOriginName().getBytes()))); + } else if (x.getDeType() == 2) { + String l = Bytes.toString(result.getValue(column_family.getBytes(), x.getOriginName().getBytes())); + if (StringUtils.isEmpty(l)) { + l = "0"; + } + list.add(Long.valueOf(l)); } - list.add(Long.valueOf(l)); - } - }); - yAxis.forEach(y -> { - if (y.getDeType() == 0 || y.getDeType() == 1) { - list.add(Bytes.toString(result.getValue(column_family.getBytes(), y.getOriginName().getBytes()))); - } else if (y.getDeType() == 2) { - String l = Bytes.toString(result.getValue(column_family.getBytes(), y.getOriginName().getBytes())); - if (StringUtils.isEmpty(l)) { - l = "0"; + }); + yAxis.forEach(y -> { + if (y.getDeType() == 0 || y.getDeType() == 1) { + list.add(Bytes.toString(result.getValue(column_family.getBytes(), y.getOriginName().getBytes()))); + } else if (y.getDeType() == 2) { + String l = Bytes.toString(result.getValue(column_family.getBytes(), y.getOriginName().getBytes())); + if (StringUtils.isEmpty(l)) { + l = "0"; + } + list.add(Long.valueOf(l)); } - list.add(Long.valueOf(l)); - } - }); - return RowFactory.create(list.toArray()); + }); + iterator.add(RowFactory.create(list.toArray())); + } + return iterator.iterator(); }); List structFields = new ArrayList<>(); @@ -104,10 +110,8 @@ public class SparkCalc { Dataset sql = sqlContext.sql(getSQL(xAxis, yAxis, tmpTable)); - List data = new ArrayList<>(); - // transform -// List list = sql.javaRDD().collect(); + List data = new ArrayList<>(); List list = sql.collectAsList(); for (Row row : list) { String[] r = new String[row.length()]; @@ -117,16 +121,6 @@ public class SparkCalc { data.add(r); } -// Iterator rowIterator = sql.toLocalIterator(); -// while (rowIterator.hasNext()){ -// Row row = rowIterator.next(); -// String[] r = new String[row.length()]; -// for (int i = 0; i < row.length(); i++) { -// r[i] = row.get(i).toString(); -// } -// data.add(r); -// } - return data; } diff --git a/frontend/src/views/chart/group/Group.vue b/frontend/src/views/chart/group/Group.vue index f6c6a58882..fc76082420 100644 --- a/frontend/src/views/chart/group/Group.vue +++ b/frontend/src/views/chart/group/Group.vue @@ -148,7 +148,7 @@ class="tree-list" @node-click="sceneClick" > - + {{ data.name }} @@ -581,6 +581,15 @@ export default { } .custom-tree-node { + flex: 1; + display: flex; + align-items: center; + justify-content: space-between; + font-size: 14px; + padding-right:8px; + } + + .custom-tree-node-list { flex: 1; display: flex; align-items: center; diff --git a/frontend/src/views/dataset/group/Group.vue b/frontend/src/views/dataset/group/Group.vue index fec0e9999a..e2b6bf3428 100644 --- a/frontend/src/views/dataset/group/Group.vue +++ b/frontend/src/views/dataset/group/Group.vue @@ -173,7 +173,7 @@ class="tree-list" @node-click="sceneClick" > - + ({{ data.type }}) @@ -595,6 +595,15 @@ export default { } .custom-tree-node { + flex: 1; + display: flex; + align-items: center; + justify-content: space-between; + font-size: 14px; + padding-right:8px; + } + + .custom-tree-node-list { flex: 1; display: flex; align-items: center;