diff --git a/backend/pom.xml b/backend/pom.xml index 80a883c497..177a028f21 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -17,6 +17,7 @@ 1.8 20.1.0 3.12.1 + 3.1.1 @@ -333,6 +334,36 @@ test + + org.apache.spark + spark-core_2.12 + ${spark.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + provided + + + + org.apache.spark + spark-streaming_2.12 + ${spark.version} + provided + + + + org.apache.spark + spark-sql_2.12 + ${spark.version} + + @@ -439,33 +470,33 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - + + org.apache.maven.plugins + maven-antrun-plugin + + + main-class-placement + generate-resources + + + + + + + + + + + + + + + + run + + + + diff --git a/backend/src/main/java/io/dataease/auth/config/ShiroConfig.java b/backend/src/main/java/io/dataease/auth/config/ShiroConfig.java index c8cdc9214f..4767cda249 100644 --- a/backend/src/main/java/io/dataease/auth/config/ShiroConfig.java +++ b/backend/src/main/java/io/dataease/auth/config/ShiroConfig.java @@ -53,7 +53,8 @@ public class ShiroConfig { /*filterMap.put("jwt", jwtFilter);*/ filterMap.put("logout", new F2CLogoutFilter()); factoryBean.setSecurityManager(securityManager); - factoryBean.setUnauthorizedUrl("/permissionMiss"); + factoryBean.setLoginUrl("/index.html"); + factoryBean.setUnauthorizedUrl("/index.html"); factoryBean.setFilterChainDefinitionMap(shiroService.loadFilterChainDefinitionMap()); factoryBean.setFilters(filterMap); return factoryBean; diff --git a/backend/src/main/java/io/dataease/auth/config/WhitelistConfig.java b/backend/src/main/java/io/dataease/auth/config/WhitelistConfig.java deleted file mode 100644 index 357222cc20..0000000000 --- a/backend/src/main/java/io/dataease/auth/config/WhitelistConfig.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.dataease.auth.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -@Configuration -@ConfigurationProperties(prefix = "dataease") -@Data -public class WhitelistConfig { - - private List whitelist; - - -} diff --git a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java index ef4feab900..8711b9d27b 100644 --- a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java +++ b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java @@ -1,13 +1,9 @@ package io.dataease.auth.service.impl; -import io.dataease.auth.config.WhitelistConfig; import io.dataease.auth.service.ShiroService; -import org.apache.commons.collections.CollectionUtils; import org.apache.shiro.spring.web.ShiroFilterFactoryBean; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; @Service @@ -15,8 +11,7 @@ public class ShiroServiceImpl implements ShiroService { private final static String ANON = "anon"; - @Autowired - private WhitelistConfig whitelistConfig; + @Override public Map loadFilterChainDefinitionMap() { @@ -47,11 +42,7 @@ public class ShiroServiceImpl implements ShiroService { filterChainDefinitionMap.put("/downline", ANON); filterChainDefinitionMap.put("/common-files/**", ANON); - List whitelist = whitelistConfig.getWhitelist(); - if (CollectionUtils.isNotEmpty(whitelist)) - whitelist.forEach(path -> { - filterChainDefinitionMap.put(path, ANON); - }); + filterChainDefinitionMap.put("/api/auth/logout", "logout"); filterChainDefinitionMap.put("/**", "jwt"); diff --git a/backend/src/main/java/io/dataease/auth/util/JWTUtils.java b/backend/src/main/java/io/dataease/auth/util/JWTUtils.java index b43ae98bd3..7219376e10 100644 --- a/backend/src/main/java/io/dataease/auth/util/JWTUtils.java +++ b/backend/src/main/java/io/dataease/auth/util/JWTUtils.java @@ -20,9 +20,9 @@ public class JWTUtils { // token过期时间1min (过期会自动刷新续命 目的是避免一直都是同一个token ) - private static final long EXPIRE_TIME = 1*60*1000/2; + private static final long EXPIRE_TIME = 5*60*1000; // 登录间隔时间10min 超过这个时间强制重新登录 - private static final long Login_Interval = 20*60*1000; + private static final long Login_Interval = 30*60*1000; /** @@ -81,6 +81,7 @@ public class JWTUtils { public static boolean loginExpire(String token){ Long now = System.currentTimeMillis(); Long lastOperateTime = tokenLastOperateTime(token); + if (lastOperateTime == null) return true; return now - lastOperateTime > Login_Interval; } diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.java b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.java new file mode 100644 index 0000000000..0303cadd8d --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.java @@ -0,0 +1,11 @@ +package io.dataease.base.mapper.ext; + +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.ext.query.GridExample; + +import java.util.List; + +public interface ExtDataSourceMapper { + + List query(GridExample example); +} diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml new file mode 100644 index 0000000000..4d4988ec9c --- /dev/null +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDataSourceMapper.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.xml index 9dcec17d12..a47f873a84 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.xml @@ -12,7 +12,7 @@ - select dept_id as id, pid from sys_dept diff --git a/backend/src/main/java/io/dataease/config/HbaseConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java similarity index 67% rename from backend/src/main/java/io/dataease/config/HbaseConfig.java rename to backend/src/main/java/io/dataease/config/CommonConfig.java index f7b8bfaece..229ee6a069 100644 --- a/backend/src/main/java/io/dataease/config/HbaseConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -1,6 +1,9 @@ package io.dataease.config; import com.fit2cloud.autoconfigure.QuartzAutoConfiguration; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; @@ -12,7 +15,7 @@ import javax.annotation.Resource; @Configuration @AutoConfigureBefore(QuartzAutoConfiguration.class) -public class HbaseConfig { +public class CommonConfig { @Resource private Environment env; // 保存了配置文件的信息 @@ -27,4 +30,14 @@ public class HbaseConfig { configuration.set("hbase.client.retries.number", env.getProperty("hbase.client.retries.number", "1")); return configuration; } + + + @Bean + @ConditionalOnMissingBean + public JavaSparkContext javaSparkContext(){ + SparkConf conf = new SparkConf().setAppName(env.getProperty("spark.appName", "DataeaseJob") ).setMaster(env.getProperty("spark.master", "local[*]") ); + SparkSession spark = SparkSession.builder().config(conf).getOrCreate(); + JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + return sc; + } } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 5660a44244..827386e0bb 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -65,4 +65,9 @@ public class DataSetTableController { public Map getPreviewData(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { return dataSetTableService.getPreviewData(dataSetTableRequest); } + + @PostMapping("sqlPreview") + public Map getSQLPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { + return dataSetTableService.getSQLPreview(dataSetTableRequest); + } } diff --git a/backend/src/main/java/io/dataease/controller/sys/SysDeptController.java b/backend/src/main/java/io/dataease/controller/sys/SysDeptController.java index 0f922996ef..1576da9a98 100644 --- a/backend/src/main/java/io/dataease/controller/sys/SysDeptController.java +++ b/backend/src/main/java/io/dataease/controller/sys/SysDeptController.java @@ -8,6 +8,7 @@ import io.dataease.controller.sys.request.DeptCreateRequest; import io.dataease.controller.sys.request.DeptDeleteRequest; import io.dataease.controller.sys.request.DeptStatusRequest; import io.dataease.controller.sys.response.DeptNodeResponse; +import io.dataease.controller.sys.response.DeptTreeNode; import io.dataease.service.sys.DeptService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -82,4 +83,9 @@ public class SysDeptController extends ResultHolder { deptService.updateStatus(request); } + @PostMapping("/nodesByDeptId/{deptId}") + public List nodesByDeptId(@PathVariable("deptId") Long deptId){ + return deptService.searchTree(deptId); + } + } diff --git a/backend/src/main/java/io/dataease/controller/sys/response/DeptTreeNode.java b/backend/src/main/java/io/dataease/controller/sys/response/DeptTreeNode.java new file mode 100644 index 0000000000..10fcc4ebf6 --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/sys/response/DeptTreeNode.java @@ -0,0 +1,27 @@ +package io.dataease.controller.sys.response; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DeptTreeNode implements Serializable { + + private Long id; + + private String label; + + private Boolean hasChildren; + + private List children; + + public List toList(){ + List lists = new ArrayList<>(); + lists.add(this); + return lists; + } +} diff --git a/backend/src/main/java/io/dataease/datasource/controller/DatasourceController.java b/backend/src/main/java/io/dataease/datasource/controller/DatasourceController.java index b019df64a5..c04b3fc470 100644 --- a/backend/src/main/java/io/dataease/datasource/controller/DatasourceController.java +++ b/backend/src/main/java/io/dataease/datasource/controller/DatasourceController.java @@ -5,6 +5,7 @@ import com.github.pagehelper.PageHelper; import io.dataease.base.domain.Datasource; import io.dataease.commons.utils.PageUtils; import io.dataease.commons.utils.Pager; +import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.datasource.service.DatasourceService; import org.springframework.web.bind.annotation.*; @@ -34,9 +35,10 @@ public class DatasourceController { } @PostMapping("/list/{goPage}/{pageSize}") - public Pager> getDatasourceList(@RequestBody Datasource request, @PathVariable int goPage, @PathVariable int pageSize) throws Exception { + public Pager> getDatasourceList(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) throws Exception { Page page = PageHelper.startPage(goPage, pageSize, true); - return PageUtils.setPageInfo(page, datasourceService.getDatasourceList(request)); + // return PageUtils.setPageInfo(page, datasourceService.getDatasourceList(request)); + return PageUtils.setPageInfo(page, datasourceService.gridQuery(request)); } @GetMapping("/delete/{datasourceID}") diff --git a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java index d1dc06956d..71b513471f 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java @@ -1,9 +1,11 @@ package io.dataease.datasource.provider; +import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.Datasource; import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.request.DatasourceRequest; +import java.sql.ResultSet; import java.util.ArrayList; import java.util.List; @@ -13,6 +15,8 @@ public abstract class DatasourceProvider { abstract public List getData(DatasourceRequest datasourceRequest) throws Exception; + abstract public ResultSet getDataResultSet(DatasourceRequest datasourceRequest) throws Exception; + abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception{ @@ -27,4 +31,8 @@ public abstract class DatasourceProvider { abstract public List getPageData(DatasourceRequest datasourceRequest) throws Exception; + abstract public List fetchResult(ResultSet rs) throws Exception; + + abstract public List fetchResultField(ResultSet rs) throws Exception; + } diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 4b53abfb49..6ef5ee0b1d 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -1,6 +1,7 @@ package io.dataease.datasource.provider; import com.google.gson.Gson; +import io.dataease.base.domain.DatasetTableField; import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.MysqlConfigrationDTO; import io.dataease.datasource.dto.SqlServerConfigration; @@ -8,12 +9,13 @@ import io.dataease.datasource.dto.TableFiled; import io.dataease.datasource.request.DatasourceRequest; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; + import java.sql.*; import java.text.MessageFormat; import java.util.*; @Service("jdbc") -public class JdbcProvider extends DatasourceProvider{ +public class JdbcProvider extends DatasourceProvider { @Override @@ -25,33 +27,48 @@ public class JdbcProvider extends DatasourceProvider{ ResultSet rs = stat.executeQuery(datasourceRequest.getQuery()) ) { list = fetchResult(rs); - } catch (SQLException e){ + } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); - }catch (Exception e) { + } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); } return list; } + @Override + public ResultSet getDataResultSet(DatasourceRequest datasourceRequest) throws Exception { + ResultSet rs; + try { + Connection connection = getConnection(datasourceRequest); + Statement stat = connection.createStatement(); + rs = stat.executeQuery(datasourceRequest.getQuery()); + } catch (SQLException e) { + throw new Exception("ERROR:" + e.getMessage(), e); + } catch (Exception e) { + throw new Exception("ERROR:" + e.getMessage(), e); + } + return rs; + } + @Override public List getPageData(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); try ( Connection connection = getConnection(datasourceRequest); Statement stat = connection.createStatement(); - ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() -1)*datasourceRequest.getPageSize(), datasourceRequest.getPageSize())) + ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() - 1) * datasourceRequest.getPageSize(), datasourceRequest.getPageSize())) ) { list = fetchResult(rs); - } catch (SQLException e){ + } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); - }catch (Exception e) { + } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); } return list; } - - private List fetchResult( ResultSet rs) throws Exception{ + @Override + public List fetchResult(ResultSet rs) throws Exception { List list = new LinkedList<>(); ResultSetMetaData metaData = rs.getMetaData(); int columnCount = metaData.getColumnCount(); @@ -73,13 +90,31 @@ public class JdbcProvider extends DatasourceProvider{ return list; } + @Override + public List fetchResultField(ResultSet rs) throws Exception { + List fieldList = new ArrayList<>(); + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int j = 0; j < columnCount; j++) { + String f = metaData.getColumnName(j + 1); + String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f; + String t = metaData.getColumnTypeName(j + 1); + TableFiled field = new TableFiled(); + field.setFieldName(l); + field.setRemarks(l); + field.setFieldType(t); + fieldList.add(field); + } + return fieldList; + } + @Override public List getTables(DatasourceRequest datasourceRequest) throws Exception { List tables = new ArrayList<>(); String queryStr = getTablesSql(datasourceRequest); try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) { ResultSet resultSet = ps.executeQuery(queryStr); - while (resultSet.next()){ + while (resultSet.next()) { tables.add(resultSet.getString(1)); } } catch (Exception e) { @@ -89,22 +124,22 @@ public class JdbcProvider extends DatasourceProvider{ } @Override - public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception{ + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { List list = new LinkedList<>(); try ( - Connection connection = getConnection(datasourceRequest); + Connection connection = getConnection(datasourceRequest); ) { DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet resultSet = databaseMetaData.getColumns(null, "%", datasourceRequest.getTable().toUpperCase(), "%"); while (resultSet.next()) { String tableName = resultSet.getString("TABLE_NAME"); String database = resultSet.getString("TABLE_CAT"); - if(tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))){ + if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) { TableFiled tableFiled = new TableFiled(); String colName = resultSet.getString("COLUMN_NAME"); tableFiled.setFieldName(colName); String remarks = resultSet.getString("REMARKS"); - if(remarks == null || remarks.equals("")){ + if (remarks == null || remarks.equals("")) { remarks = colName; } tableFiled.setRemarks(remarks); @@ -113,13 +148,15 @@ public class JdbcProvider extends DatasourceProvider{ list.add(tableFiled); } } - } catch (SQLException e){ + } catch (SQLException e) { throw new Exception("ERROR:" + e.getMessage(), e); - }catch (Exception e) { + } catch (Exception e) { throw new Exception("ERROR:" + e.getMessage(), e); } return list; - }; + } + + ; @Override public void test(DatasourceRequest datasourceRequest) throws Exception { @@ -132,7 +169,7 @@ public class JdbcProvider extends DatasourceProvider{ } - public Long count(DatasourceRequest datasourceRequest)throws Exception{ + public Long count(DatasourceRequest datasourceRequest) throws Exception { try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) { ResultSet resultSet = ps.executeQuery(datasourceRequest.getQuery()); while (resultSet.next()) { @@ -150,16 +187,16 @@ public class JdbcProvider extends DatasourceProvider{ String driver = null; String jdbcurl = null; DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); - switch (datasourceType){ + switch (datasourceType) { case mysql: - MysqlConfigrationDTO mysqlConfigrationDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigrationDTO.class); + MysqlConfigrationDTO mysqlConfigrationDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfigrationDTO.class); username = mysqlConfigrationDTO.getUsername(); password = mysqlConfigrationDTO.getPassword(); driver = mysqlConfigrationDTO.getDriver(); jdbcurl = mysqlConfigrationDTO.getJdbc(); break; case sqlServer: - SqlServerConfigration sqlServerConfigration= new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); + SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); username = sqlServerConfigration.getUsername(); password = sqlServerConfigration.getPassword(); driver = sqlServerConfigration.getDriver(); @@ -178,7 +215,7 @@ public class JdbcProvider extends DatasourceProvider{ return DriverManager.getConnection(jdbcurl, props); } - private String getDatabase(DatasourceRequest datasourceRequest){ + private String getDatabase(DatasourceRequest datasourceRequest) { DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); switch (datasourceType) { case mysql: @@ -192,9 +229,9 @@ public class JdbcProvider extends DatasourceProvider{ } } - private String getTablesSql(DatasourceRequest datasourceRequest){ + private String getTablesSql(DatasourceRequest datasourceRequest) { DatasourceTypes datasourceType = DatasourceTypes.valueOf(datasourceRequest.getDatasource().getType()); - switch (datasourceType){ + switch (datasourceType) { case mysql: return "show tables;"; case sqlServer: diff --git a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java index 5924dd3d41..24f6b2a4c0 100644 --- a/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java +++ b/backend/src/main/java/io/dataease/datasource/service/DatasourceService.java @@ -2,7 +2,10 @@ package io.dataease.datasource.service; import io.dataease.base.domain.*; import io.dataease.base.mapper.*; +import io.dataease.base.mapper.ext.ExtDataSourceMapper; +import io.dataease.base.mapper.ext.query.GridExample; import io.dataease.commons.exception.DEException; +import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; @@ -22,6 +25,9 @@ public class DatasourceService { @Resource private DatasourceMapper datasourceMapper; + @Resource + private ExtDataSourceMapper extDataSourceMapper; + public Datasource addDatasource(Datasource datasource) { DatasourceExample example = new DatasourceExample(); example.createCriteria().andNameEqualTo(datasource.getName()); @@ -49,6 +55,11 @@ public class DatasourceService { return datasourceMapper.selectByExampleWithBLOBs(example); } + public List gridQuery(BaseGridRequest request){ + GridExample gridExample = request.convertExample(); + return extDataSourceMapper.query(gridExample); + } + public void deleteDatasource(String datasourceId) { datasourceMapper.deleteByPrimaryKey(datasourceId); } diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java index 3c409114d8..8400020daa 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java @@ -18,7 +18,6 @@ public abstract class DeScheduleJob implements Job { this.taskId = jobDataMap.getString("taskId"); LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId); - LogUtil.info(jobKey.getName() + " Running: " + datasetTableId); LogUtil.info("CronExpression: " + expression); businessExecute(context); } diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 9dd307c471..381362aadc 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -102,8 +102,13 @@ public class ChartViewService { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); - datasourceRequest.setTable(dataTableInfoDTO.getTable()); - datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xAxis, yAxis)); + if (StringUtils.equalsIgnoreCase(table.getType(), "db")) { + datasourceRequest.setTable(dataTableInfoDTO.getTable()); + datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xAxis, yAxis)); + } else if (StringUtils.equalsIgnoreCase(table.getType(), "sql")) { + datasourceRequest.setQuery(getSQL(ds.getType(), " (" + dataTableInfoDTO.getSql() + ") AS tmp ", xAxis, yAxis)); + } + List data = datasourceProvider.getData(datasourceRequest); // todo 处理结果,目前做一个单系列图表，后期图表组件再扩展 diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 518042cb89..eb7e2576d4 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.sql.ResultSet; import java.text.MessageFormat; import java.util.*; import java.util.stream.Collectors; @@ -53,8 +54,8 @@ public class DataSetTableService { DataTableInfoDTO dataTableInfoDTO = new DataTableInfoDTO(); if (StringUtils.equalsIgnoreCase("db", datasetTable.getType())) { dataTableInfoDTO.setTable(datasetTable.getName()); + datasetTable.setInfo(new Gson().toJson(dataTableInfoDTO)); } - datasetTable.setInfo(new Gson().toJson(dataTableInfoDTO)); int insert = datasetTableMapper.insert(datasetTable); // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库 if (insert == 1) { @@ -142,15 +143,22 @@ public class DataSetTableService { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(ds); - String table = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getTable(); DatasetTableField datasetTableField = DatasetTableField.builder().build(); datasetTableField.setTableId(dataSetTableRequest.getId()); datasetTableField.setChecked(Boolean.TRUE); List fields = dataSetTableFieldsService.list(datasetTableField); - String[] fieldArray = fields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new); - datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT 0,10"); + + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(dataSetTableRequest.getId()); + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) { + String table = dataTableInfoDTO.getTable(); + datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT 0,10");// todo limit + } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) { + String sql = dataTableInfoDTO.getSql(); + datasourceRequest.setQuery(createQuerySQL(ds.getType(), " (" + sql + ") AS tmp ", fieldArray));// todo 因为编辑可能取消某些字段展示，这里sql看看怎么处理 + } List data = new ArrayList<>(); try { @@ -169,6 +177,35 @@ public class DataSetTableService { }).collect(Collectors.toList()); } + Map map = new HashMap<>(); + map.put("fields", fields); + map.put("data", jsonArray); + + return map; + } + + public Map getSQLPreview(DataSetTableRequest dataSetTableRequest) throws Exception { + Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + String sql = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getSql(); + datasourceRequest.setQuery(sql); + ResultSet dataResultSet = datasourceProvider.getDataResultSet(datasourceRequest); + List data = datasourceProvider.fetchResult(dataResultSet); + List fields = datasourceProvider.fetchResultField(dataResultSet); + String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + + List> jsonArray = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.length; i++) { + map.put(fieldArray[i], ele[i]); + } + return map; + }).collect(Collectors.toList()); + } Map map = new HashMap<>(); map.put("fields", fields); @@ -228,8 +265,19 @@ public class DataSetTableService { Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); DataSetTableRequest dataSetTableRequest = new DataSetTableRequest(); BeanUtils.copyBean(dataSetTableRequest, datasetTable); - List fields = getFields(dataSetTableRequest); + + List fields = new ArrayList<>(); long syncTime = System.currentTimeMillis(); + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) { + fields = getFields(dataSetTableRequest); + } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + datasourceRequest.setQuery(new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getSql()); + ResultSet dataResultSet = datasourceProvider.getDataResultSet(datasourceRequest); + fields = datasourceProvider.fetchResultField(dataResultSet); + } if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { TableFiled filed = fields.get(i); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 037fe397d5..248d98f497 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -6,6 +6,7 @@ import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.utils.CommonBeanFactory; +import io.dataease.commons.utils.LogUtil; import io.dataease.dto.dataset.DataTableInfoDTO; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; @@ -76,6 +77,9 @@ public class ExtractDataService { datasetTableTaskLog.setEndTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); }catch (Exception e){ + e.printStackTrace(); + LogUtil.error("ExtractData error, dataaset: " + datasetTableId); + LogUtil.error(e.getMessage(), e); datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setEndTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); diff --git a/backend/src/main/java/io/dataease/service/sys/DeptService.java b/backend/src/main/java/io/dataease/service/sys/DeptService.java index 290d3468af..d33adcd36e 100644 --- a/backend/src/main/java/io/dataease/service/sys/DeptService.java +++ b/backend/src/main/java/io/dataease/service/sys/DeptService.java @@ -8,10 +8,12 @@ import io.dataease.base.mapper.ext.query.GridExample; import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.controller.sys.base.ConditionEntity; import io.dataease.controller.sys.request.DeptCreateRequest; import io.dataease.controller.sys.request.DeptDeleteRequest; import io.dataease.controller.sys.request.DeptStatusRequest; import io.dataease.controller.sys.request.SimpleTreeNode; +import io.dataease.controller.sys.response.DeptTreeNode; import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -139,6 +141,33 @@ public class DeptService { return sysDepts; } + public List searchTree(Long deptId){ + List roots = nodesByPid(0L); + if (deptId == DEPT_ROOT_PID) return roots.stream().map(this::format).collect(Collectors.toList()); + SysDept sysDept = sysDeptMapper.selectByPrimaryKey(deptId); + if (roots.stream().anyMatch(node -> node.getDeptId() == deptId)) return roots.stream().map(this::format).collect(Collectors.toList()); + SysDept current = sysDept; + DeptTreeNode currentNode = format(sysDept); + while (current.getPid() != DEPT_ROOT_PID){ + SysDept parent = sysDeptMapper.selectByPrimaryKey(current.getPid()); //pid上有索引 所以效率不会太差 + DeptTreeNode parentNode = format(parent); + parentNode.setChildren(currentNode.toList()); + current = parent; + currentNode = parentNode; + } + + DeptTreeNode targetRootNode = currentNode; + return roots.stream().map(node -> node.getDeptId() == targetRootNode.getId() ? targetRootNode : format(node)).collect(Collectors.toList()); + } + + private DeptTreeNode format(SysDept sysDept){ + DeptTreeNode deptTreeNode = new DeptTreeNode(); + deptTreeNode.setId(sysDept.getDeptId()); + deptTreeNode.setLabel(sysDept.getName()); + deptTreeNode.setHasChildren(sysDept.getSubCount() > 0); + return deptTreeNode; + } + private DeptService proxy(){ return CommonBeanFactory.getBean(DeptService.class); } diff --git a/frontend/.env.production b/frontend/.env.production index a33fdd2479..d24a4671c1 100644 --- a/frontend/.env.production +++ b/frontend/.env.production @@ -2,5 +2,6 @@ ENV = 'production' # base api -VUE_APP_BASE_API = 'http://localhost:8081/' +# VUE_APP_BASE_API = 'http://localhost:8081/' +VUE_APP_BASE_API = '/' diff --git a/frontend/package.json b/frontend/package.json index 9cadee7c56..194e165f0f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,7 +27,9 @@ "screenfull": "4.2.0", "svg-sprite-loader": "4.1.3", "svgo": "1.2.2", + "umy-ui": "^1.1.6", "vue": "2.6.10", + "vue-codemirror": "^4.0.6", "vue-i18n": "7.3.2", "vue-router": "3.0.6", "vuedraggable": "^2.24.3", diff --git a/frontend/src/api/system/datasource.js b/frontend/src/api/system/datasource.js index 611338fed1..a8deddbff8 100644 --- a/frontend/src/api/system/datasource.js +++ b/frontend/src/api/system/datasource.js @@ -4,6 +4,7 @@ export function dsGrid(pageIndex, pageSize, data) { return request({ url: 'datasource/list/' + pageIndex + '/' + pageSize, method: 'post', + loading: true, data }) } diff --git a/frontend/src/api/system/dept.js b/frontend/src/api/system/dept.js index e322d2f08d..a371ee9998 100644 --- a/frontend/src/api/system/dept.js +++ b/frontend/src/api/system/dept.js @@ -41,4 +41,11 @@ export function editDept(data) { }) } -export default { addDept, delDept, editDept, getDeptTree, loadTable } +export function treeByDeptId(deptId) { + return request({ + url: '/api/dept/nodesByDeptId/' + deptId, + method: 'post' + }) +} + +export default { addDept, delDept, editDept, getDeptTree, loadTable, treeByDeptId } diff --git a/frontend/src/components/business/tree-table/index.vue b/frontend/src/components/business/tree-table/index.vue new file mode 100644 index 0000000000..c253f9621d --- /dev/null +++ b/frontend/src/components/business/tree-table/index.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/frontend/src/components/vue-drag-resize-rotate/index.vue b/frontend/src/components/vue-drag-resize-rotate/index.vue index 5146a30394..0a01e28e04 100644 --- a/frontend/src/components/vue-drag-resize-rotate/index.vue +++ b/frontend/src/components/vue-drag-resize-rotate/index.vue @@ -69,10 +69,6 @@ export default { replace: true, name: 'vue-drag-resize-rotate', props: { - viewId: { - type: String, - default: '' - }, className: { type: String, default: 'vdr' @@ -399,11 +395,6 @@ export default { }, methods: { - removeView(){ - debugger - console.log(this.viewId); - this.$emit('removeView',this.viewId) - }, // 重置边界和鼠标状态 resetBoundsAndMouseState() { this.mouseClickPosition = { mouseX: 0, mouseY: 0, x: 0, y: 0, w: 0, h: 0 } @@ -1320,15 +1311,13 @@ export default { } }, style() { - let newStyle ={ + return { transform: `translate(${this.left}px, ${this.top}px) rotate(${this.rotate}deg)`, width: this.computedWidth, height: this.computedHeight, zIndex: this.zIndex, ...(this.dragging && this.disableUserSelect ? userSelectNone : userSelectAuto) }; - this.$emit('newStyle', this.viewId,newStyle); - return newStyle; }, // 控制柄显示与否 actualHandles() { @@ -1515,11 +1504,4 @@ export default { height: 7px; background-color: #666; } - -.close { - float: right; - padding-top: 8px; - padding-bottom: 8px; -} - diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index efd0a7e737..ee5fc92263 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -666,7 +666,9 @@ export default { close: '关闭', required: '必填', input_content: '请输入内容', - add_sql_table: '添加SQL' + add_sql_table: '添加SQL', + preview: '预览', + pls_input_name: '请输入名称' }, datasource: { create: '新建数据连接', @@ -682,7 +684,7 @@ export default { please_input_password: '请输入密码', please_input_host: '请输入主机', please_input_port: '请输入端口', - modify: '修改组织', + modify: '编辑数据连接', validate_success: '校验成功', delete: '删除组织', delete_confirm: '删除该组织会关联删除该组织下的所有资源(如：相关工作空间，项目，测试用例等），确定要删除吗?', diff --git a/frontend/src/main.js b/frontend/src/main.js index 793d8b5cfc..2045dcf5e7 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -31,6 +31,10 @@ import * as echarts from 'echarts' Vue.prototype.$echarts = echarts +import UmyUi from 'umy-ui' +import 'umy-ui/lib/theme-chalk/index.css'// 引入样式 +Vue.use(UmyUi) + /** * If you don't want to use mock-server * you want to use MockJs for mock api diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 26ae503f3e..de8da4cb05 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -178,7 +178,8 @@ export const constantRoutes = [ ] const createRouter = () => new Router({ - mode: 'history', // require service support + // mode: 'history', // require service support + mode: 'hash', scrollBehavior: () => ({ y: 0 }), routes: constantRoutes }) diff --git a/frontend/src/views/chart/components/QuotaItem.vue b/frontend/src/views/chart/components/QuotaItem.vue index dcf0f26220..c616da6676 100644 --- a/frontend/src/views/chart/components/QuotaItem.vue +++ b/frontend/src/views/chart/components/QuotaItem.vue @@ -3,7 +3,7 @@ - {{ item.name }}{{ $t('chart.'+item.summary) }} + {{ item.name }}{{ $t('chart.'+item.summary) }} diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 7aa484402d..5163cf3d7c 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -291,10 +291,10 @@ export default { this.removeCheckedKey(e) this.save() }, + // 右边往左边拖动时的事件 start2(e) { console.log(e) }, - // 右边往左边拖动时的事件 end2(e) { console.log(e) this.removeDuplicateKey(e) @@ -394,8 +394,8 @@ export default { } .item { - padding: 3px 10px; - margin: 3px 3px 0 3px; + padding: 2px 10px; + margin: 2px 2px 0 2px; border: solid 1px #eee; text-align: left; color: #606266; @@ -403,15 +403,15 @@ export default { } .item-on-move { - padding: 3px 10px; - margin: 3px 3px 0 3px; + padding: 2px 10px; + margin: 2px 2px 0 2px; border: solid 1px #eee; text-align: left; color: #606266; } .item + .item { - margin-top: 3px; + margin-top: 2px; } .item:hover { diff --git a/frontend/src/views/dataset/add/AddDB.vue b/frontend/src/views/dataset/add/AddDB.vue index f47fd53219..99313ac5b0 100644 --- a/frontend/src/views/dataset/add/AddDB.vue +++ b/frontend/src/views/dataset/add/AddDB.vue @@ -42,7 +42,7 @@ - + diff --git a/frontend/src/views/dataset/add/AddSQL.vue b/frontend/src/views/dataset/add/AddSQL.vue index 4d716fd4a4..159c10325e 100644 --- a/frontend/src/views/dataset/add/AddSQL.vue +++ b/frontend/src/views/dataset/add/AddSQL.vue @@ -9,7 +9,7 @@ {{ $t('dataset.cancel') }} - + {{ $t('dataset.confirm') }} @@ -27,29 +27,133 @@ /> + + + + + {{ $t('dataset.direct_connect') }} + {{ $t('dataset.sync_data') }} + + + + + + + + +
+ {{ $t('dataset.data_preview') }} + {{ $t('dataset.preview') }} +
+
+ + + +
+ + diff --git a/frontend/src/views/system/dept/index.vue b/frontend/src/views/system/dept/index.vue index 2e9b5d8cc9..fd65bcdbdb 100644 --- a/frontend/src/views/system/dept/index.vue +++ b/frontend/src/views/system/dept/index.vue @@ -1,7 +1,7 @@