diff --git a/backend/pom.xml b/backend/pom.xml index 68fb99604a..8316c908e9 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -5,7 +5,7 @@ dataease-server io.dataease - 1.3.0 + 1.4.0 4.0.0 @@ -201,7 +201,7 @@ io.dataease dataease-plugin-interface - 1.3 + 1.4 cn.hutool diff --git a/backend/src/main/java/io/dataease/auth/api/AuthApi.java b/backend/src/main/java/io/dataease/auth/api/AuthApi.java index f5e72afbb0..2af165a370 100644 --- a/backend/src/main/java/io/dataease/auth/api/AuthApi.java +++ b/backend/src/main/java/io/dataease/auth/api/AuthApi.java @@ -5,6 +5,9 @@ import io.dataease.auth.api.dto.CurrentUserDto; import io.dataease.auth.api.dto.LoginDto; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.GetMapping; +import springfox.documentation.annotations.ApiIgnore; + import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -49,4 +52,12 @@ public interface AuthApi { @PostMapping("/isOpenOidc") boolean isOpenOidc(); + @ApiIgnore + @PostMapping("/isPluginLoaded") + boolean isPluginLoaded(); + + @ApiIgnore + @GetMapping("/getPublicKey") + String getPublicKey(); + } diff --git a/backend/src/main/java/io/dataease/auth/server/AuthServer.java b/backend/src/main/java/io/dataease/auth/server/AuthServer.java index 96979bfda7..73fea40a4f 100644 --- a/backend/src/main/java/io/dataease/auth/server/AuthServer.java +++ b/backend/src/main/java/io/dataease/auth/server/AuthServer.java @@ -10,10 +10,7 @@ import io.dataease.auth.entity.TokenInfo; import io.dataease.auth.service.AuthUserService; import io.dataease.auth.util.JWTUtils; import io.dataease.auth.util.RsaUtil; -import io.dataease.commons.utils.BeanUtils; -import io.dataease.commons.utils.CodingUtil; -import io.dataease.commons.utils.LogUtil; -import io.dataease.commons.utils.ServletUtils; +import io.dataease.commons.utils.*; import io.dataease.controller.sys.request.LdapAddRequest; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; @@ -55,10 +52,9 @@ public class AuthServer implements AuthApi { @Override public Object login(@RequestBody LoginDto loginDto) throws Exception { - String username = loginDto.getUsername(); - String password = loginDto.getPassword(); + String username = RsaUtil.decryptByPrivateKey(RsaProperties.privateKey, loginDto.getUsername());; + String pwd = RsaUtil.decryptByPrivateKey(RsaProperties.privateKey, loginDto.getPassword()); - String pwd = RsaUtil.decryptByPrivateKey(RsaProperties.privateKey, password); // 增加ldap登录方式 Integer loginType = loginDto.getLoginType(); boolean isSupportLdap = authUserService.supportLdap(); @@ -184,6 +180,18 @@ public class AuthServer implements AuthApi { return authUserService.supportOidc(); } + @Override + public boolean isPluginLoaded() { + Boolean licValid = PluginUtils.licValid(); + if(!licValid) return false; + return authUserService.pluginLoaded(); + } + + + @Override + public String getPublicKey() { + return RsaProperties.publicKey; + } /*@Override diff --git a/backend/src/main/java/io/dataease/auth/service/AuthUserService.java b/backend/src/main/java/io/dataease/auth/service/AuthUserService.java index a01fe2a3cc..13eee76696 100644 --- a/backend/src/main/java/io/dataease/auth/service/AuthUserService.java +++ b/backend/src/main/java/io/dataease/auth/service/AuthUserService.java @@ -29,6 +29,8 @@ public interface AuthUserService { Boolean supportOidc(); + Boolean pluginLoaded(); + } diff --git a/backend/src/main/java/io/dataease/auth/service/impl/AuthUserServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/AuthUserServiceImpl.java index ebd6febcec..9c65616c9a 100644 --- a/backend/src/main/java/io/dataease/auth/service/impl/AuthUserServiceImpl.java +++ b/backend/src/main/java/io/dataease/auth/service/impl/AuthUserServiceImpl.java @@ -8,6 +8,7 @@ import io.dataease.base.mapper.ext.AuthMapper; import io.dataease.auth.service.AuthUserService; import io.dataease.commons.constants.AuthConstants; import io.dataease.commons.utils.LogUtil; +import io.dataease.plugins.common.service.PluginCommonService; import io.dataease.plugins.config.SpringContextUtil; import io.dataease.plugins.xpack.ldap.service.LdapXpackService; import io.dataease.plugins.xpack.oidc.service.OidcXpackService; @@ -137,5 +138,16 @@ public class AuthUserServiceImpl implements AuthUserService { return oidcXpackService.isSuuportOIDC(); } + @Override + public Boolean pluginLoaded() { + Map beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((PluginCommonService.class)); + if(beansOfType.keySet().size() == 0) return false; + PluginCommonService pluginCommonService = SpringContextUtil.getBean(PluginCommonService.class); + if(ObjectUtils.isEmpty(pluginCommonService)) return false; + return pluginCommonService.isPluginLoaded(); + } + + + } diff --git a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java index aecb307469..f5b5d44a14 100644 --- a/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java +++ b/backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java @@ -56,10 +56,11 @@ public class ShiroServiceImpl implements ShiroService { // filterChainDefinitionMap.put("/axios.map", ANON); filterChainDefinitionMap.put("/api/auth/login", ANON); - // filterChainDefinitionMap.put("/api/auth/logout", ANON); + filterChainDefinitionMap.put("/system/requestTimeOut", ANON); filterChainDefinitionMap.put("/api/auth/validateName", ANON); filterChainDefinitionMap.put("/api/auth/isOpenLdap", ANON); filterChainDefinitionMap.put("/api/auth/isOpenOidc", ANON); + filterChainDefinitionMap.put("/api/auth/getPublicKey", ANON); filterChainDefinitionMap.put("/api/pluginCommon/component/*", ANON); filterChainDefinitionMap.put("/plugin/oidc/authInfo", ANON); filterChainDefinitionMap.put("/sso/callBack*", ANON); diff --git a/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java b/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java index f075b5149c..745b7e274a 100644 --- a/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java +++ b/backend/src/main/java/io/dataease/commons/constants/ParamConstants.java @@ -109,6 +109,18 @@ public interface ParamConstants { } } + enum BASIC implements ParamConstants { + FRONT_TIME_OUT("basic.frontTimeOut"), + MSG_TIME_OUT("basic.msgTimeOut"); + private String value; + public String getValue() { + return this.value; + } + private BASIC(String value) { + this.value = value; + } + } + enum BASE implements ParamConstants { URL("base.url"); diff --git a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java index ac6e4552fc..8454df061b 100755 --- a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java +++ b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java @@ -7,22 +7,26 @@ import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.conn.ssl.TrustStrategy; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentType; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; -import org.apache.http.ssl.SSLContexts; +import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,11 +46,15 @@ public class HttpClientUtil { private static CloseableHttpClient buildHttpClient(String url) { try { if (url.startsWith(HTTPS)) { - // https 增加信任设置 - TrustStrategy trustStrategy = new TrustSelfSignedStrategy(); - SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(trustStrategy).build(); - HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE; - return HttpClients.custom().setSSLContext(sslContext).setSSLHostnameVerifier(hostnameVerifier).build(); + SSLContextBuilder builder = new SSLContextBuilder(); + builder.loadTrustMaterial(null, (X509Certificate[] x509Certificates, String s) -> true); + SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(builder.build(), new String[]{"TLSv1.1", "TLSv1.2", "SSLv3"}, null, NoopHostnameVerifier.INSTANCE); + Registry registry = RegistryBuilder.create() + .register("http", new PlainConnectionSocketFactory()) + .register("https", socketFactory).build(); + HttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(registry); + CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(connManager).build(); + return httpClient; } else { // http return HttpClientBuilder.create().build(); diff --git a/backend/src/main/java/io/dataease/controller/sys/MsgController.java b/backend/src/main/java/io/dataease/controller/sys/MsgController.java index b100aea861..8db6b4cba9 100644 --- a/backend/src/main/java/io/dataease/controller/sys/MsgController.java +++ b/backend/src/main/java/io/dataease/controller/sys/MsgController.java @@ -42,8 +42,9 @@ public class MsgController { List sysMsgTypes = sysMsgService.queryMsgTypes(); typeIds = sysMsgTypes.stream().filter(sysMsgType -> msgRequest.getType() == sysMsgType.getPid()).map(SysMsgType::getMsgTypeId).collect(Collectors.toList()); } + Long overTime = sysMsgService.overTime(); Page page = PageHelper.startPage(goPage, pageSize, true); - Pager> listPager = PageUtils.setPageInfo(page, sysMsgService.queryGrid(userId, msgRequest, typeIds)); + Pager> listPager = PageUtils.setPageInfo(page, sysMsgService.queryGrid(userId, msgRequest, typeIds, overTime)); return listPager; } diff --git a/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java b/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java index bafbffc95f..799c69e045 100644 --- a/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java +++ b/backend/src/main/java/io/dataease/controller/sys/SystemParameterController.java @@ -2,9 +2,13 @@ package io.dataease.controller.sys; import io.dataease.base.domain.SystemParameter; import io.dataease.commons.constants.ParamConstants; +import io.dataease.controller.sys.response.BasicInfo; +import io.dataease.controller.sys.response.MailInfo; import io.dataease.dto.SystemParameterDTO; import io.dataease.service.FileService; import io.dataease.service.system.SystemParameterService; + +import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -28,11 +32,33 @@ public class SystemParameterController { @Resource private FileService fileService; + + @GetMapping("/mail/info") + public MailInfo mailInfo() { + return systemParameterService.mailInfo(ParamConstants.Classify.MAIL.getValue()); + } + + @GetMapping("/basic/info") + public BasicInfo basicInfo() { + return systemParameterService.basicInfo(); + } + + @GetMapping("/requestTimeOut") + public Integer RequestTimeOut() { + BasicInfo basicInfo = systemParameterService.basicInfo(); + return StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10; + } + @PostMapping("/edit/email") public void editMail(@RequestBody List systemParameter) { systemParameterService.editMail(systemParameter); } + @PostMapping("/edit/basic") + public void editBasic(@RequestBody List systemParameter) { + systemParameterService.editBasic(systemParameter); + } + @PostMapping("/testConnection") public void testConnection(@RequestBody HashMap hashMap) { systemParameterService.testConnection(hashMap); @@ -70,4 +96,6 @@ public class SystemParameterController { } + + } diff --git a/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java b/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java new file mode 100644 index 0000000000..19df51477f --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/sys/response/BasicInfo.java @@ -0,0 +1,16 @@ +package io.dataease.controller.sys.response; + +import java.io.Serializable; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@Data +public class BasicInfo implements Serializable{ + + @ApiModelProperty("请求超时时间") + private String frontTimeOut; + @ApiModelProperty("消息保留时间") + private String msgTimeOut; + +} diff --git a/backend/src/main/java/io/dataease/controller/sys/response/MailInfo.java b/backend/src/main/java/io/dataease/controller/sys/response/MailInfo.java new file mode 100644 index 0000000000..291345e9ed --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/sys/response/MailInfo.java @@ -0,0 +1,18 @@ +package io.dataease.controller.sys.response; + +import java.io.Serializable; + +import lombok.Data; + +@Data +public class MailInfo implements Serializable{ + + private String host; + private String port; + private String account; + private String password; + private String ssl; + private String tls; + private String recipient; + +} diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index 93d17034fe..87c76049e9 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -9,6 +9,7 @@ public enum DatasourceTypes { sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), de_doris("de_doris", "de_doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), + mongo("mongo", "mongodb", "com.mongodb.jdbc.MongoDriver", "`", "`", "\"", "\""), ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"), es("es", "es", "", "\"", "\"", "\"", "\""); diff --git a/backend/src/main/java/io/dataease/datasource/dto/MongodbConfiguration.java b/backend/src/main/java/io/dataease/datasource/dto/MongodbConfiguration.java new file mode 100644 index 0000000000..cf7ffae616 --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/MongodbConfiguration.java @@ -0,0 +1,19 @@ +package io.dataease.datasource.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class MongodbConfiguration extends JdbcConfiguration { + + private String driver = "mongodb.jdbc.MongoDriver"; + private String connectionType; + + public String getJdbc() { + return "jdbc:mongodb://HOSTNAME:PORT/DATABASE" + .replace("HOSTNAME", getHost().trim()) + .replace("PORT", getPort().toString().trim()) + .replace("DATABASE", getDataBase().trim()); + } +} diff --git a/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java b/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java index 647585971b..324756bea1 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Service; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.stream.Collectors; @Service("es") public class EsProvider extends DatasourceProvider { @@ -177,6 +178,7 @@ public class EsProvider extends DatasourceProvider { try { String response = exexQuery(datasourceRequest, "show tables", "?format=json"); tables = fetchTables(response); + tables = tables.stream().filter(table -> !table.startsWith(".")).collect(Collectors.toList()); } catch (Exception e) { DataEaseException.throwException(e); } diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 54ef94a908..9e1ad3c566 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -1,6 +1,8 @@ package io.dataease.datasource.provider; +import com.alibaba.druid.filter.Filter; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.wall.WallFilter; import com.google.gson.Gson; import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.*; @@ -10,7 +12,6 @@ import io.dataease.i18n.Translator; import io.dataease.provider.QueryProvider; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; - import javax.annotation.PostConstruct; import java.beans.PropertyVetoException; import java.io.File; @@ -18,12 +19,16 @@ import java.io.IOException; import java.net.URL; import java.sql.*; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Service("jdbc") public class JdbcProvider extends DatasourceProvider { private static Map jdbcConnection = new HashMap<>(); public ExtendedJdbcClassLoader extendedJdbcClassLoader; static private String FILE_PATH = "/opt/dataease/drivers"; + private static final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT"; + public static final Pattern WITH_SQL_FRAGMENT = Pattern.compile(REG_WITH_SQL_FRAGMENT); @PostConstruct public void init() throws Exception{ @@ -62,8 +67,10 @@ public class JdbcProvider extends DatasourceProvider { @Override public List getData(DatasourceRequest dsr) throws Exception { List list = new LinkedList<>(); - try (Connection connection = getConnectionFromPool(dsr); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(dsr.getQuery())){ + try (Connection connection = getConnectionFromPool(dsr); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(dsr.getQuery()) )){ + list = fetchResult(rs); + if(dsr.isPageable() && dsr.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name())){ Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size(); list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize); @@ -89,7 +96,7 @@ public class JdbcProvider extends DatasourceProvider { @Override public List fetchResult(DatasourceRequest datasourceRequest) throws Exception { - try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(datasourceRequest.getQuery())){ + try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))){ return fetchResult(rs); } catch (SQLException e) { DataEaseException.throwException(e); @@ -125,7 +132,7 @@ public class JdbcProvider extends DatasourceProvider { @Override public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { - try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(datasourceRequest.getQuery())){ + try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))){ return fetchResultField(rs, datasourceRequest); } catch (SQLException e) { DataEaseException.throwException(e); @@ -140,7 +147,7 @@ public class JdbcProvider extends DatasourceProvider { Map result = new HashMap<>(); List dataList = new LinkedList<>(); List fieldList = new ArrayList<>(); - try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(datasourceRequest.getQuery())){ + try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))){ dataList = fetchResult(rs); fieldList = fetchResultField(rs, datasourceRequest); result.put("dataList", dataList); @@ -195,13 +202,16 @@ public class JdbcProvider extends DatasourceProvider { } String queryView = getViewSql(datasourceRequest); - try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryView)){ - while (resultSet.next()) { - tables.add(resultSet.getString(1)); + if(queryView != null){ + try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryView)){ + while (resultSet.next()) { + tables.add(resultSet.getString(1)); + } + } catch (Exception e) { + DataEaseException.throwException(e); } - } catch (Exception e) { - DataEaseException.throwException(e); } + return tables; } @@ -318,6 +328,13 @@ public class JdbcProvider extends DatasourceProvider { driver = chConfiguration.getDriver(); jdbcurl = chConfiguration.getJdbc(); break; + case mongo: + MongodbConfiguration mongodbConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MongodbConfiguration.class); + username = mongodbConfiguration.getUsername(); + password = mongodbConfiguration.getPassword(); + driver = mongodbConfiguration.getDriver(); + jdbcurl = mongodbConfiguration.getJdbc(); + break; default: break; } @@ -332,13 +349,19 @@ public class JdbcProvider extends DatasourceProvider { return conn; } - private void addToPool(DatasourceRequest datasourceRequest) throws PropertyVetoException { - DruidDataSource dataSource = new DruidDataSource(); - JdbcConfiguration jdbcConfiguration = setCredential(datasourceRequest, dataSource); - dataSource.setInitialSize(jdbcConfiguration.getInitialPoolSize());// 初始连接数 - dataSource.setMinIdle(jdbcConfiguration.getMinPoolSize()); // 最小连接数 - dataSource.setMaxActive(jdbcConfiguration.getMaxPoolSize()); // 最大连接数 - jdbcConnection.put(datasourceRequest.getDatasource().getId(), dataSource); + private void addToPool(DatasourceRequest datasourceRequest) throws PropertyVetoException, SQLException { + DruidDataSource druidDataSource = new DruidDataSource(); + JdbcConfiguration jdbcConfiguration = setCredential(datasourceRequest, druidDataSource); + druidDataSource.setInitialSize(jdbcConfiguration.getInitialPoolSize());// 初始连接数 + druidDataSource.setMinIdle(jdbcConfiguration.getMinPoolSize()); // 最小连接数 + druidDataSource.setMaxActive(jdbcConfiguration.getMaxPoolSize()); // 最大连接数 + if(datasourceRequest.getDatasource().getType().equals(DatasourceTypes.mongo.name())){ + WallFilter wallFilter = new WallFilter(); + wallFilter.setDbType(DatasourceTypes.mysql.name()); + druidDataSource.setProxyFilters(Arrays.asList(new Filter[]{wallFilter})); + } + druidDataSource.init(); + jdbcConnection.put(datasourceRequest.getDatasource().getId(), druidDataSource); } @@ -351,47 +374,51 @@ public class JdbcProvider extends DatasourceProvider { case de_doris: case ds_doris: MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MysqlConfiguration.class); - dataSource.setUsername(mysqlConfiguration.getUsername()); - dataSource.setDriverClassLoader(extendedJdbcClassLoader); - dataSource.setPassword(mysqlConfiguration.getPassword()); dataSource.setUrl(mysqlConfiguration.getJdbc()); + dataSource.setDriverClassName(mysqlConfiguration.getDriver()); + dataSource.setValidationQuery("select 1"); jdbcConfiguration = mysqlConfiguration; break; case sqlServer: SqlServerConfiguration sqlServerConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfiguration.class); - dataSource.setUsername(sqlServerConfiguration.getUsername()); - dataSource.setDriverClassLoader(extendedJdbcClassLoader); - dataSource.setPassword(sqlServerConfiguration.getPassword()); + dataSource.setDriverClassName(sqlServerConfiguration.getDriver()); dataSource.setUrl(sqlServerConfiguration.getJdbc()); + dataSource.setValidationQuery("select 1"); jdbcConfiguration = sqlServerConfiguration; break; case oracle: OracleConfiguration oracleConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), OracleConfiguration.class); - dataSource.setUsername(oracleConfiguration.getUsername()); - dataSource.setDriverClassLoader(extendedJdbcClassLoader); - dataSource.setPassword(oracleConfiguration.getPassword()); + dataSource.setDriverClassName(oracleConfiguration.getDriver()); dataSource.setUrl(oracleConfiguration.getJdbc()); + dataSource.setValidationQuery("select 1 from dual"); jdbcConfiguration = oracleConfiguration; break; case pg: PgConfiguration pgConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfiguration.class); - dataSource.setUsername(pgConfiguration.getUsername()); - dataSource.setDriverClassLoader(extendedJdbcClassLoader); - dataSource.setPassword(pgConfiguration.getPassword()); + dataSource.setDriverClassName(pgConfiguration.getDriver()); dataSource.setUrl(pgConfiguration.getJdbc()); jdbcConfiguration = pgConfiguration; break; case ck: CHConfiguration chConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), CHConfiguration.class); - dataSource.setUsername(chConfiguration.getUsername()); - dataSource.setDriverClassLoader(extendedJdbcClassLoader); - dataSource.setPassword(chConfiguration.getPassword()); + dataSource.setDriverClassName(chConfiguration.getDriver()); dataSource.setUrl(chConfiguration.getJdbc()); jdbcConfiguration = chConfiguration; break; + case mongo: + MongodbConfiguration mongodbConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MongodbConfiguration.class); + dataSource.setDriverClassName(mongodbConfiguration.getDriver()); + dataSource.setUrl(mongodbConfiguration.getJdbc()); + jdbcConfiguration = mongodbConfiguration; + break; default: break; } + + dataSource.setUsername(jdbcConfiguration.getUsername()); + dataSource.setDriverClassLoader(extendedJdbcClassLoader); + dataSource.setPassword(jdbcConfiguration.getPassword()); + return jdbcConfiguration; } @@ -479,4 +506,23 @@ public class JdbcProvider extends DatasourceProvider { } } + private static String rebuildSqlWithFragment(String sql) { + if (!sql.toLowerCase().startsWith("with")) { + Matcher matcher = WITH_SQL_FRAGMENT.matcher(sql); + if (matcher.find()) { + String withFragment = matcher.group(); + if (!com.alibaba.druid.util.StringUtils.isEmpty(withFragment)) { + if (withFragment.length() > 6) { + int lastSelectIndex = withFragment.length() - 6; + sql = sql.replace(withFragment, withFragment.substring(lastSelectIndex)); + withFragment = withFragment.substring(0, lastSelectIndex); + } + sql = withFragment + " " + sql; + sql = sql.replaceAll(" " + "{2,}", " "); + } + } + } + return sql; + } + } diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index a36b050f45..9081c0d09a 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -48,6 +48,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("esQuery", QueryProvider.class); case ck: return context.getBean("ckQuery", QueryProvider.class); + case mongo: + return context.getBean("mongoQuery", QueryProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } diff --git a/backend/src/main/java/io/dataease/provider/mongodb/MongoConstants.java b/backend/src/main/java/io/dataease/provider/mongodb/MongoConstants.java new file mode 100644 index 0000000000..ecb326b191 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/mongodb/MongoConstants.java @@ -0,0 +1,44 @@ +package io.dataease.provider.mongodb; + +import io.dataease.provider.SQLConstants; + +import static io.dataease.datasource.constants.DatasourceTypes.mongo; +import static io.dataease.datasource.constants.DatasourceTypes.oracle; + +/** + * @Author gin + * @Date 2021/7/8 7:22 下午 + */ +public class MongoConstants extends SQLConstants { + public static final String KEYWORD_TABLE = "%s"; + + public static final String KEYWORD_FIX = "%s." + mongo.getKeywordPrefix() + "%s" + mongo.getKeywordSuffix(); + + public static final String ALIAS_FIX = mongo.getAliasPrefix() + "%s" + oracle.getAliasSuffix(); + + public static final String toInt32 = "toInt32(%s)"; + + public static final String toDateTime = "toDateTime(%s)"; + + public static final String toInt64 = "toInt64(%s)"; + + public static final String toFloat64 = "toFloat64(%s)"; + + public static final String formatDateTime = "formatDateTime(%s,'%s')"; + + public static final String toDecimal = "toDecimal64(%s,2)"; + + public static final String DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"; + + public static final String WHERE_VALUE_NULL = "(NULL,'')"; + + public static final String WHERE_VALUE_VALUE = "'%s'"; + + public static final String AGG_COUNT = "COUNT(*)"; + + public static final String AGG_FIELD = "%s(%s)"; + + public static final String WHERE_BETWEEN = "'%s' AND '%s'"; + + public static final String BRACKETS = "(%s)"; +} diff --git a/backend/src/main/java/io/dataease/provider/mongodb/MongoQueryProvider.java b/backend/src/main/java/io/dataease/provider/mongodb/MongoQueryProvider.java new file mode 100644 index 0000000000..4a8430627c --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/mongodb/MongoQueryProvider.java @@ -0,0 +1,995 @@ +package io.dataease.provider.mongodb; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.commons.constants.DeTypeConstants; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; +import io.dataease.provider.oracle.OracleConstants; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; + +import javax.annotation.Resource; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + +/** + * @Author gin + * @Date 2021/5/17 2:43 下午 + */ +@Service("mongoQuery") +public class MongoQueryProvider extends QueryProvider { + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Override + public Integer transFieldType(String field) { + System.out.println(field); + field = field.toUpperCase(); + switch (field) { + case "CHAR": + case "VARCHAR": + case "TEXT": + case "TINYTEXT": + case "MEDIUMTEXT": + case "LONGTEXT": + case "STRING": + return 0;// 文本 + case "DATE": + case "TIME": + case "YEAR": + case "DATETIME": + case "TIMESTAMP": + return 1;// 时间 + case "INT": + case "SMALLINT": + case "MEDIUMINT": + case "INTEGER": + case "BIGINT": + return 2;// 整型 + case "FLOAT": + case "DOUBLE": + case "DECIMAL": + return 3;// 浮点 + case "BIT": + case "TINYINT": + return 4;// 布尔 + default: + return 0; + } + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 0,1000"; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField; + if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(f.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + fieldName = originField; + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); + } + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", isGroup); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + return st_sql.render(); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup) { + return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null); + } + + @Override + public String createQueryTableWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null) + " LIMIT 0," + limit; + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT 0," + limit; + } + + @Override + public String createQuerySQLWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT " + (page - 1) * pageSize + "," + realSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(MongoConstants.ALIAS_FIX, String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i)); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MongoConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLTableInfo(String table, List xAxis, List customFilter, List extFilterRequestList, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", false); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("previewSql"); + st.add("isGroup", false); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MongoConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpTableInfo(String sql, List xAxis, List customFilter, List extFilterRequestList, Datasource ds) { + return getSQLTableInfo("(" + sqlFix(sql) + ")", xAxis, customFilter, extFilterRequestList, null); + } + + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + List xList = new ArrayList<>(); + xList.addAll(xAxis); + xList.addAll(extStack); + if (CollectionUtils.isNotEmpty(xList)) { + for (int i = 0; i < xList.size(); i++) { + ChartViewFieldDTO x = xList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(MongoConstants.ALIAS_FIX, String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i)); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MongoConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + List yList = new ArrayList<>(); + yList.addAll(yAxis); + yList.addAll(extBubble); + if (CollectionUtils.isNotEmpty(yList)) { + for (int i = 0; i < yList.size(); i++) { + ChartViewFieldDTO y = yList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(MongoConstants.ALIAS_FIX, String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i)); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MongoConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + @Override + public String getSQLSummary(String table, List yAxis, List customFilter, List extFilterRequestList) { + // 字段汇总 排序等 + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MongoConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(MongoConstants.ALIAS_FIX, String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i)); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(MongoConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList) { + return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList); + } + + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0"; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + if (f.getDeExtractType() == 4) { // 处理 tinyint + stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName()); + } else { + stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); + } + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + } + + @Override + public String convertTableToSql(String tableName, Datasource ds) { + return createSQLPreview("SELECT * FROM " + String.format(MongoConstants.KEYWORD_TABLE, tableName), null); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IS NULL "; + case "not_null": + return " IS NOT NULL "; + case "empty": + return " = "; + case "not_empty": + return " <> "; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public List transCustomFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartCustomFilterDTO request : requestList) { + DatasetTableField field = request.getField(); + if (ObjectUtils.isEmpty(field)) { + continue; + } + String value = request.getValue(); + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getTerm()); + String whereValue = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == DeTypeConstants.DE_TIME) { + originName = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } +// if (field.getDeType() == DeTypeConstants.DE_TIME) { +// if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { +// whereName = String.format(MongoConstants.STR_TO_DATE, originName, MongoConstants.DEFAULT_DATE_FORMAT); +// } +// if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) { +// String cast = String.format(MongoConstants.CAST, originName, MongoConstants.DEFAULT_INT_FORMAT) + "/1000"; +// whereName = String.format(MongoConstants.FROM_UNIXTIME, cast, MongoConstants.DEFAULT_DATE_FORMAT); +// } +// if (field.getDeExtractType() == DeTypeConstants.DE_TIME) { +// whereName = originName; +// } +// } else { +// whereName = originName; +// } + whereName = originName; + if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) { +// whereValue = MongoConstants.WHERE_VALUE_NULL; + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) { +// whereTerm = String.format(whereTerm, originName); + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + whereValue = String.format(MongoConstants.WHERE_VALUE_VALUE, value); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + public List transExtFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == DeTypeConstants.DE_TIME) { + originName = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + +// if (field.getDeType() == DeTypeConstants.DE_TIME) { +// if (field.getDeExtractType() == DeTypeConstants.DE_STRING || field.getDeExtractType() == 5) { +// whereName = String.format(MongoConstants.STR_TO_DATE, originName, MongoConstants.DEFAULT_DATE_FORMAT); +// } +// if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == 4) { +// String cast = String.format(MongoConstants.CAST, originName, MongoConstants.DEFAULT_INT_FORMAT) + "/1000"; +// whereName = String.format(MongoConstants.FROM_UNIXTIME, cast, MongoConstants.DEFAULT_DATE_FORMAT); +// } +// if (field.getDeExtractType() == DeTypeConstants.DE_TIME) { +// whereName = originName; +// } +// } else { +// whereName = originName; +// } + whereName = originName; + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == DeTypeConstants.DE_TIME) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + whereValue = String.format(MongoConstants.WHERE_BETWEEN, startTime, endTime); + } else { + whereValue = String.format(MongoConstants.WHERE_BETWEEN, value.get(0), value.get(1)); + } + } else { + whereValue = String.format(MongoConstants.WHERE_VALUE_VALUE, value.get(0)); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } else { + split = "-"; + } + + if (StringUtils.isEmpty(dateStyle)) { + return "%Y-%m-%d %H:%i:%S"; + } + + switch (dateStyle) { + case "y": + return "%Y"; + case "y_M": + return "%Y" + split + "%m"; + case "y_M_d": + return "%Y" + split + "%m" + split + "%d"; + case "H_m_s": + return "%H:%i:%S"; + case "y_M_d_H_m": + return "%Y" + split + "%m" + split + "%d" + " %H:%i"; + case "y_M_d_H_m_s": + return "%Y" + split + "%m" + split + "%d" + " %H:%i:%S"; + default: + return "%Y-%m-%d %H:%i:%S"; + } + } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { +// if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) { +// fieldName = String.format(MongoConstants.UNIX_TIMESTAMP, originField) + "*1000"; +// } else if (x.getDeType() == DeTypeConstants.DE_TIME) { +// String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); +// fieldName = String.format(MongoConstants.DATE_FORMAT, originField, format); +// } else { +// fieldName = originField; +// } + fieldName = originField; + } else { +// if (x.getDeType() == DeTypeConstants.DE_TIME) { +// String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); +// if (x.getDeExtractType() == DeTypeConstants.DE_STRING) { +// fieldName = String.format(MongoConstants.DATE_FORMAT, originField, format); +// } else { +// String cast = String.format(MongoConstants.CAST, originField, MongoConstants.DEFAULT_INT_FORMAT) + "/1000"; +// String from_unixtime = String.format(MongoConstants.FROM_UNIXTIME, cast, MongoConstants.DEFAULT_DATE_FORMAT); +// fieldName = String.format(MongoConstants.DATE_FORMAT, from_unixtime, format); +// } +// } else { +// fieldName = originField; +// } + fieldName = originField; + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + fieldName = MongoConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(MongoConstants.AGG_FIELD, y.getSummary(), originField); + } else { +// if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { +// String cast = String.format(MongoConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? MongoConstants.DEFAULT_INT_FORMAT : MongoConstants.DEFAULT_FLOAT_FORMAT); +// String agg = String.format(MongoConstants.AGG_FIELD, y.getSummary(), cast); +// fieldName = String.format(MongoConstants.CAST, agg, MongoConstants.DEFAULT_FLOAT_FORMAT); +// } else { +// String cast = String.format(MongoConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? MongoConstants.DEFAULT_INT_FORMAT : MongoConstants.DEFAULT_FLOAT_FORMAT); +// fieldName = String.format(MongoConstants.AGG_FIELD, y.getSummary(), cast); +// } + fieldName = String.format(MongoConstants.AGG_FIELD, y.getSummary(), originField); + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { +// whereValue = MongoConstants.WHERE_VALUE_NULL; + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { +// whereTerm = String.format(whereTerm, originField); + whereValue = ""; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "empty")) { + whereValue = "''"; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_empty")) { + whereValue = "''"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(MongoConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private String calcFieldRegex(String originField, SQLObj tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(MongoConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } + return originField; + } +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index af288d0ae7..6b417d51f8 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1037,6 +1037,7 @@ public class DataSetTableService { if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { TableFiled filed = fields.get(i); + System.out.println(new Gson().toJson(filed)); DatasetTableField datasetTableField = DatasetTableField.builder().build(); datasetTableField.setTableId(datasetTable.getId()); datasetTableField.setOriginName(filed.getFieldName()); diff --git a/backend/src/main/java/io/dataease/service/message/SysMsgService.java b/backend/src/main/java/io/dataease/service/message/SysMsgService.java index 5e587b1fb9..8766d60b57 100644 --- a/backend/src/main/java/io/dataease/service/message/SysMsgService.java +++ b/backend/src/main/java/io/dataease/service/message/SysMsgService.java @@ -16,14 +16,16 @@ import io.dataease.controller.sys.request.MsgSettingRequest; import io.dataease.controller.sys.response.MsgGridDto; import io.dataease.controller.sys.response.SettingTreeNode; import io.dataease.controller.sys.response.SubscribeNode; +import io.dataease.service.system.SystemParameterService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; - import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -31,7 +33,7 @@ import java.util.stream.Collectors; @Service public class SysMsgService { - private static final long overDays = 30; + private static int overDays = 30; @Resource private SysMsgMapper sysMsgMapper; @@ -50,7 +52,10 @@ public class SysMsgService { @Resource private SysMsgSettingMapper sysMsgSettingMapper; - public List query(Long userId, MsgRequest msgRequest) { + @Autowired + private SystemParameterService systemParameterService; + + /* public List query(Long userId, MsgRequest msgRequest) { String orderClause = " create_time desc"; SysMsgExample example = new SysMsgExample(); SysMsgExample.Criteria criteria = example.createCriteria(); @@ -76,8 +81,8 @@ public class SysMsgService { List sysMsgs = sysMsgMapper.selectByExample(example); return sysMsgs; } - - public List queryGrid(Long userId, MsgRequest msgRequest, List typeIds) { + */ + public List queryGrid(Long userId, MsgRequest msgRequest, List typeIds, Long startTime) { String orderClause = " create_time desc"; SysMsgExample example = new SysMsgExample(); SysMsgExample.Criteria criteria = example.createCriteria(); @@ -105,7 +110,8 @@ public class SysMsgService { criteria.andStatusEqualTo(msgRequest.getStatus()); } - criteria.andCreateTimeGreaterThanOrEqualTo(overTime()); + criteria.andCreateTimeGreaterThanOrEqualTo(startTime); + /* criteria.andCreateTimeGreaterThanOrEqualTo(overTime()); */ example.setOrderByClause(orderClause); List msgGridDtos = extSysMsgMapper.queryGrid(example); @@ -330,6 +336,10 @@ public class SysMsgService { public Long overTime() { + String msgTimeOut = systemParameterService.basicInfo().getMsgTimeOut(); + if(StringUtils.isNotBlank(msgTimeOut)) { + overDays = Integer.parseInt(msgTimeOut); + } Long currentTime = System.currentTimeMillis(); long oneDayTime = 24 * 60 * 60 * 1000; @@ -340,11 +350,6 @@ public class SysMsgService { } - /* public static void main(String[] args) { - - Long overTime = overTime(); - System.out.println(overTime); - - } */ + } diff --git a/backend/src/main/java/io/dataease/service/system/SystemParameterService.java b/backend/src/main/java/io/dataease/service/system/SystemParameterService.java index aa4c0c41ca..0e3c77a4bc 100644 --- a/backend/src/main/java/io/dataease/service/system/SystemParameterService.java +++ b/backend/src/main/java/io/dataease/service/system/SystemParameterService.java @@ -1,6 +1,5 @@ package io.dataease.service.system; -import com.alibaba.fastjson.JSON; import io.dataease.base.domain.FileMetadata; import io.dataease.base.domain.SystemParameter; import io.dataease.base.domain.SystemParameterExample; @@ -11,6 +10,8 @@ import io.dataease.commons.exception.DEException; import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.EncryptUtils; import io.dataease.commons.utils.LogUtil; +import io.dataease.controller.sys.response.BasicInfo; +import io.dataease.controller.sys.response.MailInfo; import io.dataease.dto.SystemParameterDTO; import io.dataease.i18n.Translator; import io.dataease.service.FileService; @@ -47,6 +48,52 @@ public class SystemParameterService { return extSystemParameterMapper.email(); } + public BasicInfo basicInfo() { + List paramList = this.getParamList("basic"); + BasicInfo result = new BasicInfo(); + if (!CollectionUtils.isEmpty(paramList)) { + for (SystemParameter param : paramList) { + if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.FRONT_TIME_OUT.getValue())) { + /* result.setFrontTimeOut(StringUtils.isBlank(param.getParamValue()) ? 0 : Integer.parseInt(param.getParamValue())); */ + result.setFrontTimeOut(param.getParamValue()); + } + if (StringUtils.equals(param.getParamKey(), ParamConstants.BASIC.MSG_TIME_OUT.getValue())) { + /* result.setMsgTimeOut(StringUtils.isBlank(param.getParamValue()) ? 0 : Integer.parseInt(param.getParamValue())); */ + result.setMsgTimeOut(param.getParamValue()); + } + } + } + return result; + } + + + public MailInfo mailInfo(String type) { + List paramList = this.getParamList(type); + MailInfo mailInfo = new MailInfo(); + if (!CollectionUtils.isEmpty(paramList)) { + for (SystemParameter param : paramList) { + if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.SERVER.getValue())) { + mailInfo.setHost(param.getParamValue()); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.PORT.getValue())) { + mailInfo.setPort(param.getParamValue()); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.ACCOUNT.getValue())) { + mailInfo.setAccount(param.getParamValue()); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.PASSWORD.getValue())) { + String password = EncryptUtils.aesDecrypt(param.getParamValue()).toString(); + mailInfo.setPassword(password); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.SSL.getValue())) { + mailInfo.setSsl(param.getParamValue()); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.TLS.getValue())) { + mailInfo.setTls(param.getParamValue()); + } else if (StringUtils.equals(param.getParamKey(), ParamConstants.MAIL.RECIPIENTS.getValue())) { + mailInfo.setRecipient(param.getParamValue()); + } + } + } + return mailInfo; + } + + public String getSystemLanguage() { String result = StringUtils.EMPTY; SystemParameterExample example = new SystemParameterExample(); @@ -61,6 +108,8 @@ public class SystemParameterService { return result; } + + public void editMail(List parameters) { List paramList = this.getParamList(ParamConstants.Classify.MAIL.getValue()); boolean empty = paramList.size() <= 0; @@ -84,6 +133,21 @@ public class SystemParameterService { }); } + public void editBasic(List parameters) { + parameters.forEach(parameter -> { + SystemParameterExample example = new SystemParameterExample(); + + example.createCriteria().andParamKeyEqualTo(parameter.getParamKey()); + if (systemParameterMapper.countByExample(example) > 0) { + systemParameterMapper.updateByPrimaryKey(parameter); + } else { + systemParameterMapper.insert(parameter); + } + example.clear(); + + }); + } + public List getParamList(String type) { SystemParameterExample example = new SystemParameterExample(); example.createCriteria().andParamKeyLike(type + "%"); @@ -120,7 +184,7 @@ public class SystemParameterService { try { helper = new MimeMessageHelper(mimeMessage, true); helper.setFrom(javaMailSender.getUsername()); - helper.setSubject("MeterSphere测试邮件 " ); + helper.setSubject("DataEase测试邮件 " ); helper.setText("这是一封测试邮件，邮件发送成功", true); helper.setTo(recipients); javaMailSender.send(mimeMessage); diff --git a/backend/src/main/resources/db/migration/V27__de1.4.sql b/backend/src/main/resources/db/migration/V27__de1.4.sql index 30afc73245..5bb6867aab 100644 --- a/backend/src/main/resources/db/migration/V27__de1.4.sql +++ b/backend/src/main/resources/db/migration/V27__de1.4.sql @@ -35,3 +35,7 @@ CREATE TABLE `panel_link_jump_target_view_info` ( `target_field_id` varchar(50) DEFAULT NULL, PRIMARY KEY (`target_id`) USING BTREE ) ENGINE=InnoDB ; + +BEGIN; +INSERT INTO `sys_menu` VALUES (6, 1, 0, 1, '系统参数', 'system-param', 'system/SysParam/index', 6, 'sys-tools', 'system-param', b'0', b'0', b'0', 'sysparam:read', NULL, NULL, NULL, NULL); +COMMIT; diff --git a/drivers/mongodb_unityjdbc_full.jar b/drivers/mongodb_unityjdbc_full.jar new file mode 100644 index 0000000000..bb61b2840a Binary files /dev/null and b/drivers/mongodb_unityjdbc_full.jar differ diff --git a/frontend/pom.xml b/frontend/pom.xml index 604c6b2027..a0d56dfe10 100644 --- a/frontend/pom.xml +++ b/frontend/pom.xml @@ -6,7 +6,7 @@ dataease-server io.dataease - 1.3.0 + 1.4.0 4.0.0 diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 9aaba1031a..66a91a15c0 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -5,11 +5,12 @@ diff --git a/frontend/src/api/system/basic.js b/frontend/src/api/system/basic.js new file mode 100644 index 0000000000..21d30b7bc6 --- /dev/null +++ b/frontend/src/api/system/basic.js @@ -0,0 +1,19 @@ +import request from '@/utils/request' + +export function basicInfo() { + return request({ + url: '/system/basic/info', + method: 'get', + loading: true + }) +} + +export function updateInfo(data) { + return request({ + url: '/system/edit/basic', + method: 'post', + loading: true, + data + + }) +} diff --git a/frontend/src/api/system/email.js b/frontend/src/api/system/email.js new file mode 100644 index 0000000000..c5a7e6af41 --- /dev/null +++ b/frontend/src/api/system/email.js @@ -0,0 +1,30 @@ +import request from '@/utils/request' + +export function validate(data) { + return request({ + url: '/system/testConnection', + method: 'post', + timeout: 30000, + loading: true, + data + + }) +} + +export function emailInfo() { + return request({ + url: '/system/mail/info', + method: 'get', + loading: true + }) +} + +export function updateInfo(data) { + return request({ + url: '/system/edit/email', + method: 'post', + loading: true, + data + + }) +} diff --git a/frontend/src/api/user.js b/frontend/src/api/user.js index e9ddcd6ab0..01850eecc5 100644 --- a/frontend/src/api/user.js +++ b/frontend/src/api/user.js @@ -64,3 +64,17 @@ export function oidcStatus() { method: 'post' }) } + +export function pluginLoaded() { + return request({ + url: '/api/auth/isPluginLoaded', + method: 'post' + }) +} + +export function getPublicKey() { + return request({ + url: '/api/auth/getPublicKey', + method: 'get' + }) +} diff --git a/frontend/src/components/canvas/components/TextAttr.vue b/frontend/src/components/canvas/components/TextAttr.vue index 0d90b84d2a..ac270ee15d 100644 --- a/frontend/src/components/canvas/components/TextAttr.vue +++ b/frontend/src/components/canvas/components/TextAttr.vue @@ -70,6 +70,22 @@ + + + + +
+ +
+ + + + + +
+ +
+ @@ -121,7 +137,7 @@ -
+
@@ -242,7 +258,9 @@ export default { 'borderRadius', 'color', 'backgroundColor', - 'date-format' + 'date-format', + 'time_margin' + /* 'margin' */ ], // 文本组件显示的属性 'v-text': [ diff --git a/frontend/src/components/canvas/custom-component/UserViewDialog.vue b/frontend/src/components/canvas/custom-component/UserViewDialog.vue index 141010bd88..1a077dc2a0 100644 --- a/frontend/src/components/canvas/custom-component/UserViewDialog.vue +++ b/frontend/src/components/canvas/custom-component/UserViewDialog.vue @@ -1,7 +1,8 @@