diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 0ca60d8dd0..fd4f05edfb 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -6,7 +6,6 @@ import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.DatasourceMapper; import io.dataease.base.mapper.ext.ExtChartViewMapper; -import io.dataease.base.mapper.ext.UtilMapper; import io.dataease.commons.constants.*; import io.dataease.commons.model.AuthURD; import io.dataease.commons.utils.*; @@ -16,7 +15,6 @@ import io.dataease.datasource.provider.DatasourceProvider; import io.dataease.datasource.provider.JdbcProvider; import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; -import io.dataease.datasource.service.DatasourceService; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.exception.DataEaseException; import io.dataease.listener.util.CacheUtils; @@ -24,8 +22,8 @@ import io.dataease.provider.QueryProvider; import io.dataease.service.message.DeMsgutil; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; @@ -92,12 +90,9 @@ public class ExtractDataService { private DatasetTableMapper datasetTableMapper; @Resource private DatasetTableTaskMapper datasetTableTaskMapper; - @Resource - private DatasourceService datasourceService; + @Resource private ExtChartViewMapper extChartViewMapper; - @Resource - private UtilMapper utilMapper; private static String lastUpdateTime = "${__last_update_time__}"; private static String currentUpdateTime = "${__current_update_time__}"; @@ -133,26 +128,25 @@ public class ExtractDataService { "fi\n" + "rm -rf %s\n"; - public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask){ + public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask) { datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusNotEqualTo(JobStatus.Underway.name()); example.or(example.createCriteria().andIdEqualTo(datasetTable.getId()).andSyncStatusIsNull()); Boolean existSyncTask = datasetTableMapper.updateByExampleSelective(datasetTable, example) == 0; - if(existSyncTask){ + if (existSyncTask) { DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTaskId(datasetTableTask.getId()); datasetTableTaskLog.setTableId(datasetTable.getId()); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if(CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())){ + if (CollectionUtils.isNotEmpty(datasetTableTaskLogs) && datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name())) { return false; } return true; - }else { + } else { datasetTableTask.setLastExecTime(System.currentTimeMillis()); datasetTableTask.setLastExecStatus(JobStatus.Underway.name()); - datasetTableTask.setStatus(TaskStatus.Exec.name()); dataSetTableTaskService.update(datasetTableTask); return false; } @@ -195,12 +189,12 @@ public class ExtractDataService { extractData(datasetTable, "all_scope"); replaceTable(DorisTableUtils.dorisName(datasetTableId)); saveSucessLog(datasetTableTaskLog); -// sendWebMsg(datasetTable, null, true); + sendWebMsg(datasetTable, null, true); deleteFile("all_scope", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } catch (Exception e) { saveErrorLog(datasetTableId, null, e); -// sendWebMsg(datasetTable, null, false); + sendWebMsg(datasetTable, null, false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); deleteFile("all_scope", datasetTableId); @@ -230,7 +224,7 @@ public class ExtractDataService { } //侵入式清除下属视图缓存 List viewIds = extChartViewMapper.allViewIds(datasetTableId); - if (CollectionUtils.isNotEmpty(viewIds)){ + if (CollectionUtils.isNotEmpty(viewIds)) { viewIds.forEach(viewId -> { CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId); }); @@ -239,26 +233,26 @@ public class ExtractDataService { public void extractData(String datasetTableId, String taskId, String type, JobExecutionContext context) { DatasetTable datasetTable = getDatasetTable(datasetTableId); - if(datasetTable == null){ + if (datasetTable == null) { LogUtil.error("Can not find DatasetTable: " + datasetTableId); return; } DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(taskId); - if(datasetTableTask == null){ + if (datasetTableTask == null) { return; } - if(datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name())|| datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Pending.name())){ + if (datasetTableTask.getStatus().equalsIgnoreCase(TaskStatus.Stopped.name())) { LogUtil.info("Skip synchronization task, task ID : " + datasetTableTask.getId()); return; } - if(existSyncTask(datasetTable, datasetTableTask)){ + if (existSyncTask(datasetTable, datasetTableTask)) { LogUtil.info("Skip synchronization task for dataset, dataset ID : " + datasetTableId); return; } DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); UpdateType updateType = UpdateType.valueOf(type); - if(context != null){ + if (context != null) { datasetTable.setQrtzInstance(context.getFireInstanceId()); datasetTableMapper.updateByPrimaryKeySelective(datasetTable); } @@ -282,19 +276,19 @@ public class ExtractDataService { switch (updateType) { case all_scope: // 全量更新 - try{ - if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { + try { + if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } - if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { + if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql); generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); - if(datasetTable.getType().equalsIgnoreCase("sql")){ + if (datasetTable.getType().equalsIgnoreCase("sql")) { generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource)); - }else { + } else { generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); } Long execTime = System.currentTimeMillis(); @@ -302,24 +296,24 @@ public class ExtractDataService { replaceTable(DorisTableUtils.dorisName(datasetTableId)); saveSucessLog(datasetTableTaskLog); - sendWebMsg(datasetTable, taskId,true); + sendWebMsg(datasetTable, datasetTableTask, true); deleteFile("all_scope", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); + datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); + dataSetTableTaskService.update(datasetTableTask); - dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Completed); - - }catch (Exception e){ + } catch (Exception e) { saveErrorLog(datasetTableId, taskId, e); + datasetTableTask.setLastExecStatus(JobStatus.Error.name()); + dataSetTableTaskService.update(datasetTableTask); - dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Error); - - sendWebMsg(datasetTable, taskId,false); + sendWebMsg(datasetTable, datasetTableTask, false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); deleteFile("all_scope", datasetTableId); - }finally { + } finally { } break; @@ -336,10 +330,10 @@ public class ExtractDataService { return; } - if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { + if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } - if(datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { + if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); } Long execTime = System.currentTimeMillis(); @@ -360,34 +354,29 @@ public class ExtractDataService { } saveSucessLog(datasetTableTaskLog); - sendWebMsg(datasetTable, taskId,true); + sendWebMsg(datasetTable, datasetTableTask, true); deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); - - dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Completed); - }catch (Exception e){ + datasetTableTask.setLastExecStatus(JobStatus.Completed.name()); + dataSetTableTaskService.update(datasetTableTask); + } catch (Exception e) { saveErrorLog(datasetTableId, taskId, e); - sendWebMsg(datasetTable, taskId,false); + sendWebMsg(datasetTable, datasetTableTask, false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); - - dataSetTableTaskService.updateTaskStatus(datasetTableTask, JobStatus.Error); - + datasetTableTask.setLastExecStatus(JobStatus.Error.name()); + dataSetTableTaskService.update(datasetTableTask); deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); - }finally { - if(datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())){ - datasetTableTask.setStatus(TaskStatus.Stopped.name()); - dataSetTableTaskService.update(datasetTableTask); - } + } finally { } break; } //侵入式清除下属视图缓存 List viewIds = extChartViewMapper.allViewIds(datasetTableId); - if (CollectionUtils.isNotEmpty(viewIds)){ + if (CollectionUtils.isNotEmpty(viewIds)) { viewIds.forEach(viewId -> { CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId); }); @@ -395,7 +384,8 @@ public class ExtractDataService { } - private void sendWebMsg(DatasetTable datasetTable, String taskId, Boolean status) { + private void sendWebMsg(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Boolean status) { + String taskId = datasetTableTask.getId(); String msg = status ? "成功" : "失败"; Long typeId = status ? 5L : 6L; String id = datasetTable.getId(); @@ -403,18 +393,22 @@ public class ExtractDataService { Set userIds = AuthUtils.userIdsByURD(authURD); Gson gson = new Gson(); userIds.forEach(userId -> { - Map param = new HashMap<>(); + Map param = new HashMap<>(); param.put("tableId", id); - if(StringUtils.isNotEmpty(taskId)){ - param.put("taskId", taskId); - } - DeMsgutil.sendMsg(userId, typeId, 1L, "数据集【"+datasetTable.getName()+"】同步"+msg, gson.toJson(param)); + if (StringUtils.isNotEmpty(taskId)) { + param.put("taskId", taskId); + } + String content = "数据集【" + datasetTable.getName() + "】同步" + msg; + if (ObjectUtils.isNotEmpty(datasetTableTask) && ObjectUtils.isNotEmpty(datasetTableTask.getName())) { + content += " 任务名称【" + datasetTableTask.getName() + "】"; + } + DeMsgutil.sendMsg(userId, typeId, 1L, content, gson.toJson(param)); }); } private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) { datasetTable.setSyncStatus(completed.name()); - if(execTime != null){ + if (execTime != null) { datasetTable.setLastUpdateTime(execTime); } DatasetTableExample example = new DatasetTableExample(); @@ -428,16 +422,16 @@ public class ExtractDataService { dataSetTableTaskLogService.save(datasetTableTaskLog); } - private void saveErrorLog(String datasetTableId, String taskId, Exception e){ + private void saveErrorLog(String datasetTableId, String taskId, Exception e) { LogUtil.error("Extract data error: " + datasetTableId, e); DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - if(StringUtils.isNotEmpty(taskId)){ + if (StringUtils.isNotEmpty(taskId)) { datasetTableTaskLog.setTaskId(taskId); } - List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){ + List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); + if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { datasetTableTaskLog = datasetTableTaskLogs.get(0); datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setInfo(e.getMessage()); @@ -456,7 +450,7 @@ public class ExtractDataService { if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { Column_Fields = Column_Fields + "varchar(65533)" + ",`"; } else { - Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize()*3)) + ",`"; + Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize() * 3)) + ",`"; } break; case 1: @@ -499,7 +493,8 @@ public class ExtractDataService { datasourceRequest.setDatasource(dorisDatasource); datasourceRequest.setQuery(dropTableSql.replace("TABLE_NAME", dorisTableName)); jdbcProvider.exec(datasourceRequest); - }catch (Exception ignore){} + } catch (Exception ignore) { + } } private void replaceTable(String dorisTableName) throws Exception { @@ -512,14 +507,15 @@ public class ExtractDataService { jdbcProvider.exec(datasourceRequest); } - private DatasetTable getDatasetTable(String datasetTableId){ - for (int i=0;i<5;i++){ - DatasetTable datasetTable = dataSetTableService.get(datasetTableId); - if(datasetTable == null){ + private DatasetTable getDatasetTable(String datasetTableId) { + for (int i = 0; i < 5; i++) { + DatasetTable datasetTable = dataSetTableService.get(datasetTableId); + if (datasetTable == null) { try { Thread.sleep(1000); - }catch (Exception ignore){} - }else { + } catch (Exception ignore) { + } + } else { return datasetTable; } } @@ -532,11 +528,11 @@ public class ExtractDataService { datasetTableTaskLog.setStatus(JobStatus.Underway.name()); datasetTableTaskLog.setTriggerType(TriggerType.Cron.name()); List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if(CollectionUtils.isEmpty(datasetTableTaskLogs)){ + if (CollectionUtils.isEmpty(datasetTableTaskLogs)) { datasetTableTaskLog.setStartTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); return datasetTableTaskLog; - }else { + } else { return datasetTableTaskLogs.get(0); } } @@ -545,14 +541,15 @@ public class ExtractDataService { datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); - for (int i=0;i<5;i++){ + for (int i = 0; i < 5; i++) { List datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog); - if(CollectionUtils.isNotEmpty(datasetTableTaskLogs)){ + if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { return datasetTableTaskLogs.get(0); } try { Thread.sleep(1000); - }catch (Exception ignore){} + } catch (Exception ignore) { + } } datasetTableTaskLog.setStartTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); @@ -560,7 +557,6 @@ public class ExtractDataService { } private void extractData(DatasetTable datasetTable, String extractType) throws Exception { - datasourceService.validate(datasetTable.getDataSourceId()); KettleFileRepository repository = CommonBeanFactory.getBean(KettleFileRepository.class); RepositoryDirectoryInterface repositoryDirectoryInterface = repository.loadRepositoryDirectoryTree(); JobMeta jobMeta = null; @@ -706,7 +702,7 @@ public class ExtractDataService { datasourceRequest.setDatasource(ds); datasourceRequest.setQuery(qp.wrapSql(sql)); List dorisFileds = new ArrayList<>(); - datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed ->{ + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> { dorisFileds.add(DorisTableUtils.columnName(filed)); }); return String.join(",", dorisFileds); @@ -728,7 +724,7 @@ public class ExtractDataService { case mysql: MysqlConfigration mysqlConfigration = new Gson().fromJson(datasource.getConfiguration(), MysqlConfigration.class); dataMeta = new DatabaseMeta("db", "MYSQL", "Native", mysqlConfigration.getHost(), mysqlConfigration.getDataBase(), mysqlConfigration.getPort().toString(), mysqlConfigration.getUsername(), mysqlConfigration.getPassword()); - dataMeta.addExtraOption("MYSQL","characterEncoding", "UTF-8"); + dataMeta.addExtraOption("MYSQL", "characterEncoding", "UTF-8"); transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputStep = inputStep(transMeta, selectSQL); @@ -744,13 +740,13 @@ public class ExtractDataService { break; case oracle: OracleConfigration oracleConfigration = new Gson().fromJson(datasource.getConfiguration(), OracleConfigration.class); - if(oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")){ + if (oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")) { String database = "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ORACLE_HOSTNAME)(PORT = ORACLE_PORT))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = ORACLE_SERVICE_NAME )))".replace("ORACLE_HOSTNAME", oracleConfigration.getHost()).replace("ORACLE_PORT", oracleConfigration.getPort().toString()).replace("ORACLE_SERVICE_NAME", oracleConfigration.getDataBase()); dataMeta = new DatabaseMeta("db", "ORACLE", "Native", "", database, "-1", oracleConfigration.getUsername(), oracleConfigration.getPassword()); - }else { + } else { dataMeta = new DatabaseMeta("db", "ORACLE", "Native", oracleConfigration.getHost(), oracleConfigration.getDataBase(), oracleConfigration.getPort().toString(), oracleConfigration.getUsername(), oracleConfigration.getPassword()); } - transMeta.addDatabase(dataMeta); + transMeta.addDatabase(dataMeta); selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); inputStep = inputStep(transMeta, selectSQL); @@ -806,12 +802,12 @@ public class ExtractDataService { selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields); } - if(extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")){ + if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")) { selectSQL = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(); QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); } - if(!extractType.equalsIgnoreCase("all_scope")){ + if (!extractType.equalsIgnoreCase("all_scope")) { QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); selectSQL = qp.createRawQuerySQLAsTmp(selectSQL, datasetTableFields); } @@ -834,23 +830,25 @@ public class ExtractDataService { ExcelInputMeta excelInputMeta = new ExcelInputMeta(); if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI); - try{ + try { InputStream inputStream = new FileInputStream(filePath); XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream); XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0); excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } if (StringUtils.equalsIgnoreCase(suffix, "xls")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL); - try{ + try { InputStream inputStream = new FileInputStream(filePath); HSSFWorkbook workbook = new HSSFWorkbook(inputStream); HSSFSheet sheet0 = workbook.getSheetAt(0); excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - }catch (Exception e){e.printStackTrace();} + } catch (Exception e) { + e.printStackTrace(); + } } excelInputMeta.setPassword("Encrypted"); excelInputMeta.setFileName(new String[]{filePath}); @@ -860,10 +858,10 @@ public class ExtractDataService { for (int i = 0; i < datasetTableFields.size(); i++) { ExcelInputField field = new ExcelInputField(); field.setName(datasetTableFields.get(i).getOriginName()); - if(datasetTableFields.get(i).getDeExtractType() == 1){ + if (datasetTableFields.get(i).getDeExtractType() == 1) { field.setType("String"); field.setFormat("yyyy-MM-dd HH:mm:ss"); - }else { + } else { field.setType("String"); } fields[i] = field; @@ -904,14 +902,14 @@ public class ExtractDataService { tmp_code = tmp_code.replace("handleWraps", handleWraps); String Column_Fields = ""; - if(datasourceType.equals(DatasourceTypes.excel) || datasourceType.equals(DatasourceTypes.oracle)){ + if (datasourceType.equals(DatasourceTypes.excel) || datasourceType.equals(DatasourceTypes.oracle)) { Column_Fields = String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList())); - }else { + } else { Column_Fields = String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())); } - if(datasourceType.equals(DatasourceTypes.excel)){ + if (datasourceType.equals(DatasourceTypes.excel)) { tmp_code = tmp_code.replace("handleExcelIntColumn", handleExcelIntColumn).replace("Column_Fields", Column_Fields); - }else { + } else { tmp_code = tmp_code.replace("handleExcelIntColumn", "").replace("Column_Fields", Column_Fields); } UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code); @@ -926,7 +924,7 @@ public class ExtractDataService { return userDefinedJavaClassStep; } - public void deleteFile(String type, String dataSetTableId){ + public void deleteFile(String type, String dataSetTableId) { String transName = null; String jobName = null; String fileName = null; @@ -941,7 +939,7 @@ public class ExtractDataService { transName = "trans_add_" + DorisTableUtils.dorisName(dataSetTableId); jobName = "job_add_" + DorisTableUtils.dorisName(dataSetTableId); fileName = DorisTableUtils.dorisAddName(dataSetTableId); - break; + break; case "incremental_delete": transName = "trans_delete_" + DorisTableUtils.dorisName(dataSetTableId); jobName = "job_delete_" + DorisTableUtils.dorisName(dataSetTableId); @@ -950,28 +948,31 @@ public class ExtractDataService { default: break; } - try{ + try { File file = new File(root_path + fileName + "." + extention); FileUtils.forceDelete(file); - }catch (Exception e){} - try{ + } catch (Exception e) { + } + try { File file = new File(root_path + jobName + ".kjb"); FileUtils.forceDelete(file); - }catch (Exception e){} - try{ + } catch (Exception e) { + } + try { File file = new File(root_path + transName + ".ktr"); FileUtils.forceDelete(file); - }catch (Exception e){} + } catch (Exception e) { + } } public boolean isKettleRunning() { - try { - if (!InetAddress.getByName(carte).isReachable(1000)) { - return false; - } - }catch (Exception e){ - return false; - } + try { + if (!InetAddress.getByName(carte).isReachable(1000)) { + return false; + } + } catch (Exception e) { + return false; + } HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); diff --git a/backend/src/main/resources/db/migration/V10__web_msg.sql b/backend/src/main/resources/db/migration/V10__web_msg.sql index 139c31b8e8..21689b402f 100644 --- a/backend/src/main/resources/db/migration/V10__web_msg.sql +++ b/backend/src/main/resources/db/migration/V10__web_msg.sql @@ -56,9 +56,9 @@ BEGIN; INSERT INTO `sys_msg_type` VALUES (1, 0, 'i18n_msg_type_panel_share', 'panel', 'to-msg-share'); INSERT INTO `sys_msg_type` VALUES (2, 1, 'i18n_msg_type_panel_share', 'panel', 'to-msg-share'); INSERT INTO `sys_msg_type` VALUES (3, 1, 'i18n_msg_type_panel_share_cacnel', 'panel', 'to-msg-share'); -INSERT INTO `sys_msg_type` VALUES (4, 0, 'i18n_msg_type_dataset_sync', 'dataset', 'to-msg-dataset'); -INSERT INTO `sys_msg_type` VALUES (5, 4, 'i18n_msg_type_dataset_sync_success', 'dataset', 'to-msg-dataset'); -INSERT INTO `sys_msg_type` VALUES (6, 4, 'i18n_msg_type_dataset_sync_faild', 'dataset', 'to-msg-dataset'); +INSERT INTO `sys_msg_type` VALUES (4, 0, 'i18n_msg_type_dataset_sync', 'sys-task-dataset', 'to-msg-dataset'); +INSERT INTO `sys_msg_type` VALUES (5, 4, 'i18n_msg_type_dataset_sync_success', 'sys-task-dataset', 'to-msg-dataset'); +INSERT INTO `sys_msg_type` VALUES (6, 4, 'i18n_msg_type_dataset_sync_faild', 'sys-task-dataset', 'to-msg-dataset'); COMMIT; -- ----------------------------