diff --git a/.gitignore b/.gitignore index aa5f9b252..983d6454a 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,9 @@ dss-apps/dss-data-governance/dss-data-warehouse-dao/target dss-apps/dss-data-governance/dss-data-warehouse-service/target dss-apps/dss-data-governance/dss-data-warehouse-server/target +#dss-git +dss-git/dss-git-common/target +dss-git/dss-git-server/target # plugins plugins/azkaban/linkis-jobtype/target diff --git a/assembly/dss-package/pom.xml b/assembly/dss-package/pom.xml index 842307e24..7607ea840 100644 --- a/assembly/dss-package/pom.xml +++ b/assembly/dss-package/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/assembly/pom.xml b/assembly/pom.xml index 0e97116a4..5de3c2e08 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml pom diff --git a/conf/dss-framework-orchestrator-server.properties b/conf/dss-framework-orchestrator-server.properties index b70b14792..d6d0a1c4d 100644 --- a/conf/dss-framework-orchestrator-server.properties +++ b/conf/dss-framework-orchestrator-server.properties @@ -32,7 +32,6 @@ wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.server. wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.orchestrator.core.dao,com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.workspace.mapper,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.orchestrator.db.dao,com.webank.wedatasphere.dss.workflow.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.flow.execution.entrance.dao -wds.dss.server.scheduling.clear.cs.cron=0 0 3 * * ? wds.dss.publish.max.remain.version=3 diff --git a/db/dss_ddl.sql b/db/dss_ddl.sql index 1b52e2c60..551b4b59a 100644 --- a/db/dss_ddl.sql +++ b/db/dss_ddl.sql @@ -51,6 +51,7 @@ CREATE TABLE `dss_orchestrator_info` ( `orchestrator_level` varchar(32) DEFAULT NULL COMMENT '工作流级别', `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + `status` VARCHAR(64), PRIMARY KEY (`id`) USING BTREE, UNIQUE KEY `unique_idx_uuid` (`uuid`) ) ENGINE=InnoDB AUTO_INCREMENT=326 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; @@ -70,6 +71,7 @@ CREATE TABLE `dss_orchestrator_version_info` ( `content` varchar(255) DEFAULT NULL, `context_id` varchar(200) DEFAULT NULL COMMENT '上下文ID', `valid_flag` INT(1) DEFAULT '1' COMMENT '版本有效标示，0:无效；1：有效', + `commit_id` varchar(64), PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=422 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; @@ -116,6 +118,7 @@ CREATE TABLE `dss_project` ( `dev_process` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '开发流程，多个以英文逗号分隔，取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_develop_process)', `orchestrator_mode` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '编排模式，多个以英文逗号分隔，取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_arrangement_mode或下面一级)', `visible` tinyint(4) DEFAULT '1' COMMENT '0:已删除；1：未删除(默认)', + `associate_git` TINYINT DEFAULT '0' COMMENT '0:未接入git，1:已接入git', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=313 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; @@ -639,3 +642,33 @@ key `idx_limit_name` (`limit_name`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='dss用户限制表'; +DROP TABLE IF EXISTS `dss_workspace_associate_git`; +CREATE TABLE `dss_workspace_associate_git` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `git_user` varchar(64) DEFAULT NULL COMMENT 'git登录用户名', + `git_password` VARCHAR(255) DEFAULT NULL COMMENT 'git登录密码，用于跳转', + `git_token` varchar(255) COMMENT '用户配置的git token', + `git_url` varchar(255), + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `create_by` varchar(128) DEFAULT NULL, + `update_by` varchar(128) DEFAULT NULL, + `type` varchar(32) DEFAULT NULL, + `git_user_id` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='工作空间绑定的git信息'; + + +DROP TABLE IF EXISTS `dss_orchestrator_submit_job_info`; +CREATE TABLE `dss_orchestrator_submit_job_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `orchestrator_id` bigint(20) NOT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `instance_name` varchar(128) DEFAULT NULL COMMENT '提交任务的实例', + `status` varchar(128) DEFAULT NULL COMMENT '提交任务状态', + `error_msg` varchar(2048) DEFAULT NULL COMMENT '提交任务异常信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='dss_orchestrator_submit_job_info表'; + diff --git a/dss-appconn/appconns/dss-datachecker-appconn/pom.xml b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml index 51cf70eaa..94ac53ec0 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/pom.xml +++ b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -54,7 +54,7 @@ com.webank.wedatasphere.dss dss-origin-sso-integration-standard - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT org.apache.linkis diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java index 61ff244b5..e77938da5 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java @@ -48,7 +48,7 @@ public DataChecker(Properties p, DataCheckerExecutionAction action) { maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; //test over time // maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000; - queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); + queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "60000")); } diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java index e3bc2ae27..c2275aa1b 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java @@ -54,13 +54,6 @@ public class DataCheckerDao { private static final String SQL_SOURCE_TYPE_JOB_PARTITION = "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID JOIN PARTITIONS p ON p.TBL_ID = t.TBL_ID WHERE d.NAME=? AND t.TBL_NAME=? AND p.PART_NAME=?"; - private static final String SQL_SOURCE_TYPE_BDP = - "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? AND status = '1';"; - - private static final String SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION = - "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? " + - "AND (UNIX_TIMESTAMP() - UNIX_TIMESTAMP(STR_TO_DATE(modify_time, '%Y-%m-%d %H:%i:%s'))) <= ? AND status = '1';"; - private static final String SQL_DOPS_CHECK_TABLE = "SELECT * FROM dops_clean_task_list WHERE db_name = ? AND tb_name = ? AND part_name is null AND task_state NOT IN (10,13) order by order_id desc limit 1"; private static final String SQL_DOPS_CHECK_PARTITION = @@ -72,7 +65,6 @@ public class DataCheckerDao { private static final String MASK_SOURCE_TYPE = "maskdb"; private static DataSource jobDS; - private static DataSource bdpDS; private static DataSource dopsDS; private static volatile DataCheckerDao instance; @@ -96,13 +88,6 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe return false; } } - if (bdpDS == null) { - bdpDS = DataDruidFactory.getBDPInstance(props, log); - if (bdpDS == null) { - log.warn("Error getting job Druid DataSource instance"); - return false; - } - } boolean systemCheck = Boolean.valueOf(props.getProperty(DataChecker.QUALITIS_SWITCH)); if (systemCheck && dopsDS == null) { dopsDS = DataDruidFactory.getDopsInstance(props, log);//通过alibaba的druid数据库连接池获取JOB数据库连接 @@ -122,7 +107,7 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe } log.info("(DataChecker info) database table partition info : " + dataCheckerInfo); long waitTime = Long.valueOf(props.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; - int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); + int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "60000")); // String timeScape = props.getProperty(DataChecker.TIME_SCAPE, "NULL"); log.info("(DataChecker info) wait time : " + waitTime); log.info("(DataChecker info) query frequency : " + queryFrequency); @@ -134,13 +119,12 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe }); QualitisUtil qualitisUtil = new QualitisUtil(props); try (Connection jobConn = jobDS.getConnection(); - Connection bdpConn = bdpDS.getConnection(); Connection dopsConn = dopsDS != null ? dopsDS.getConnection() : null) { List allCheckRes = dataObjectList .parallelStream() .map(proObjectMap -> { log.info("Begin to Check dataObject:" + proObjectMap.entrySet().toString()); - boolean checkRes = getDataCheckResult(proObjectMap, jobConn, bdpConn, dopsConn, props, log,action,qualitisUtil); + boolean checkRes = getDataCheckResult(proObjectMap, jobConn, dopsConn, props, log,action,qualitisUtil); if (null != action.getExecutionRequestRefContext()) { if (checkRes) { action.getExecutionRequestRefContext().appendLog("Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " has arrived"); @@ -178,7 +162,6 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe private boolean getDataCheckResult(Map proObjectMap, Connection jobConn, - Connection bdpConn, Connection dopsConn, Properties props, Logger log, @@ -231,7 +214,7 @@ private boolean getDataCheckResult(Map proObjectMap, } log.info("start to check maskis"); proObjectMap.put(DataChecker.SOURCE_TYPE, MASK_SOURCE_TYPE); - normalCheck= (getBdpTotalCount(dataObject, bdpConn, log, props) > 0 || "success".equals(fetchMaskCode(dataObject, log, props).get("maskStatus"))); + normalCheck= "success".equals(fetchMaskCode(dataObject, log, props).get("maskStatus")); if (null != action.getExecutionRequestRefContext()){ action.getExecutionRequestRefContext().appendLog(dataObjectStr+" check maskis end,check result:"+normalCheck); } @@ -316,25 +299,6 @@ private PreparedStatement getJobStatement(Connection conn, CheckDataObject dataO } } - /** - * 构造查询maskis的查询 - */ - private PreparedStatement getBdpStatement(Connection conn, CheckDataObject dataObject, String timeScape) throws SQLException { - PreparedStatement pstmt = null; - if (timeScape.equals("NULL")) { - pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP); - } else { - pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION); - pstmt.setInt(4, Integer.valueOf(timeScape) * 3600); - } - if (dataObject.getPartitionName() == null) { - dataObject.setPartitionName(""); - } - pstmt.setString(1, dataObject.getDbName()); - pstmt.setString(2, dataObject.getTableName()); - pstmt.setString(3, dataObject.getPartitionName()); - return pstmt; - } /** * 构造查询dops库的查询 @@ -414,27 +378,6 @@ private long getJobTotalCount(CheckDataObject dataObject, Connection conn, Logge } } - /** - * 查mask db - */ - private long getBdpTotalCount(CheckDataObject dataObject, Connection conn, Logger log, Properties props) { - String timeScape = props.getOrDefault(DataChecker.TIME_SCAPE, "NULL").toString(); - log.info("-------------------------------------- search bdp data "); - log.info("-------------------------------------- dataObject: " + dataObject.toString()); - try (PreparedStatement pstmt = getBdpStatement(conn, dataObject, timeScape)) { - ResultSet rs = pstmt.executeQuery(); - long ret = 0L; - while (rs.next()) { - ret ++; - } -// long ret=rs.last() ? rs.getRow() : 0; - log.info("-------------------------------------- bdp data result:"+ret); - return ret; - } catch (SQLException e) { - log.error("fetch data from bdp error", e); - return 0; - } - } /** * - 返回0表示未找到任何记录 ； diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java index 32dba01eb..6d8afb3fe 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java @@ -25,7 +25,9 @@ public class DataDruidFactory { private static volatile DruidDataSource jobInstance; - private static volatile DruidDataSource bdpInstance; + + private static volatile DruidDataSource dopsInstance; + private static volatile DruidDataSource msgInstance; public static DruidDataSource getJobInstance(Properties props, Logger log) { @@ -42,6 +44,7 @@ public static DruidDataSource getJobInstance(Properties props, Logger log) { } return jobInstance; } + public static DruidDataSource getBDPInstance(Properties props, Logger log) { if (bdpInstance == null ) { synchronized (DataDruidFactory.class) { diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml index 8543d92a9..e56b91b69 100644 --- a/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml @@ -6,7 +6,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -18,7 +18,6 @@ com.webank.wedatasphere.dss dss-scheduler-appconn ${dss.version} - provided @@ -81,7 +80,7 @@ com.google.guava guava - 28.2-android + 33.1.0-jre diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml index 7539d9e68..6d375adc2 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml +++ b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java index 1854973ae..525c19861 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java @@ -67,10 +67,10 @@ public EventChecker(Properties p, EventCheckerExecutionAction action) { String waitTime = p.getProperty(EventChecker.WAIT_TIME, "1"); Double doubleWaitTime = Double.valueOf(waitTime) * 3600 * 1000; maxWaitTime = Long.valueOf(doubleWaitTime.longValue()); - String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "60000"); queryFrequency = Integer.valueOf(query_frequency); - if(queryFrequency <10000){ - queryFrequency = 10000; + if(queryFrequency <60000){ + queryFrequency = 60000; } } diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java index 23ed8576f..8d51d4cf1 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java @@ -70,7 +70,7 @@ void initECParams(Properties props){ runDate = props.getProperty("run_date"); userTime = props.getProperty(EventChecker.USER_TIME); waitTime = props.getProperty(EventChecker.WAIT_TIME, "1"); - query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "60000"); afterSend = props.getProperty(EventChecker.AFTERSEND); } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/pom.xml b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml index ae044adb0..5d2acc145 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/pom.xml +++ b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java index 821c3a6d0..87d5265a0 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java @@ -29,5 +29,6 @@ public class AzkabanConstant { public final static String FLOW_CONTEXT_ID = "wds.linkis.flow.contextID="; public final static String LINKIS_VERSION = "linkis.version"; public final static String JOB_COMMENT = "comment"; + public final static String AUTO_DISABLED = "auto.disabled"; } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java index 76e7ef885..2d269872f 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java @@ -23,6 +23,8 @@ public class LinkisJob { private String type; private String linkistype; private String proxyUser; + + private String autoDisabled; private String dependencies; private Map conf; private String command; @@ -60,6 +62,14 @@ public void setProxyUser(String proxyUser) { this.proxyUser = proxyUser; } + public String getAutoDisabled() { + return autoDisabled; + } + + public void setAutoDisabled(String autoDisabled) { + this.autoDisabled = autoDisabled; + } + public String getDependencies() { return dependencies; } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java index f175b6ca3..c60882d50 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java @@ -70,6 +70,7 @@ private String convertJobToString(LinkisJob job){ map.put(WorkflowConstant.PROXY_USER,job.getProxyUser()); map.put(AzkabanConstant.JOB_COMMAND,job.getCommand()); map.put(AzkabanConstant.JOB_COMMENT,job.getComment()); + map.put(AzkabanConstant.AUTO_DISABLED,job.getAutoDisabled()); Map labels = new HashMap<>(1); labels.put("route", SchedulerConf.JOB_LABEL.getValue()); map.put(AzkabanConstant.JOB_LABELS, DSSCommonUtils.COMMON_GSON.toJson(labels)); @@ -114,7 +115,8 @@ private void convertConfiguration(WorkflowNode workflowNode, LinkisJob job){ configuration.forEach((k,v)-> { if(null!=v) { v.forEach((k2, v2) -> { - if(null !=v2) {job.getConf().put(confprefix + k + "." + k2, v2.toString());} + if(AzkabanConstant.AUTO_DISABLED.equals(k2) && null !=v2){job.setAutoDisabled(v2.toString());} + else if(null !=v2) {job.getConf().put(confprefix + k + "." + k2, v2.toString());} }); } }); diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java index b15e20d30..7514853d6 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java @@ -28,6 +28,7 @@ public ProjectResponseRef searchProject(RefProjectContentRequestRef.RefProjectCo params.put("project", requestRef.getProjectName()); params.put("ajax", "fetchprojectflows"); try { + logger.info("request url from Schedulis is: {}.", queryUrl); String responseBody = SchedulisHttpUtils.getHttpGetResult(queryUrl, params, ssoRequestOperation, requestRef.getWorkspace()); logger.info("responseBody from Schedulis is: {}.", responseBody); Map map = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, new TypeToken>(){}.getType()); diff --git a/dss-appconn/appconns/dss-scriptis-appconn/pom.xml b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml index 1fbb75e66..2454f4a74 100644 --- a/dss-appconn/appconns/dss-scriptis-appconn/pom.xml +++ b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sendemail-appconn/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml index 792ccf982..adc773dac 100644 --- a/dss-appconn/appconns/dss-sendemail-appconn/pom.xml +++ b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml index 050c80d84..fb80a8bb5 100644 --- a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sso-appconn/pom.xml b/dss-appconn/appconns/dss-sso-appconn/pom.xml index 2b4eb6f8a..44f684239 100644 --- a/dss-appconn/appconns/dss-sso-appconn/pom.xml +++ b/dss-appconn/appconns/dss-sso-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-workflow-appconn/pom.xml b/dss-appconn/appconns/dss-workflow-appconn/pom.xml index 8ec03fe62..555e07756 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/pom.xml +++ b/dss-appconn/appconns/dss-workflow-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java index de702dd62..ebb4a1991 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.appconn.workflow.opertion; +import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; @@ -28,6 +29,7 @@ import org.apache.linkis.rpc.Sender; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -54,7 +56,9 @@ public RefJobContentResponseRef copyRef(ThirdlyRequestRef.CopyWitContextRequestR workflowCopyRequestRef.getWorkspace(), appId, contextIdStr, projectName, version, description, workflowCopyRequestRef.getDSSLabels(), targetProjectId, (String) nodeSuffix.orElse(null), (String) newFlowName.orElse(null)); - ResponseCopyWorkflow responseCopyWorkflow = RpcAskUtils.processAskException(sender.ask(requestCopyWorkflow), + List dssLabels = workflowCopyRequestRef.getDSSLabels(); + Sender tempSend = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabels); + ResponseCopyWorkflow responseCopyWorkflow = RpcAskUtils.processAskException(tempSend.ask(requestCopyWorkflow), ResponseCopyWorkflow.class, RequestCopyWorkflow.class); Map refJobContent = new HashMap<>(2); refJobContent.put(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, responseCopyWorkflow.getDssFlow().getId()); diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java index 038db5913..febdcdf8a 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java @@ -48,7 +48,8 @@ public ExportResponseRef exportRef(ThirdlyRequestRef.RefJobContentRequestRefImpl projectId, projectName, toJson(requestRef.getWorkspace()), - requestRef.getDSSLabels()); + requestRef.getDSSLabels(), + true); Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(requestRef.getDSSLabels()); ResponseExportWorkflow responseExportWorkflow = RpcAskUtils.processAskException(sender.ask(requestExportWorkflow), ResponseExportWorkflow.class, RequestExportWorkflow.class); diff --git a/dss-appconn/dss-appconn-core/pom.xml b/dss-appconn/dss-appconn-core/pom.xml index 60f42dc4b..bcb89d155 100644 --- a/dss-appconn/dss-appconn-core/pom.xml +++ b/dss-appconn/dss-appconn-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-loader/pom.xml b/dss-appconn/dss-appconn-loader/pom.xml index 2155e4f32..73bd4b8a3 100644 --- a/dss-appconn/dss-appconn-loader/pom.xml +++ b/dss-appconn/dss-appconn-loader/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml index f12b02dd2..ae12dadfa 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java index 2f16d6fdb..25a1c0ef3 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java @@ -125,6 +125,8 @@ public String getAppConnHome(AppConnInfo appConnInfo) { } catch (DSSErrorException e) { throw new AppConnHomeNotExistsWarnException(20350, "Unzip " + zipFilePath + " failed, AppConn is " + appConnName, e); } + //解压完了，zip包就没用了，删掉。 + deleteFile(zipFilePath, "Delete the zip file " + zipFilePath.getName() + " of AppConn " + appConnName + " failed"); File oldIndexFile = AppConnIndexFileUtils.getIndexFile(appConnPath); // delete old index file. @@ -157,6 +159,7 @@ public String getAppConnHome(AppConnInfo appConnInfo) { private void deleteFile(File file, String errorMsg) { try { + LOGGER.info("delete appconn file:{}", file.getAbsolutePath()); FileUtils.forceDelete(file); } catch (IOException e) { if(StringUtils.isNotEmpty(errorMsg)) { diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml index a2693bc65..23535c78d 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index a6d019e7a..c811e515c 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -52,7 +53,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAppConnManager.class); private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); - private final Map appConns = new HashMap<>(); + private final Map appConns = new ConcurrentHashMap<>(); private volatile boolean isLoaded = false; private List appConnList = null; AppConnInfoService appConnInfoService; @@ -60,6 +61,8 @@ public abstract class AbstractAppConnManager implements AppConnManager { private AppConnRefreshThread appConnRefreshThread; private static volatile AppConnManager appConnManager; + + private static volatile boolean appConnManagerInited = false; private static boolean lazyLoad = false; public static void setLazyLoad() { @@ -67,11 +70,11 @@ public static void setLazyLoad() { } public static AppConnManager getAppConnManager() { - if (appConnManager != null) { + if (appConnManagerInited) { return appConnManager; } synchronized (AbstractAppConnManager.class) { - if (appConnManager == null) { + if (!appConnManagerInited) { //appconn-manager-core包无法引入manager-client包，会有maven循环依赖，这里通过反射获取client的实现类 //ismanager=false时，获取client端的AppConnManager实现类，ismanager=true时，获取appconn-framework端的AppConnManager实现类。 if (Objects.equals(AppConnManagerCoreConf.IS_APPCONN_MANAGER.getValue(), AppConnManagerCoreConf.hostname) @@ -87,6 +90,7 @@ public static AppConnManager getAppConnManager() { LOGGER.info("The instance of AppConnManager is {}.", appConnManager.getClass().getName()); appConnManager.init(); } + appConnManagerInited = true; return appConnManager; } } @@ -101,6 +105,7 @@ public void init() { loadAppConns(); isLoaded = true; } + LOGGER.info("AppConnManager init successfully"); } protected abstract AppConnInfoService createAppConnInfoService(); @@ -117,9 +122,6 @@ protected void loadAppConns() { LOGGER.warn("No AppConnInfos returned, ignore it."); return; } - long refreshInterval = AppInstanceConstants.APP_CONN_REFRESH_INTERVAL.getValue().toLong(); - appConnRefreshThread = new AppConnRefreshThread(this, appConnInfos); - Utils.defaultScheduler().scheduleAtFixedRate(appConnRefreshThread, refreshInterval, refreshInterval, TimeUnit.MILLISECONDS); Map appConns = new HashMap<>(); Consumer loadAndAdd = DSSExceptionUtils.handling(appConnInfo -> { AppConn appConn = loadAppConn(appConnInfo); @@ -150,6 +152,9 @@ protected void loadAppConns() { appConnList = Collections.unmodifiableList(new ArrayList<>(appConns.values())); } LOGGER.info("Inited all AppConns, the AppConn list are {}.", this.appConns.keySet()); + long refreshInterval = AppInstanceConstants.APP_CONN_REFRESH_INTERVAL.getValue().toLong(); + appConnRefreshThread = new AppConnRefreshThread(this, appConnInfos); + Utils.defaultScheduler().scheduleAtFixedRate(appConnRefreshThread, refreshInterval, refreshInterval, TimeUnit.MILLISECONDS); } protected AppConn loadAppConn(AppConnInfo appConnInfo) throws Exception { diff --git a/dss-appconn/dss-appconn-manager/pom.xml b/dss-appconn/dss-appconn-manager/pom.xml index e2b61cf83..8133bff0f 100644 --- a/dss-appconn/dss-appconn-manager/pom.xml +++ b/dss-appconn/dss-appconn-manager/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-scheduler-appconn/pom.xml b/dss-appconn/dss-scheduler-appconn/pom.xml index cc14c6c04..5d718b49b 100644 --- a/dss-appconn/dss-scheduler-appconn/pom.xml +++ b/dss-appconn/dss-scheduler-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/linkis-appconn-engineplugin/pom.xml b/dss-appconn/linkis-appconn-engineplugin/pom.xml index 363d409de..4dfe6f4b1 100644 --- a/dss-appconn/linkis-appconn-engineplugin/pom.xml +++ b/dss-appconn/linkis-appconn-engineplugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/pom.xml b/dss-appconn/pom.xml index 616ea2f66..3c1c0bf08 100644 --- a/dss-appconn/pom.xml +++ b/dss-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-apps/dss-apiservice-server/pom.xml b/dss-apps/dss-apiservice-server/pom.xml index 0c82fbb7e..e31b06c35 100644 --- a/dss-apps/dss-apiservice-server/pom.xml +++ b/dss-apps/dss-apiservice-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -143,7 +143,7 @@ org.postgresql postgresql - 42.3.3 + 42.3.9 org.apache.linkis diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java index 19b030477..8cac07324 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java @@ -195,10 +195,11 @@ public static String getResultList(JobExecuteResult executeResult,UJESClient cl - public static String getResultContent(String user, String path, int maxSize, UJESClient client) { + public static String getResultContent(String user, String path, int maxSize, UJESClient client, boolean enableLimit) { return client.resultSet(ResultSetAction.builder() .setPath(path) .setUser(user) + .setEnableLimit(enableLimit) .setPageSize(maxSize).build()).getResponseBody(); } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java index f6a66fc6e..7a085b2ef 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java @@ -173,7 +173,8 @@ public Message openFile(HttpServletRequest req, @RequestParam(required = false, name = "taskId") String taskId, @RequestParam(required = false, name = "page", defaultValue = "1") Integer page, @RequestParam(required = false, name = "pageSize", defaultValue = "5000") Integer pageSize, - @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset) { + @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset, + @RequestParam(required = false, name = "enableLimit", defaultValue = "false") Boolean enableLimit) { String userName = SecurityFilter.getLoginUsername(req); logger.info("User {} wants to open resultSet file {} in task {}.", userName, path, taskId); if (!isNumber(taskId)) { @@ -191,7 +192,7 @@ public Message openFile(HttpServletRequest req, } else if (userName.equals(apiServiceJob.getSubmitUser())) { UJESClient client = LinkisJobSubmit.getClient(); try { - String fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client); + String fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client,enableLimit); return DSSCommonUtils.COMMON_GSON.fromJson(fileContent, Message.class); } catch (Exception e) { logger.error("User {} fetch resultSet {} failed.", userName, path, e); diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java index 95203fc07..6a9896ffb 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java @@ -38,7 +38,7 @@ public class TokenStatusCheckerTask { @Autowired ApiServiceTokenManagerDao atmd; - @Scheduled(cron = "0 0/5 * * * ?") + @Scheduled(cron = "0 0/10 * * * ?") public void doTokenStatusCheckTask() { // 查询启用状态的token List tokenManagerVos = atmd.queryTokenByStatus(1); diff --git a/dss-apps/dss-apps-server/pom.xml b/dss-apps/dss-apps-server/pom.xml index 827480e12..16b69981f 100644 --- a/dss-apps/dss-apps-server/pom.xml +++ b/dss-apps/dss-apps-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/dss-api-sql-template/pom.xml b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml index 24a2d57c0..9197eddcb 100644 --- a/dss-apps/dss-data-api/dss-api-sql-template/pom.xml +++ b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/dss-data-api-server/pom.xml b/dss-apps/dss-data-api/dss-data-api-server/pom.xml index ab98a16c4..13833499c 100644 --- a/dss-apps/dss-data-api/dss-data-api-server/pom.xml +++ b/dss-apps/dss-data-api/dss-data-api-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/pom.xml b/dss-apps/dss-data-api/pom.xml index 1e1730298..cfd07aa2b 100644 --- a/dss-apps/dss-data-api/pom.xml +++ b/dss-apps/dss-data-api/pom.xml @@ -6,7 +6,7 @@ dss com.webank.wedatasphere.dss ../../pom.xml - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT 4.0.0 dss-data-api diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml index f0061c59b..d4949fbcb 100644 --- a/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml index 8e39df352..9d9897e27 100644 --- a/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml index 891faa5f4..7e6ce0ae9 100644 --- a/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml index 25689c268..d5ea31ffb 100644 --- a/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/pom.xml b/dss-apps/dss-data-governance/pom.xml index 668d5c2c2..9faa587d4 100644 --- a/dss-apps/dss-data-governance/pom.xml +++ b/dss-apps/dss-data-governance/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/dss-scriptis-server/pom.xml b/dss-apps/dss-scriptis-server/pom.xml index 74d84e93f..397f06dcc 100644 --- a/dss-apps/dss-scriptis-server/pom.xml +++ b/dss-apps/dss-scriptis-server/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml index e7e6add8d..552594e02 100644 --- a/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml +++ b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml @@ -5,7 +5,7 @@ dss-user-guide com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-apps/dss-user-guide/pom.xml b/dss-apps/dss-user-guide/pom.xml index b506cc4b3..2fbc510a3 100644 --- a/dss-apps/dss-user-guide/pom.xml +++ b/dss-apps/dss-user-guide/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/pom.xml b/dss-apps/pom.xml index 3731e59ae..35380ec38 100644 --- a/dss-apps/pom.xml +++ b/dss-apps/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-commons/dss-common/pom.xml b/dss-commons/dss-common/pom.xml index 23c2acc60..139edd738 100644 --- a/dss-commons/dss-common/pom.xml +++ b/dss-commons/dss-common/pom.xml @@ -22,7 +22,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml dss-common @@ -30,6 +30,7 @@ 5.1.49 2.30.1 + 1.21 @@ -109,7 +110,7 @@ com.monitorjbl xlsx-streamer - 1.2.1 + 2.1.0 com.fasterxml.jackson.core @@ -117,7 +118,11 @@ org.apache.poi - ooxml-schemas + poi-ooxml-schemas + + + org.apache.commons + commons-compress @@ -155,6 +160,11 @@ ${mysql.connector.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java index 7595a45ec..1e0112d98 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java @@ -40,6 +40,11 @@ public class DSSProject implements Project { private Boolean editable; + /** + * 1-接入git 0-不接入（默认） + */ + private Boolean associateGit; + @Override public Long getId() { @@ -169,6 +174,15 @@ public String toString() { ", workspaceId=" + workspaceId + ", workspaceName='" + workspaceName + '\'' + ", editable=" + editable + + ", associateGit=" + associateGit + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java index fd04be0f7..72fd0186e 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java @@ -68,7 +68,7 @@ public class DSSCommonUtils { public static final String DSS_LABELS_KEY = "labels"; public static final String DSS_EXECUTE_BY_PROXY_USER_KEY = "execByProxyUser"; - public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", ""); + public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", "/appcom/Install/dss"); public static long parseToLong(Object val) { if (val instanceof Double) { diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java index ab0610e61..555972a13 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java @@ -24,9 +24,7 @@ import java.io.*; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Properties; +import java.util.*; /** * IO通用类。提供对一次IO操作的临时目录创建、文件创建、IO流创建功能。 @@ -37,6 +35,7 @@ public class IoUtils { private static Logger logger = LoggerFactory.getLogger(IoUtils.class); private static final String DATE_FORMAT = "yyyyMMddHHmmssSSS"; private static final String DEFAULT_IO_FILE_NAME = "IO.properties"; + public static final String FLOW_META_DIRECTORY_NAME = ".metaConf"; /** * 生成一个基础目录，这个目录用于某次任务的临时文件目录 @@ -51,7 +50,62 @@ public static String generateIOPath(String userName,String projectName,String su return addFileSeparator(baseUrl, dataStr, userName, projectName, subDir); } - private static String addFileSeparator(String... str) { + public static List getSubdirectoriesNames(String directoryPath) { + File directory = new File(directoryPath); + File[] subdirectories = directory.listFiles(File::isDirectory); // 使用文件过滤器获取只有目录的File数组 + + List subdirectoriesNames = new ArrayList<>(); + if (subdirectories != null) { + for (File subdirectory : subdirectories) { + subdirectoriesNames.add(subdirectory.getName()); // 将目录名称添加到列表中 + } + } + return subdirectoriesNames; + } + + /** + * 生成一个临时目录 + * @param userName 用户名 + * @return 临时目录 + */ + public static String generateTempIOPath(String userName){ + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName); + } + + /** + * 生产一个项目文件夹 + * @param userName + * @param projectName + * @return + */ + public static String generateProjectIOPath(String userName,String projectName){ + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName, projectName); + } + + /** + * 生成一个工作流元数据目录，这个目录用于某次任务的临时文件目录 + */ + public static String generateFlowMetaIOPath(String projectPath,String flowName) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME,flowName); + } + + + /** + * 生成一个工作流代码目录，这个目录用于某次任务的临时文件目录 + */ + public static String generateFlowCodeIOPath(String projectPath,String flowName) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(projectPath, flowName); + } + + public static String addFileSeparator(String... str) { return Arrays.stream(str).reduce((a, b) -> a + File.separator + b).orElse(""); } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java index 55652b6fb..687860493 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java @@ -24,11 +24,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; +import java.io.*; import java.util.ArrayList; +import java.util.Enumeration; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; public class ZipHelper { @@ -126,8 +128,53 @@ public static String zip(String dirPath, boolean deleteOriginDir)throws DSSError * @return 解压后的文件夹名 * @throws DSSErrorException 解压出现异常 */ - - public static String unzip(String dirPath)throws DSSErrorException { + @Deprecated + public static String unzip(String dirPath)throws DSSErrorException{ + return unzip(dirPath, false); + } + /** + * 解压一个zip文件 + * @param zipFilePath zip文件的全路径名 + * @destDirectory 解压到的目的地 + * @return 解压后的文件夹名 + * @throws DSSErrorException 解压出现异常 + */ + public static void unzipFile(String zipFilePath, String destDirectory, boolean deleteOriginZip) throws DSSErrorException { + File destDir = new File(destDirectory); + // 使用 try-with-resources 确保资源被自动关闭 + File zipFile = new File(zipFilePath); + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath))) { + byte[] buffer = new byte[1024]; + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + File newFile = new File(destDir, zipEntry.getName()); + if (zipEntry.isDirectory()) { + newFile.mkdirs(); + } else { + // 为解压后的文件创建文件输出流 + try (FileOutputStream fos = new FileOutputStream(newFile)) { + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + } + } + zipEntry = zis.getNextEntry(); + } + zis.closeEntry(); + }catch(final Exception e){ + logger.error(" 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009, "unzip" + zipFilePath + " to " + destDirectory + "zip file failed"); + exception.initCause(e); + throw exception; + } + // 如果指定删除原zip文件 + if (deleteOriginZip) { + new File(zipFilePath).delete(); + } + } + @Deprecated + public static String unzip(String dirPath,boolean deleteOriginZip)throws DSSErrorException { File file = new File(dirPath); if(!file.exists()){ logger.error("{} 不存在, 不能解压zip文件", dirPath); @@ -166,6 +213,9 @@ public static String unzip(String dirPath)throws DSSErrorException { if (exitCode != 0){ throw new DSSErrorException(90007,errMsg.toString()); } + if(deleteOriginZip){ + file.delete(); + } }catch(final Exception e){ logger.error("{} 解压缩 zip 文件失败, reason: ", e); DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); @@ -179,6 +229,19 @@ public static String unzip(String dirPath)throws DSSErrorException { } return longZipFilePath; } + public static String readImportZipProjectName(String zipFilePath) throws IOException { + try(ZipFile zipFile =new ZipFile(zipFilePath)){ + Enumeration entries =zipFile.entries(); + if(entries.hasMoreElements()){ + String projectName=entries.nextElement().getName(); + while (projectName.endsWith(File.separator)){ + projectName = projectName.substring(0, projectName.length() - 1); + } + return projectName; + } + } + throw new IOException(); + } private static boolean deleteDir(File dir) { if (dir.isDirectory()) { diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala index 41e6b87c8..e895f798d 100644 --- a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala @@ -41,7 +41,8 @@ case class RequestExportWorkflow(userName: String, projectId: Long, projectName: String, workspaceStr: String, - dssLabelList: java.util.List[DSSLabel] + dssLabelList: java.util.List[DSSLabel], + exportExternalNodeAppConnResource: Boolean ) case class ResponseExportWorkflow(resourceId: String, version: String, flowID: Long) diff --git a/dss-commons/dss-contextservice/pom.xml b/dss-commons/dss-contextservice/pom.xml index e72f0c672..6c4a3af2c 100644 --- a/dss-commons/dss-contextservice/pom.xml +++ b/dss-commons/dss-contextservice/pom.xml @@ -21,7 +21,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml dss-contextservice diff --git a/dss-commons/dss-sender-service/pom.xml b/dss-commons/dss-sender-service/pom.xml index ba32deb18..3c270bb87 100644 --- a/dss-commons/dss-sender-service/pom.xml +++ b/dss-commons/dss-sender-service/pom.xml @@ -21,7 +21,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java index 212d631ec..4aec347e1 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java @@ -36,5 +36,7 @@ public interface DSSSenderService { Sender getProjectServerSender(); + Sender getGitSender(); + } diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java index ff33f617c..37fd3a90c 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java @@ -28,6 +28,9 @@ public class DSSSenderServiceConf { public static final CommonVars PROJECT_SERVER_NAME = CommonVars.apply("wds.dss.project.sever.name", "dss-framework-project-server"); + public static final CommonVars GIT_SERVER_NAME = + CommonVars.apply("wds.dss.git.sever.name", "dss-framework-git-server"); + public static final CommonVars DSS_SERVER_NAME = CommonVars.apply("wds.dss.sever.name.dev", "dss-server-dev"); diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java index ff403f86c..372370571 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java @@ -29,6 +29,8 @@ public class DSSSenderServiceImpl implements DSSSenderService { private final Sender workflowSender = Sender.getSender(DSSSenderServiceConf.DSS_WORKFLOW_APPLICATION_NAME_DEV.getValue()); private final Sender projectSender = Sender.getSender(DSSSenderServiceConf.PROJECT_SERVER_NAME.getValue()); + + private final Sender gitSender = Sender.getSender(DSSSenderServiceConf.GIT_SERVER_NAME.getValue()); @Override public Sender getOrcSender() { return orcSender; @@ -64,4 +66,9 @@ public Sender getProjectServerSender() { return projectSender; } + @Override + public Sender getGitSender() { + return gitSender; + } + } diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java index 4ebe11e5f..58046caf8 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java @@ -44,4 +44,9 @@ public Sender getSchedulerWorkflowSender() { public Sender getProjectServerSender() { return dssServerSender; } + + @Override + public Sender getGitSender() { + return dssServerSender; + } } diff --git a/dss-commons/pom.xml b/dss-commons/pom.xml index f719aa12c..0a6b56722 100644 --- a/dss-commons/pom.xml +++ b/dss-commons/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-framework/dss-appconn-framework/pom.xml b/dss-framework/dss-appconn-framework/pom.xml index be1f80ef5..de304b515 100644 --- a/dss-framework/dss-appconn-framework/pom.xml +++ b/dss-framework/dss-appconn-framework/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-framework/dss-framework-admin-service/pom.xml b/dss-framework/dss-framework-admin-service/pom.xml index 901db373b..04284ad36 100644 --- a/dss-framework/dss-framework-admin-service/pom.xml +++ b/dss-framework/dss-framework-admin-service/pom.xml @@ -4,7 +4,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml diff --git a/dss-framework/dss-framework-common/pom.xml b/dss-framework/dss-framework-common/pom.xml index 942384562..7a9df017c 100644 --- a/dss-framework/dss-framework-common/pom.xml +++ b/dss-framework/dss-framework-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-framework/dss-framework-orchestrator-server/pom.xml b/dss-framework/dss-framework-orchestrator-server/pom.xml index 48b2caf90..de0b1f30a 100644 --- a/dss-framework/dss-framework-orchestrator-server/pom.xml +++ b/dss-framework/dss-framework-orchestrator-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -29,6 +29,36 @@ dss-framework-orchestrator-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + + + org.seleniumhq.selenium + selenium-java + 3.9.1 + + + com.google.guava + guava + + + httpclient + org.apache.httpcomponents + + + + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-workflow-server + ${dss.version} + com.webank.wedatasphere.dss dss-framework-common diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java index 4ab4f9867..a8b51fa8b 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java @@ -23,6 +23,6 @@ public class OrchestratorConf { public static final CommonVars DSS_UPLOAD_PATH = CommonVars.apply("wds.dss.file.upload.dir", "/appcom/tmp/uploads"); public static final CommonVars DSS_PUBLISH_MAX_VERSION = CommonVars.apply("wds.dss.publish.max.remain.version", 30); public static final CommonVars DSS_CS_CLEAR_ENV = CommonVars.apply("wds.dss.server.cs.clear.env", "DEV"); - public static final CommonVars DSS_CS_CLEAR_CRON = CommonVars.apply("wds.dss.server.scheduling.clear.cs.cron", ""); + public static final CommonVars DSS_CS_CLEAR_CRON = CommonVars.apply("wds.dss.server.scheduling.clear.cs.cron", "0 0 0/6 * * ?"); } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java index 40fe3495a..ca259265f 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.orchestrator.server.conf; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -42,9 +43,9 @@ public BMLService createBmlService() { } @Bean - public ScheduledThreadPoolExecutor scheduledExecutorService() { - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(3); - return executor; + public ScheduledThreadPoolExecutor dssDefaultScheduledExecutor() { + return new ScheduledThreadPoolExecutor(30, + new ThreadFactoryBuilder().setNameFormat("Dss-Default-Spring-Scheduler-Thread-%d").build()); } @Bean diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java index 9fe2c841f..6b5d7b616 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java @@ -21,4 +21,5 @@ public class DSSOrchestratorConstant { //每次提交linkis清理数量控制在10个以下，超了接口就会返回超时 public static final int MAX_CLEAR_SIZE = 10; + public static final String CHROME_DRIVER_PATH = "chromedriver"; } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java index d374b668c..7c63b2c59 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java @@ -52,6 +52,8 @@ public class OrchestratorCreateRequest extends OrchestratorRequest { private String workspaceName; + private String isDefaultReference; + public OrchestratorCreateRequest() { } @@ -130,6 +132,14 @@ public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + @Override public String toString() { return "OrchestratorCreateRequest{" + diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java index fb62a9fac..b38bc74d4 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java @@ -47,6 +47,8 @@ public class OrchestratorModifyRequest extends OrchestratorRequest { private List dssLabels; + private String isDefaultReference; + public Long getId() { return id; } @@ -117,4 +119,12 @@ public String getOrchestratorLevel() { public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java index 3c22462cb..10e70ba6e 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java @@ -42,6 +42,14 @@ public class OrchestratorRequest { */ private LabelRouteVO labels; + public OrchestratorRequest() { + } + + public OrchestratorRequest(Long workspaceId, Long projectId) { + this.workspaceId = workspaceId; + this.projectId = projectId; + } + public LabelRouteVO getLabels() { return labels; } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java new file mode 100644 index 000000000..13e42481f --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class OrchestratorSubmitRequest { + private Long flowId; + private LabelRouteVO labels; + private String projectName; + private String comment; + private Long orchestratorId; + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java index 953307bec..228ec2255 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java @@ -16,6 +16,9 @@ package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -34,6 +37,10 @@ public class OrchestratorBaseInfo { * 工程id */ private Long projectId; + /** + * 工作流的唯一id，不同环境里，同一个工作流保持一致，用来判断是否是同一个工作流。 + */ + private String uuid; /** * 编排模式id（工作流,调用orchestrator服务返回的orchestratorId） @@ -94,6 +101,8 @@ public class OrchestratorBaseInfo { private String orchestratorLevel; + private String isDefaultReference; + private boolean flowEditLockExist = false; public Boolean getEditable() { @@ -123,6 +132,7 @@ public void setReleasable(Boolean releasable) { */ private Boolean releasable; + private String status; public Long getId() { return id; @@ -148,6 +158,14 @@ public void setProjectId(Long projectId) { this.projectId = projectId; } + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + public Long getOrchestratorId() { return orchestratorId; } @@ -251,4 +269,76 @@ public String getOrchestratorLevel() { public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + + public static OrchestratorBaseInfo convertFrom(DSSOrchestratorInfo dssInfo){ + OrchestratorBaseInfo baseInfo = new OrchestratorBaseInfo(); + + // 直接对应的属性 + baseInfo.setOrchestratorVersionId(dssInfo.getId()); + baseInfo.setProjectId(dssInfo.getProjectId()); + baseInfo.setWorkspaceId(dssInfo.getWorkspaceId()); + baseInfo.setOrchestratorName(dssInfo.getName()); + baseInfo.setDescription(dssInfo.getDesc()); + baseInfo.setUses(dssInfo.getUses()); + baseInfo.setCreateUser(dssInfo.getCreator()); + baseInfo.setCreateTime(dssInfo.getCreateTime()); + baseInfo.setUpdateUser(dssInfo.getUpdateUser()); + baseInfo.setUpdateTime(dssInfo.getUpdateTime()); + + // 可能需要根据实际情况调整的属性 + baseInfo.setOrchestratorMode(dssInfo.getOrchestratorMode()); + baseInfo.setOrchestratorLevel(dssInfo.getOrchestratorLevel()); + + // DSSOrchestratorInfo中的orchestratorWay可能需要映射到OrchestratorBaseInfo的orchestratorWays列表中 + List orchestratorWays = new ArrayList<>(); + orchestratorWays.add(dssInfo.getOrchestratorWay()); + baseInfo.setOrchestratorWays(orchestratorWays); + return baseInfo; + } + + public DSSOrchestratorInfo convertToDSSOrchestratorInfo() { + OrchestratorBaseInfo baseInfo=this; + DSSOrchestratorInfo dssInfo = new DSSOrchestratorInfo(); + + // 直接对应的属性 + dssInfo.setId(baseInfo.getId()); + dssInfo.setProjectId(baseInfo.getProjectId()); + dssInfo.setWorkspaceId(baseInfo.getWorkspaceId()); + dssInfo.setName(baseInfo.getOrchestratorName()); + dssInfo.setDesc(baseInfo.getDescription()); + dssInfo.setUses(baseInfo.getUses()); + dssInfo.setCreator(baseInfo.getCreateUser()); + dssInfo.setCreateTime(baseInfo.getCreateTime()); + dssInfo.setUpdateUser(baseInfo.getUpdateUser()); + dssInfo.setUpdateTime(baseInfo.getUpdateTime()); + dssInfo.setUUID(baseInfo.getUuid()); + + // 可能需要根据实际情况调整的属性 + dssInfo.setOrchestratorMode(baseInfo.getOrchestratorMode()); + dssInfo.setOrchestratorLevel(baseInfo.getOrchestratorLevel()); + + // 处理没有直接对应的字段 + if (baseInfo.getOrchestratorWays() != null && !baseInfo.getOrchestratorWays().isEmpty()) { + // 取第一个作为orchestratorWay，这需要根据实际情况来决定是否合适 + dssInfo.setOrchestratorWay(baseInfo.getOrchestratorWays().get(0)); + } + + return dssInfo; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java new file mode 100644 index 000000000..d78a0826c --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; + +public class OrchestratorRollBackGitVo { + private DSSOrchestratorVersion oldOrcVersion; + private DSSOrchestratorVersion dssOrchestratorVersion; + private DSSOrchestratorInfo dssOrchestratorInfo; + private String version; + + public DSSOrchestratorVersion getOldOrcVersion() { + return oldOrcVersion; + } + + public void setOldOrcVersion(DSSOrchestratorVersion oldOrcVersion) { + this.oldOrcVersion = oldOrcVersion; + } + + public DSSOrchestratorVersion getDssOrchestratorVersion() { + return dssOrchestratorVersion; + } + + public void setDssOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion) { + this.dssOrchestratorVersion = dssOrchestratorVersion; + } + + public DSSOrchestratorInfo getDssOrchestratorInfo() { + return dssOrchestratorInfo; + } + + public void setDssOrchestratorInfo(DSSOrchestratorInfo dssOrchestratorInfo) { + this.dssOrchestratorInfo = dssOrchestratorInfo; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java index f982e439e..d57a96d38 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java @@ -19,34 +19,61 @@ import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnManagerUtils; import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserInfoRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserInfoResponse; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorSubmitJob; import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.server.conf.OrchestratorConf; import com.webank.wedatasphere.dss.orchestrator.server.constant.OrchestratorLevelEnum; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.*; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyHistory; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRollBackGitVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.common.protocol.RequestLockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseLockWorkflow; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; import org.apache.commons.math3.util.Pair; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; -import scala.tools.nsc.typechecker.Implicits; import javax.annotation.PostConstruct; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutionException; @RequestMapping(path = "/dss/framework/orchestrator", produces = {"application/json"}) @RestController @@ -59,12 +86,11 @@ public class DSSFrameworkOrchestratorRestful { @Autowired private OrchestratorService orchestratorService; @Autowired + private OrchestratorPluginService orchestratorPluginService; + @Autowired private HttpServletRequest httpServletRequest; - - @PostConstruct - public void init() { - AppConnManagerUtils.autoLoadAppConnManager(); - } + @Autowired + private LockMapper lockMapper; /** * 创建编排模式 @@ -95,7 +121,7 @@ public Message getAllOrchestrator(@RequestBody OrchestratorRequest orchestratorR try { String username = SecurityFilter.getLoginUsername(httpServletRequest); LOGGER.info("user {} begin to geyAllOrchestrator, requestBody:{}", username, orchestratorRequest); - return Message.ok("获取编排模式成功").data("page", orchestratorService.getListByPage(orchestratorRequest, username)); + return Message.ok("获取编排模式成功").data("page", orchestratorService.getOrchestratorInfos(orchestratorRequest, username)); } catch (Exception e) { LOGGER.error("getAllOrchestratorError ", e); return Message.error("获取编排模式失败:" + e.getMessage()); @@ -134,6 +160,12 @@ public Message deleteOrchestrator(@RequestBody OrchestratorDeleteRequest deleteR if (orchestratorFrameworkService.getOrchestratorCopyStatus(deleteRequest.getId())) { return Message.error("当前工作流正在被复制，不允许删除"); } + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(deleteRequest.getProjectId()); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } CommonOrchestratorVo orchestratorVo = orchestratorFrameworkService.deleteOrchestrator(username, deleteRequest, workspace); AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, orchestratorVo.getOrchestratorId(), orchestratorVo.getOrchestratorName(), OperateTypeEnum.DELETE, deleteRequest); @@ -229,11 +261,16 @@ public Message rollbackOrchestrator(HttpServletRequest request, @RequestBody Rol Long projectId = rollbackOrchestratorRequest.getProjectId(); String projectName = rollbackOrchestratorRequest.getProjectName(); Workspace workspace = SSOHelper.getWorkspace(request); - DSSLabel envDSSLabel = new EnvDSSLabel(rollbackOrchestratorRequest.getLabels().getRoute()); + LabelRouteVO labels = rollbackOrchestratorRequest.getLabels(); try { LOGGER.info("user {} begin to rollbackOrchestrator, params:{}", username, rollbackOrchestratorRequest); - String newVersion = orchestratorService.rollbackOrchestrator(username, projectId, projectName, orchestratorId, version, envDSSLabel, workspace); - Message message = Message.ok("回滚版本成功").data("newVersion", newVersion); + OrchestratorRollBackGitVo rollbackOrchestrator = orchestratorService.rollbackOrchestrator(username, projectId, projectName, orchestratorId, version, labels, workspace); + try { + orchestratorService.rollbackOrchestratorGit(rollbackOrchestrator, username, projectId, projectName, orchestratorId, labels, workspace); + } catch (Exception e) { + return Message.ok("回滚版本成功,git回滚失败，请重新保存并提交工作流").data("newVersion", rollbackOrchestrator.getVersion()); + } + Message message = Message.ok("回滚版本成功").data("newVersion", rollbackOrchestrator.getVersion()); return message; } catch (final Throwable t) { LOGGER.error("Failed to rollback orchestrator for user {} orchestratorId {}, projectId {} version {}", @@ -272,4 +309,170 @@ public Message getVersionByOrchestratorId(@RequestBody QueryOrchestratorVersion List list = orchestratorService.getVersionByOrchestratorId(queryOrchestratorVersion.getOrchestratorId()); return Message.ok().data("list", list); } + + @RequestMapping(value = "diffFlow", method = RequestMethod.POST) + public Message diff(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(submitFlowRequest.getLabels().getRoute())); + + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(submitFlowRequest.getFlowId()); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + GitTree gitTree = orchestratorPluginService.diffFlow(submitFlowRequest, userName, workspace); + return Message.ok().data("tree", gitTree.getChildren()); + } + + @RequestMapping(value = "submitFlow", method = RequestMethod.POST) + public Message submitFlow(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + Long orchestratorId = submitFlowRequest.getOrchestratorId(); + try { + checkWorkspace(orchestratorId, workspace); + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + return Message.error("提交失败，原因为：" + e.getMessage()); + } + + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(submitFlowRequest.getLabels().getRoute())); + + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(submitFlowRequest.getFlowId()); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + try { + orchestratorPluginService.submitFlow(submitFlowRequest, userName, workspace); + } catch (Exception e) { + return Message.error("提交工作流失败，请保存工作流重试，原因为："+ e.getMessage()); + } + + + return Message.ok(); + } + + @RequestMapping(path = "gitUrl", method = RequestMethod.GET) + public Message gitUrl(@RequestParam(required = true, name = "projectName") String projectName, + @RequestParam(required = false, name = "workflowName") String workflowName, + @RequestParam(required = false, name = "workflowNodeName") String workflowNodeName, + HttpServletResponse response) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitUserInfoRequest gitUserInfoRequest = new GitUserInfoRequest(); + gitUserInfoRequest.setWorkspaceId(workspace.getWorkspaceId()); + gitUserInfoRequest.setType(GitConstant.GIT_ACCESS_READ_TYPE); + // 跳转git需解密处理 + gitUserInfoRequest.setDecrypt(true); + GitUserInfoResponse readInfoResponse = RpcAskUtils.processAskException(sender.ask(gitUserInfoRequest), GitUserInfoResponse.class, GitUserInfoRequest.class); + String gitUsername = readInfoResponse.getGitUser().getGitUser(); + String gitPassword = readInfoResponse.getGitUser().getGitPassword(); + String gitUrlPre = UrlUtils.normalizeIp(readInfoResponse.getGitUser().getGitUrl()); + String authenToken = ""; + try { + authenToken = orchestratorService.getAuthenToken(gitUrlPre, gitUsername, gitPassword); + } catch (ExecutionException e) { + LOGGER.error("git登陆失败，原因为: ", e); + return Message.error("git登陆失败，请检查git节点配置的用户名/密码/url"); + } + // 获取顶级域名 eg: ***REMOVED*** -> weoa.com + String domainIp = UrlUtils.normalizeIp(gitUrlPre); + int lastDotIndex = domainIp.lastIndexOf("."); + String topDomain = ""; + if (lastDotIndex != -1) { + int secondToLastIndexOf = domainIp.substring(0, lastDotIndex).lastIndexOf("."); + if (secondToLastIndexOf != -1) { + topDomain = domainIp.substring(secondToLastIndexOf); + } + } + String cookie = "_gitlab_session=" + authenToken + ";path=/; Domain="+ topDomain +"; HttpOnly; +"; + LOGGER.info("Cookie is {}", cookie); + response.setHeader("Access-Control-Allow-Origin", topDomain); + response.addHeader("Set-Cookie", cookie); + // 获取命名空间 + GitUserInfoRequest gitWriteUserRequest = new GitUserInfoRequest(); + gitWriteUserRequest.setWorkspaceId(workspace.getWorkspaceId()); + gitWriteUserRequest.setType(GitConstant.GIT_ACCESS_WRITE_TYPE); + GitUserInfoResponse writeInfoResponse = RpcAskUtils.processAskException(sender.ask(gitWriteUserRequest), GitUserInfoResponse.class, GitUserInfoRequest.class); + String namespace = writeInfoResponse.getGitUser().getGitUser(); + // 拼接跳转的url + String gitUrlProject = gitUrlPre + "/" + namespace + "/" + projectName; + if (!StringUtils.isEmpty(workflowName)) { + gitUrlProject += "/tree/master/" + workflowName; + } + return Message.ok().data("gitUrl", UrlUtils.normalizeIp(gitUrlProject)); + + } + + @RequestMapping(value = "submitFlow/status", method = RequestMethod.GET) + public Message submitFlow(@RequestParam Long orchestratorId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + checkWorkspace(orchestratorId, workspace); + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + return Message.error("提交失败，原因为：" + e.getMessage()); + } + + OrchestratorSubmitJob orchestratorSubmitJob = orchestratorFrameworkService.getOrchestratorStatus(orchestratorId); + // 未提交 + if (orchestratorSubmitJob == null) { + return Message.error("该编排未开始提交"); + } + String status = orchestratorSubmitJob.getStatus(); + if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equals(status)) { + return Message.error("提交失败，原因为：" + orchestratorSubmitJob.getErrorMsg()); + } + return Message.ok().data("status", status); + } + + @RequestMapping(value = "publish/history", method = RequestMethod.GET) + public Message getHistory(@RequestParam Long orchestratorId, @RequestParam String projectName) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + checkWorkspace(orchestratorId, workspace); + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + return Message.error("提交失败，原因为：" + e.getMessage()); + } + + GitHistoryResponse history = orchestratorFrameworkService.getHistory(workspace.getWorkspaceId(), orchestratorId, projectName); + + return Message.ok().data("history", history); + } + + private void checkWorkspace(Long orchestratorId, Workspace workspace) throws DSSErrorException{ + OrchestratorVo orchestratorVoById = orchestratorService.getOrchestratorVoById(orchestratorId); + if (orchestratorVoById == null) { + DSSExceptionUtils.dealErrorException(80001, "编排不存在", DSSErrorException.class); + } + long projectId = orchestratorVoById.getDssOrchestratorInfo().getProjectId(); + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } + } + + @RequestMapping(value = "allType", method = RequestMethod.GET) + public Message allType(HttpServletRequest req) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + return Message.ok().data("type", GitConstant.GIT_SERVER_SEARCH_TYPE); + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java index 29f36fd91..6149bf0e6 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java @@ -114,7 +114,7 @@ public Message importOrcFile(HttpServletRequest req, RequestImportOrchestrator importRequest = new RequestImportOrchestrator(userName, projectName, projectID, resultMap.getResourceId(), resultMap.getVersion(), null, dssLabelList, workspace); - dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestrator(importRequest); + dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestratorNew(importRequest); AuditLogUtils.printLog(userName, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, projectID, projectName, OperateTypeEnum.CREATE, importRequest); } catch (Exception e) { @@ -153,7 +153,7 @@ public void exportOrcFile(HttpServletRequest req, OrchestratorFrameworkServiceImpl.validateOperation(orchestratorVo.getDssOrchestratorInfo().getProjectId(), userName); logger.info("export orchestrator orchestratorId " + orchestratorId + ",orcVersionId:" + orcVersionId); try { - res = orchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class).exportOrchestrator(userName, + res = orchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class).exportOrchestratorNew(userName, orchestratorId, orcVersionId, projectName, dssLabelList, addOrcVersion, workspace).getBmlResource(); AuditLogUtils.printLog(userName, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, orcVersionId, orchestratorVo.getDssOrchestratorInfo().getName(), OperateTypeEnum.UPDATE, orchestratorVo); diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java index 3d56192a9..75d924b87 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java @@ -17,7 +17,10 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorSubmitJob; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.*; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyHistory; @@ -45,4 +48,9 @@ public interface OrchestratorFrameworkService { Boolean getOrchestratorCopyStatus(Long sourceOrchestratorId); DSSOrchestratorCopyInfo getOrchestratorCopyInfoById(String copyInfoId); + + + OrchestratorSubmitJob getOrchestratorStatus(Long orchestratorId); + + GitHistoryResponse getHistory(Long workspaceId, Long orchestratorId, String projectName); } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java index e15ae5050..a4db1a80c 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java @@ -16,8 +16,13 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; public interface OrchestratorPluginService { @@ -29,4 +34,9 @@ public interface OrchestratorPluginService { ResponseConvertOrchestrator getConvertOrchestrationStatus(String id); + GitTree diffFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace); + + void submitFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException; + + GitCommitResponse submitWorkflowToBML(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace); } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java index 0e19e5860..f2e58a5cf 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java @@ -17,21 +17,26 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRollBackGitVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import java.util.List; +import java.util.concurrent.ExecutionException; public interface OrchestratorService { @@ -137,18 +142,26 @@ OrchestratorUnlockVo unlockOrchestrator(String userName, List getOrchestratorVersions(String username, Long projectId, Long orchestratorId); - String rollbackOrchestrator(String username, Long projectId, String projectName, - Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception; + OrchestratorRollBackGitVo rollbackOrchestrator(String username, Long projectId, String projectName, + Long orchestratorId, String version, LabelRouteVO labels, Workspace workspace) throws Exception; + void rollbackOrchestratorGit(OrchestratorRollBackGitVo rollBackGitVo, String userName, Long projectId, String projectName, + Long orchestratorId, LabelRouteVO labels, Workspace workspace) throws Exception; //**** new method void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String orchestratorName) throws DSSFrameworkErrorException; - Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException; + Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest, DSSProject dssProject, String username) throws DSSFrameworkErrorException; - List getListByPage(OrchestratorRequest orchestratorRequest, String username); + List getOrchestratorInfos(OrchestratorRequest orchestratorRequest, String username); ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos); void batchClearContextId(); + String getAuthenToken(String gitUrl, String gitUsername, String gitPassword) throws ExecutionException; + + OrchestratorVo getOrchestratorByAppId(Long appId); + + List getOrchestratorInfoByLabel(OrchestratorRequest orchestratorRequest); + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java index a57950020..2258b1068 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java @@ -38,10 +38,14 @@ import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCommitInfoBetweenRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitHistoryRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRemoveRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; import com.webank.wedatasphere.dss.orchestrator.core.exception.DSSOrchestratorErrorException; import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelation; @@ -67,6 +71,7 @@ import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; @@ -104,6 +109,8 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe private OrchestratorCopyEnv orchestratorCopyEnv; @Autowired private OrchestratorOperateService orchestratorOperateService; + @Autowired + private LockMapper lockMapper; private static final int MAX_DESC_LENGTH = 250; private static final int MAX_NAME_LENGTH = 128; @@ -115,7 +122,6 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe private final ExecutorService orchestratorCopyThreadPool = new ThreadPoolExecutor(0, 200, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1024), orchestratorCopyThreadFactory, new ThreadPoolExecutor.AbortPolicy()); - /** * 1.拿到的dss orchestrator的appconn * 2.然后创建 @@ -156,6 +162,7 @@ public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCrea dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorCreateRequest.getOrchestratorWays())); dssOrchestratorInfo.setOrchestratorMode(orchestratorCreateRequest.getOrchestratorMode()); dssOrchestratorInfo.setOrchestratorLevel(orchestratorCreateRequest.getOrchestratorLevel()); + dssOrchestratorInfo.setIsDefaultReference(orchestratorCreateRequest.getIsDefaultReference()); //1.去orchestratorFramework创建编排模式 LOGGER.info("{} begins to create a orchestrator {}.", username, orchestratorCreateRequest); List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorCreateRequest.getLabels().getRoute())); @@ -235,7 +242,7 @@ public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModi DSSProject dssProject = validateOperation(orchestratorModifyRequest.getProjectId(), username); workspace.setWorkspaceName(dssProject.getWorkspaceName()); //是否存在相同的编排名称 //todo 返回orchestratorInfo而不是id - Long orchestratorId = orchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest); + Long orchestratorId = orchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest, dssProject, username); LOGGER.info("{} begins to update a orchestrator {}.", username, orchestratorModifyRequest.getOrchestratorName()); List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorModifyRequest.getLabels().getRoute())); DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorModifyRequest.getOrchestratorMode()); @@ -255,6 +262,7 @@ public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModi dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorModifyRequest.getOrchestratorWays())); dssOrchestratorInfo.setOrchestratorLevel(orchestratorModifyRequest.getOrchestratorLevel()); dssOrchestratorInfo.setUses(orchestratorModifyRequest.getUses()); + dssOrchestratorInfo.setIsDefaultReference(orchestratorModifyRequest.getIsDefaultReference()); //1.如果调度系统要求同步创建工作流，向调度系统发送更新工作流的请求 tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, dssOrchestratorInfo, OrchestrationService::getOrchestrationUpdateOperation, @@ -280,7 +288,25 @@ public CommonOrchestratorVo deleteOrchestrator(String username, OrchestratorDele (structureOperation, structureRequestRef) -> ((OrchestrationDeletionOperation) structureOperation) .deleteOrchestration((RefOrchestrationContentRequestRef) structureRequestRef), "delete"); } - + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + try { + // git删除成功之后再删除库表记录 + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + List path = new ArrayList<>(); + path.add(orchestratorInfo.getName()); + GitRemoveRequest removeRequest = new GitRemoveRequest(workspace.getWorkspaceId(), dssProject.getName(), path, username); + GitCommitResponse commitResponse = RpcAskUtils.processAskException(sender.ask(removeRequest), GitCommitResponse.class, GitRemoveRequest.class); + lockMapper.updateOrchestratorStatus(orchestratorDeleteRequest.getId(), OrchestratorRefConstant.FLOW_STATUS_PUSH); + DSSOrchestratorVersion versionById = orchestratorMapper.getLatestOrchestratorVersionById(orchestratorInfo.getId()); + if (versionById != null) { + lockMapper.updateOrchestratorVersionCommitId(commitResponse.getCommitId(), versionById.getAppId()); + } + } catch (Exception e) { + LOGGER.error("git remove failed, the reason is: ", e); + lockMapper.updateOrchestratorStatus(orchestratorDeleteRequest.getId(), OrchestratorRefConstant.FLOW_STATUS_SAVE); + throw new GitErrorException(800001, e.getMessage()); + } + } orchestratorService.deleteOrchestrator(username, workspace, dssProject.getName(), orchestratorInfo.getId(), dssLabels); orchestratorOperateService.deleteTemplateOperate(orchestratorInfo.getId()); LOGGER.info("delete orchestrator {} by orchestrator framework succeed.", orchestratorInfo.getName()); @@ -325,9 +351,15 @@ public String copyOrchestrator(String username, OrchestratorCopyRequest orchestr } else if (orchestratorCopyRequest.getWorkflowNodeSuffix().length() > 10) { DSSExceptionUtils.dealErrorException(6014, "The node suffix length can not exceed 10. (节点后缀长度不能超过10)", DSSOrchestratorErrorException.class); } + String dssLabel = null; + if (orchestratorCopyRequest.getLabels()!= null && orchestratorCopyRequest.getLabels().getRoute() != null) { + dssLabel = orchestratorCopyRequest.getLabels().getRoute(); + } else { + dssLabel = DSSCommonUtils.ENV_LABEL_VALUE_DEV; + } OrchestratorCopyVo orchestratorCopyVo = new OrchestratorCopyVo.Builder(username, sourceProject.getId(), sourceProject.getName(), targetProject.getId(), targetProject.getName(), sourceOrchestratorInfo, orchestratorCopyRequest.getTargetOrchestratorName(), - orchestratorCopyRequest.getWorkflowNodeSuffix(), new EnvDSSLabel(DSSCommonUtils.ENV_LABEL_VALUE_DEV), + orchestratorCopyRequest.getWorkflowNodeSuffix(), new EnvDSSLabel(dssLabel), workspace, Sender.getThisInstance()).setCopyTaskId(null).build(); OrchestratorCopyJob orchestratorCopyJob = new OrchestratorCopyJob(); orchestratorCopyJob.setOrchestratorCopyVo(orchestratorCopyVo); @@ -423,4 +455,40 @@ private static boolean hasProjectEditPriv(Long projectId, String username) { return hasEditPriv || projectUserAuthResponse.getProjectOwner().equals(username); } + @Override + public OrchestratorSubmitJob getOrchestratorStatus(Long orchestratorId) { + return orchestratorMapper.selectSubmitJobStatus(orchestratorId); + } + + @Override + public GitHistoryResponse getHistory(Long workspaceId, Long orchestratorId, String projectName) { + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + List versionByOrchestratorId = orchestratorMapper.getVersionByOrchestratorId(orchestratorId); + if (CollectionUtils.isEmpty(versionByOrchestratorId)) { + return new GitHistoryResponse(); + } + String workflowName = orchestrator.getName(); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitHistoryResponse historyResponse = null; + // 当前未有版本发布 + GitCommitInfoBetweenRequest commitInfoBetweenRequest = new GitCommitInfoBetweenRequest(); + String newCommitId = versionByOrchestratorId.get(0).getCommitId(); + commitInfoBetweenRequest.setNewCommitId(newCommitId); + String oldCommitId = null; + int i = 0; + for (DSSOrchestratorVersion version : versionByOrchestratorId) { + if (version.getCommitId()!= null && i++ != 0) { + oldCommitId = version.getCommitId(); + break; + } + } + commitInfoBetweenRequest.setOldCommitId(oldCommitId); + commitInfoBetweenRequest.setProjectName(projectName); + commitInfoBetweenRequest.setDirName(workflowName); + commitInfoBetweenRequest.setWorkspaceId(workspaceId); + historyResponse = RpcAskUtils.processAskException(sender.ask(commitInfoBetweenRequest), GitHistoryResponse.class, GitCommitInfoBetweenRequest.class); + + return historyResponse; + } + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java index 999b70bdd..c67cd50b8 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java @@ -16,37 +16,46 @@ package com.webank.wedatasphere.dss.orchestrator.server.service.impl; +import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; import com.webank.wedatasphere.dss.common.protocol.JobStatus; +import com.webank.wedatasphere.dss.common.protocol.RequestExportWorkflow; +import com.webank.wedatasphere.dss.common.protocol.ResponseExportWorkflow; import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCommitRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitDiffRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitDiffResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorJobMapper; import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorPublishJob; import com.webank.wedatasphere.dss.orchestrator.publish.ConversionDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.publish.conf.DSSOrchestratorConf; import com.webank.wedatasphere.dss.orchestrator.publish.job.CommonUpdateConvertJobStatus; import com.webank.wedatasphere.dss.orchestrator.publish.job.ConversionJobEntity; import com.webank.wedatasphere.dss.orchestrator.publish.job.OrchestratorConversionJob; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.common.utils.Utils; @@ -76,6 +85,9 @@ public class OrchestratorPluginServiceImpl implements OrchestratorPluginService @Autowired private CommonUpdateConvertJobStatus commonUpdateConvertJobStatus; + @Autowired + private LockMapper lockMapper; + private ExecutorService releaseThreadPool = Utils.newCachedThreadPool(50, "Convert-Orchestration-Thread-", true); @Override @@ -203,6 +215,118 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO return new ResponseConvertOrchestrator(job.getId(), entity.getResponse()); } + @Override + public void submitFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException { + Long orchestratorId = flowRequest.getOrchestratorId(); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (!StringUtils.isEmpty(status) && !status.equals(OrchestratorRefConstant.FLOW_STATUS_SAVE)) { + throw new DSSErrorException(800001, "工作流无改动或改动未提交，请确认改动并保存再进行提交"); + } + releaseThreadPool.submit(() ->{ + //1. 异步提交，更新提交状态 + try { + submitWorkflowToBML(flowRequest, username, workspace); + } catch (Exception e) { + LOGGER.error("push failed, the reason is : ", e); + orchestratorMapper.updateOrchestratorSubmitJobStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, e.toString()); + } + }); + } + + @Override + public GitCommitResponse submitWorkflowToBML(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) { + Long orchestratorId = flowRequest.getOrchestratorId(); + OrchestratorSubmitJob orchestratorSubmitJob = orchestratorMapper.selectSubmitJobStatus(orchestratorId); + if (orchestratorSubmitJob == null) { + OrchestratorSubmitJob submitJob = new OrchestratorSubmitJob(); + submitJob.setOrchestratorId(orchestratorId); + submitJob.setInstanceName(Sender.getThisInstance()); + submitJob.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + orchestratorMapper.insertOrchestratorSubmitJob(submitJob); + } else { + orchestratorMapper.updateOrchestratorSubmitJobStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSHING, ""); + } + //2. 获取编排信息 + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + //3. 获取上传工作流信息 + BmlResource bmlResource = uploadWorkflowToGit(flowRequest, username, workspace, orchestrator); + // todo 3. diff（第一步补充后，使用去掉第三方节点的zip包上传到bml，替换下方的BML） + //4. 调用git服务上传 + GitCommitResponse commit = push(orchestrator.getName(), bmlResource, username, workspace.getWorkspaceId(), flowRequest.getProjectName(), flowRequest.getComment()); + if (commit == null) { + LOGGER.info("change is empty"); + } + orchestratorMapper.updateOrchestratorSubmitJobStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS, ""); + //5. 返回文件列表 + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSH); + // 更新commitId + lockMapper.updateOrchestratorVersionCommitId(commit.getCommitId(), flowRequest.getFlowId()); + return commit; + } + + + private GitCommitResponse push (String path, BmlResource bmlResource, String username, Long workspaceId, String projectName, String comment) { + LOGGER.info("-------=======================begin to add testGit1=======================-------"); + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + file.put(path, bmlResource); + GitCommitRequest request1 = new GitCommitRequest(workspaceId, projectName, file, comment, username); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitCommitResponse.class, GitCommitRequest.class); + LOGGER.info("-------=======================End to add testGit1=======================-------: {}", responseWorkflowValidNode); + return responseWorkflowValidNode; + } + + private BmlResource uploadWorkflowToGit(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace, DSSOrchestratorInfo orchestrator) { + // 1. 将序列化好的工作流文件包提交给git服务，并拿到diff文件列表结果, + long flowId = flowRequest.getFlowId(); + + Long projectId = orchestrator.getProjectId(); + String projectName = flowRequest.getProjectName(); + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(flowRequest.getLabels().getRoute())); + RequestExportWorkflow requestExportWorkflow = new RequestExportWorkflow(username, + flowId, + projectId, + projectName, + DSSCommonUtils.COMMON_GSON.toJson(workspace), + dssLabelList, + false); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabelList); + ResponseExportWorkflow responseExportWorkflow = RpcAskUtils.processAskException(sender.ask(requestExportWorkflow), + ResponseExportWorkflow.class, RequestExportWorkflow.class); + Map resourceMap = new HashMap<>(2); + BmlResource bmlResource = new BmlResource(responseExportWorkflow.resourceId(), responseExportWorkflow.version()); + + return bmlResource; + } + + @Override + public GitTree diffFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) { + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(flowRequest.getOrchestratorId()); + BmlResource bmlResource = uploadWorkflowToGit(flowRequest, username, workspace, orchestrator); + + // todo 3. diff（第一步补充后，使用去掉第三方节点的zip包上传到bml，替换下方的BML） + GitDiffResponse diff = diff(orchestrator.getName(), bmlResource, username, workspace.getWorkspaceId(), flowRequest.getProjectName()); + if (diff == null) { + LOGGER.info("change is empty"); + return null; + } + + //4. 返回文件列表 + return diff.getTree(); + } + + private GitDiffResponse diff(String path, BmlResource bmlResource, String username, Long workspaceId, String projectName) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + file.put(path, bmlResource); + GitDiffRequest request1 = new GitDiffRequest(workspaceId, projectName, file, username); + LOGGER.info("-------=======================begin to diff {}=======================-------", request1.getProjectName()); + GitDiffResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitDiffResponse.class, GitDiffRequest.class); + LOGGER.info("-------=======================End to diff testGit1=======================-------: {}", responseWorkflowValidNode); + return responseWorkflowValidNode; + } + /** * 发布整个项目中所有未发布过的工作流。 * 使用在先导入，后发布两步走的场景中 diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java index 11312bdce..21a9b5b3c 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java @@ -19,10 +19,12 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthRequest; import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; @@ -30,8 +32,15 @@ import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRenameRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRevertRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfoList; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectUpdateOrcVersion; @@ -48,8 +57,11 @@ import com.webank.wedatasphere.dss.orchestrator.server.constant.DSSOrchestratorConstant; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRollBackGitVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.development.operation.*; @@ -63,20 +75,32 @@ import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.protocol.*; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; import org.apache.commons.collections.CollectionUtils; import org.apache.linkis.cs.client.ContextClient; import org.apache.linkis.cs.client.builder.ContextClientFactory; import org.apache.linkis.rpc.Sender; +import org.openqa.selenium.*; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeDriverService; +import org.openqa.selenium.chrome.ChromeOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; +import java.io.File; import java.time.LocalDateTime; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -94,6 +118,12 @@ public class OrchestratorServiceImpl implements OrchestratorService { private ContextService contextService; @Autowired AddOrchestratorVersionHook addOrchestratorVersionHook; + @Autowired + private OrchestratorPluginService orchestratorPluginService; + @Autowired + private LockMapper lockMapper; + @Autowired + private FlowMapper flowMapper; private static final int VALID_FLAG = 1; @@ -297,7 +327,7 @@ public OrchestratorUnlockVo unlockOrchestrator(String userName, DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorInfoId, VALID_FLAG); LOGGER.info("user {} try to unlock the project {} 's orchestration(such as DSS workflow) {} of orchestrator {} in version {}.", userName, projectName, dssOrchestratorVersion.getAppId(), dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion()); - RequestUnlockWorkflow requestUnlockWorkflow = new RequestUnlockWorkflow(userName, dssOrchestratorVersion.getAppId(), confirmDelete); + RequestUnlockWorkflow requestUnlockWorkflow = new RequestUnlockWorkflow(userName, dssOrchestratorVersion.getAppId(), confirmDelete, workspace); ResponseUnlockWorkflow responseUnlockWorkflow = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() .getWorkflowSender(dssLabels).ask(requestUnlockWorkflow), ResponseUnlockWorkflow.class, RequestUnlockWorkflow.class); switch (responseUnlockWorkflow.getUnlockStatus()) { @@ -387,16 +417,16 @@ public List getOrchestratorVersions(String username, Lon @Override @Transactional(rollbackFor = Exception.class) - public String rollbackOrchestrator(String userName, Long projectId, String projectName, - Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception { + public OrchestratorRollBackGitVo rollbackOrchestrator(String userName, Long projectId, String projectName, + Long orchestratorId, String version, LabelRouteVO labels, Workspace workspace) throws Exception { //1.新建一个版本 //2.然后将version的版本内容进行去workflow进行cp //3.然后把生产的内容进行update到数据库 DSSOrchestratorVersion oldOrcVersion=orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, 1); String latestVersion = oldOrcVersion.getVersion(); - List labels = new ArrayList<>(); - labels.add(dssLabel); + DSSLabel envDSSLabel = new EnvDSSLabel(labels.getRoute()); DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + String newVersion = OrchestratorUtils.increaseVersion(latestVersion); DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); String comment = "回滚工作流到版本:" + version; @@ -415,7 +445,7 @@ public String rollbackOrchestrator(String userName, Long projectId, String proje String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); dssOrchestratorVersion.setContextId(contextId); LOGGER.info("Create a new ContextId {} for rollback the orchestration {} to version {}.", contextId, dssOrchestratorInfo.getName(), version); - RefJobContentResponseRef responseRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, Collections.singletonList(dssLabel), null, + RefJobContentResponseRef responseRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, Collections.singletonList(envDSSLabel), null, developmentService -> ((RefCRUDService) developmentService).getRefCopyOperation(), dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()).setProjectName(projectName), @@ -436,7 +466,31 @@ public String rollbackOrchestrator(String userName, Long projectId, String proje orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); addOrchestratorVersionHook.afterAdd(dssOrchestratorVersion, Collections.singletonMap(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY,paramConfTemplateIds)); // synProjectOrchestratorVersionId(dssOrchestratorVersion, labels); - return dssOrchestratorVersion.getVersion(); + + OrchestratorRollBackGitVo rollBackGitVo =new OrchestratorRollBackGitVo(); + rollBackGitVo.setOldOrcVersion(dbOrcVersion); + rollBackGitVo.setDssOrchestratorInfo(dssOrchestratorInfo); + rollBackGitVo.setDssOrchestratorVersion(dssOrchestratorVersion); + rollBackGitVo.setVersion(dssOrchestratorVersion.getVersion()); + + return rollBackGitVo; + } + + @Override + public void rollbackOrchestratorGit(OrchestratorRollBackGitVo rollBackGitVo, String userName, Long projectId, String projectName, + Long orchestratorId, LabelRouteVO labels, Workspace workspace) throws Exception{ + if (rollBackGitVo == null) { + return; + } + DSSOrchestratorVersion oldOrcVersion = rollBackGitVo.getOldOrcVersion(); + DSSOrchestratorInfo dssOrchestratorInfo = rollBackGitVo.getDssOrchestratorInfo(); + DSSOrchestratorVersion dssOrchestratorVersion = rollBackGitVo.getDssOrchestratorVersion(); + DSSProject projectInfo = DSSFlowEditLockManager.getProjectInfo(projectId); + if (projectInfo.getAssociateGit() != null && projectInfo.getAssociateGit()) { + DSSFlow dssFlow = flowMapper.selectFlowByID(dssOrchestratorVersion.getAppId()); + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + } @@ -465,7 +519,7 @@ public void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String //是否存在相同的编排名称,如果不存在相同的编排名称則返回编排id @Override - public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException { + public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest, DSSProject dssProject, String username) throws DSSFrameworkErrorException { DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorModifRequest.getId()); if (orchestratorInfo == null) { DSSFrameworkErrorException.dealErrorException(60000, "编排模式ID=" + orchestratorModifRequest.getId() + "不存在"); @@ -473,6 +527,11 @@ public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorMo //若修改了编排名称，检查是否存在相同的编排名称 if (!orchestratorModifRequest.getOrchestratorName().equals(orchestratorInfo.getName())) { isExistSameNameBeforeCreate(orchestratorModifRequest.getWorkspaceId(), orchestratorModifRequest.getProjectId(), orchestratorModifRequest.getOrchestratorName()); + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitRenameRequest renameRequest = new GitRenameRequest(orchestratorInfo.getWorkspaceId(), dssProject.getName(), orchestratorInfo.getName(), orchestratorModifRequest.getOrchestratorName(), username); + RpcAskUtils.processAskException(sender.ask(renameRequest), GitCommitResponse.class, GitRenameRequest.class); + } } return orchestratorInfo.getId(); } @@ -485,12 +544,17 @@ public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorMo * @return list of OrchestratorBaseInfo */ @Override - public List getListByPage(OrchestratorRequest orchestratorRequest, String username) { - List list = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ - put("workspace_id", orchestratorRequest.getWorkspaceId()); - put("project_id", orchestratorRequest.getProjectId()); - put("orchestrator_mode", orchestratorRequest.getOrchestratorMode()); - }}); + public List getOrchestratorInfos(OrchestratorRequest orchestratorRequest, String username) { + LabelRouteVO labels = orchestratorRequest.getLabels(); + List list = new ArrayList<>(); + if (labels == null || "dev".equals(labels.getRoute())) { + list = getOrchestratorInfoByLabel(orchestratorRequest); + } else { + List dssLabelList = Arrays.asList(new EnvDSSLabel(labels.getRoute())); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabelList); + DSSOrchestratorInfoList orchestratorInfoList = RpcAskUtils.processAskException(sender.ask(orchestratorRequest), DSSOrchestratorInfoList.class, OrchestratorRequest.class); + list = orchestratorInfoList.getOrchestratorInfos(); + } List retList = new ArrayList<>(list.size()); if (!CollectionUtils.isEmpty(list)) { //todo Is used in front-end? @@ -514,12 +578,26 @@ public List getListByPage(OrchestratorRequest orchestrator orchestratorBaseInfo.setOrchestratorId(dssOrchestratorInfo.getId()); orchestratorBaseInfo.setEditable(isEditable || isReleasable); orchestratorBaseInfo.setReleasable(isReleasable); + orchestratorBaseInfo.setIsDefaultReference(dssOrchestratorInfo.getIsDefaultReference()); + orchestratorBaseInfo.setStatus(dssOrchestratorInfo.getStatus()); + retList.add(orchestratorBaseInfo); } } return retList; } + @Override + public List getOrchestratorInfoByLabel(OrchestratorRequest orchestratorRequest) { + List dssOrchestratorInfos = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ + put("workspace_id", orchestratorRequest.getWorkspaceId()); + put("project_id", orchestratorRequest.getProjectId()); + put("orchestrator_mode", orchestratorRequest.getOrchestratorMode()); + }}); + if (dssOrchestratorInfos == null) + return new ArrayList<>(); + return dssOrchestratorInfos; + } @Override public ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos) { List orchestratorInfos = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ @@ -588,4 +666,80 @@ public void batchClearContextId() { } } + @Override + public String getAuthenToken(String gitUrlPre, String gitUsername, String gitPassword) throws ExecutionException { + // 启动chromedriver + WebDriver driver = generateChromeDriver(this.getClass().getClassLoader().getResource(DSSOrchestratorConstant.CHROME_DRIVER_PATH).getPath(), null); + String token = ""; + try { + //设置超时时间 + driver.manage().timeouts().implicitlyWait(Long.parseLong(GitServerConfig.GIT_TIME.getValue()), TimeUnit.SECONDS); + driver.manage().window().maximize(); + driver.manage().window().setSize(new Dimension(1920, 1080)); + driver.get(UrlUtils.normalizeIp(gitUrlPre)); + WebElement elementUserName = driver.findElement(By.id(GitServerConfig.GIT_USER.getValue())); + WebElement elementPassWord = driver.findElement(By.id(GitServerConfig.GIT_PASSWD.getValue())); + WebElement elementBtn = driver.findElement(By.cssSelector(GitServerConfig.GIT_SUBMIT.getValue())); + elementUserName.sendKeys(gitUsername); + elementPassWord.sendKeys(gitPassword); + elementBtn.submit(); + driver.navigate().refresh(); + LOGGER.info("for user getting... " + UrlUtils.normalizeIp(gitUrlPre)); + Set cookies = driver.manage().getCookies(); + LOGGER.info("cookies： {}", cookies.toString()); + for (Cookie cookie:cookies) { + if (cookie.getName().equals("_gitlab_session")) + { + token = cookie.getValue(); + break; + } + } + LOGGER.info("driver cookies: {}", token); + } catch (Exception e) { + LOGGER.info("error bescause: ", e); + } finally { + driver.manage().deleteAllCookies(); + driver.quit(); + } + return token; + } + + + private WebDriver generateChromeDriver(String path, Proxy seleniumProxy) throws ExecutionException { + File file = new File(path); + if (!file.canExecute() && !file.setExecutable(true)) { + throw new ExecutionException(new Exception(path + "is not executable!")); + } + + System.setProperty(ChromeDriverService.CHROME_DRIVER_EXE_PROPERTY, path); + + ChromeOptions options = new ChromeOptions(); + options.setProxy(seleniumProxy); + options.addArguments("headless"); + options.addArguments("no-sandbox"); + options.addArguments("disable-gpu"); + options.addArguments("--disable-notifications"); + options.addArguments("--disable-popup-blocking"); + options.addArguments("disable-features=NetworkService"); + options.addArguments("ignore-certificate-errors"); + options.addArguments("silent"); + options.addArguments("--disable-application-cache"); + + options.addArguments("disable-dev-shm-usage"); + options.addArguments("remote-debugging-port=9012"); + + + + return new ChromeDriver(options); + } + + @Override + public OrchestratorVo getOrchestratorByAppId(Long appId) { + OrchestratorInfo orcInfoByAppId = orchestratorMapper.getOrcInfoByAppId(appId); + if (orcInfoByAppId == null) { + return null; + } + return getOrchestratorVoById(orcInfoByAppId.getOrchestratorId()); + } + } \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala index b26472b07..c2fefcdad 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala @@ -18,7 +18,9 @@ package com.webank.wedatasphere.dss.orchestrator.server.receiver import com.webank.wedatasphere.dss.orchestrator.common.protocol._ import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} + import javax.annotation.PostConstruct import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} import org.springframework.beans.factory.annotation.Autowired @@ -52,6 +54,9 @@ class DSSOrchestratorChooser extends ReceiverChooser { case _: RequestOrchestratorVersion => receiver case _: RequestOrchestratorInfos => receiver case _: RequestQueryByIdOrchestrator => receiver + case _: RequestQuertByAppIdOrchestrator => receiver + case _: RequestSubmitOrchestratorSync => receiver + case _: OrchestratorRequest => receiver case _ => None } } \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala index 25add1c6e..53cefe64a 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala @@ -18,11 +18,12 @@ package com.webank.wedatasphere.dss.orchestrator.server.receiver import com.webank.wedatasphere.dss.common.exception.DSSErrorException import com.webank.wedatasphere.dss.common.protocol.{ResponseExportOrchestrator, ResponseImportOrchestrator} -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo +import com.webank.wedatasphere.dss.orchestrator.common.entity.{DSSOrchestratorInfoList, OrchestratorVo} import com.webank.wedatasphere.dss.orchestrator.common.protocol._ import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext import com.webank.wedatasphere.dss.orchestrator.publish.entity.OrchestratorExportResult import com.webank.wedatasphere.dss.orchestrator.publish.{ExportDSSOrchestratorPlugin, ImportDSSOrchestratorPlugin} +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.{OrchestratorRequest, OrchestratorSubmitRequest} import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} import org.apache.linkis.rpc.{Receiver, Sender} import org.slf4j.{Logger, LoggerFactory} @@ -40,7 +41,7 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr override def receiveAndReply(message: Any, sender: Sender): Any = message match { case reqExportOrchestrator: RequestExportOrchestrator => - val dssExportOrcResource: OrchestratorExportResult = orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).exportOrchestrator( + val dssExportOrcResource: OrchestratorExportResult = orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).exportOrchestratorNew( reqExportOrchestrator.getUserName, reqExportOrchestrator.getOrchestratorId, reqExportOrchestrator.getOrcVersionId, @@ -53,7 +54,7 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr ) case requestImportOrchestrator: RequestImportOrchestrator => - val dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(classOf[ImportDSSOrchestratorPlugin]).importOrchestrator(requestImportOrchestrator) + val dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(classOf[ImportDSSOrchestratorPlugin]).importOrchestratorNew(requestImportOrchestrator) ResponseImportOrchestrator(dssOrchestratorVersion.getOrchestratorId,dssOrchestratorVersion.getVersion) case addVersionAfterPublish: RequestAddVersionAfterPublish => @@ -90,13 +91,30 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr case requestQueryByIdOrchestrator: RequestQueryByIdOrchestrator => { val orcVersionId = requestQueryByIdOrchestrator.getOrcVersionId val orchestratorId = requestQueryByIdOrchestrator.getOrchestratorId - if (orchestratorId != null) { + if (orcVersionId != null) { orchestratorService.getOrchestratorVoByIdAndOrcVersionId(orchestratorId, orcVersionId) } else { orchestratorService.getOrchestratorVoById(orchestratorId) } } + case requestQuertByAppIdOrchestrator: RequestQuertByAppIdOrchestrator => + orchestratorService.getOrchestratorByAppId(requestQuertByAppIdOrchestrator.getAppId) + case requestSubmitOrchestratorSync: RequestSubmitOrchestratorSync => + val request = new OrchestratorSubmitRequest() + request.setOrchestratorId(requestSubmitOrchestratorSync.getOrchestratorId) + request.setProjectName(requestSubmitOrchestratorSync.getProjectName) + request.setFlowId(requestSubmitOrchestratorSync.getFlowId) + request.setLabels(requestSubmitOrchestratorSync.getLabels) + request.setComment(requestSubmitOrchestratorSync.getComment) + val username = requestSubmitOrchestratorSync.getUsername + val workspace = requestSubmitOrchestratorSync.getWorkspace + orchestratorPluginService.submitWorkflowToBML(request, username, workspace) + + case orchestratorRequest: OrchestratorRequest => + val list = orchestratorService.getOrchestratorInfoByLabel(orchestratorRequest) + new DSSOrchestratorInfoList(list) + case _ => throw new DSSErrorException(90000, "Not support message type " + message) } diff --git a/dss-framework/dss-framework-project-server/pom.xml b/dss-framework/dss-framework-project-server/pom.xml index d8bd3ed2d..480ae364c 100644 --- a/dss-framework/dss-framework-project-server/pom.xml +++ b/dss-framework/dss-framework-project-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -29,6 +29,11 @@ dss-framework-project-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + com.webank.wedatasphere.dss dss-framework-common @@ -207,6 +212,11 @@ + + com.webank.wedatasphere.dss + dss-framework-orchestrator-server + ${dss.version} + diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java index c3f9dfc19..124118663 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java @@ -32,8 +32,6 @@ @Mapper public interface DSSProjectMapper extends BaseMapper { - void addProject(DSSProjectPo dssProjectPo); - @Select("select id from dss_project where `name` = #{projectName}") Long getProjectIdByName(@Param("projectName") String projectName); diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml index fb1e37b25..c82aa064b 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml @@ -23,14 +23,6 @@ , `user_id`, `create_by_str`, `update_by_str` - - INSERT INTO dss_project () - VALUES - (#{id},#{name},#{source}, #{workspaceId}, #{description},#{orgID},#{visibility},#{isTransfer},#{initialOrgID}, - #{username},#{createTime},#{createBy},#{product},#{applicationArea},#{business}, - #{userID},#{createByStr},#{updateByStr}) - + select id, workspace_id, git_project_id, project_name from dss_project_associate_git where workspace_id = #{workspaceId} + + + + update dss_workspace_associate_git + + + git_user = #{gitUser}, + + + git_password = #{gitPassword}, + + + git_token = #{gitToken}, + + + git_user_id = #{gitUserId}, + + update_by = #{updateBy}, + + where workspace_id = #{workspaceId} and type = #{type} + + + + + + + + + + \ No newline at end of file diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java new file mode 100644 index 000000000..fad73fb48 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.git.dto; + +public class GitProjectGitInfo { + private Long id; + private Long workspaceId; + private String gitProjectId; + private String projectName; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getGitProjectId() { + return gitProjectId; + } + + public void setGitProjectId(String gitProjectId) { + this.gitProjectId = gitProjectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java new file mode 100644 index 000000000..cf27dc74d --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java @@ -0,0 +1,291 @@ +package com.webank.wedatasphere.dss.git.manage; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitConnectRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserInfoByRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserInfoRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserUpdateRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitConnectResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserInfoListResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserInfoResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserUpdateResponse; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.git.dao.DSSWorkspaceGitMapper; +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.linkis.DataWorkCloudApplication; +import org.apache.linkis.common.utils.Utils; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.security.Key; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class GitProjectManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(GitProjectManager.class); + + private volatile static boolean isInit; + + private static DSSWorkspaceGitMapper workspaceGitMapper; + + + protected GitProjectManager() { + } + + static { + LOGGER.info("已归档项目移除定时线程开启..."); + init(); + Utils.defaultScheduler().scheduleAtFixedRate(() -> { + try { + List allWorkspaceId = workspaceGitMapper.getAllWorkspaceId(); + for (Long workspaceId : allWorkspaceId) { + GitUserEntity gitUser = selectGit(workspaceId, GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + break; + } + List allGitProjectName = DSSGitUtils.getAllProjectName(gitUser); + List localProjectName = FileUtils.getLocalProjectName(workspaceId); + LOGGER.info("localProjectName is : {}", localProjectName.toString()); + localProjectName.removeAll(allGitProjectName); + if (localProjectName.size() > 0) { + LOGGER.info("Project to delete : {}", localProjectName.toString()); + for (String projectName : localProjectName) { + // 删除本地项目 + DSSGitUtils.archiveLocal(projectName, workspaceId); + } + } else { + LOGGER.info("Nothing need to delete"); + } + + } + } catch (Exception e) { + LOGGER.error("定时清理归档项目错误", e); + } + }, 0,1, TimeUnit.DAYS); + } + + public static void init() { + if (!isInit) { + synchronized (GitProjectManager.class) { + if (!isInit) { + workspaceGitMapper = DataWorkCloudApplication.getApplicationContext().getBean(DSSWorkspaceGitMapper.class); + isInit = true; + } + } + } + } + + public static GitUserUpdateResponse associateGit(GitUserUpdateRequest gitUserCreateRequest) throws DSSErrorException, IOException { + if (gitUserCreateRequest == null) { + throw new DSSErrorException(010101, "gitUserCreateRequest is null"); + } + GitUserEntity gitUser = gitUserCreateRequest.getGitUser(); + String userName = gitUserCreateRequest.getUsername(); + if (gitUser == null) { + throw new DSSErrorException(010101, "gitUser is null"); + } + String type = gitUser.getType(); + + + // 不存在则更新，存在则新增 + GitUserEntity oldGitUserDo = selectGit(gitUser.getWorkspaceId(), type, true); + gitUser.setUpdateBy(userName); + // 密码 token 加密处理 + if (!StringUtils.isEmpty(gitUser.getGitPassword())) { + String encryptPassword = generateKeys(gitUser.getGitPassword(), Cipher.ENCRYPT_MODE); + gitUser.setGitPassword(encryptPassword); + } + if (!StringUtils.isEmpty(gitUser.getGitToken())) { + String encryptToken = generateKeys(gitUser.getGitToken(), Cipher.ENCRYPT_MODE); + gitUser.setGitToken(encryptToken); + } + GitUserEntity gitUserEntity = workspaceGitMapper.selectByUser(gitUser.getGitUser()); + gitUser.setGitUrl(UrlUtils.normalizeIp(GitServerConfig.GIT_URL_PRE.getValue())); + if (oldGitUserDo == null) { + // 工作空间--git用户 为一一对应关系 + if (gitUserEntity != null) { + return new GitUserUpdateResponse(80001, "该用户已配置为" + gitUserEntity.getWorkspaceId() + "工作空间的读写或只读用户，请更换用户", gitUserEntity.getWorkspaceId()); + } + gitUser.setCreateBy(userName); + GitUserUpdateResponse userIdFromGit = getUserIdFromGit(gitUser, type, false, oldGitUserDo); + if (userIdFromGit != null) { + return userIdFromGit; + } + workspaceGitMapper.insert(gitUser); + }else { + if (GitConstant.GIT_ACCESS_WRITE_TYPE.equals(type) && !oldGitUserDo.getGitUser().equals(gitUser.getGitUser())) { + throw new DSSErrorException(800001, "用户名不得修改"); + } + if (gitUserEntity != null && (!gitUserEntity.getWorkspaceId().equals(gitUser.getWorkspaceId()) || !gitUserEntity.getType().equals(type))) { + return new GitUserUpdateResponse(010101, "该用户已配置为" + gitUserEntity.getWorkspaceId() + "工作空间的读写或只读用户，请更换用户", gitUserEntity.getWorkspaceId()); + } + GitUserUpdateResponse userIdFromGit = getUserIdFromGit(gitUser, type, true, oldGitUserDo); + if (userIdFromGit != null) { + return userIdFromGit; + } + workspaceGitMapper.update(gitUser); + } + + return new GitUserUpdateResponse(0,"", null); + } + + private static GitUserUpdateResponse getUserIdFromGit(GitUserEntity gitUser, String type, Boolean update, GitUserEntity oldGitUser) throws IOException, com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException { + if (GitConstant.GIT_ACCESS_READ_TYPE.equals(type)) { + GitUserEntity writeGitUser = selectGit(gitUser.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (writeGitUser == null) { + return new GitUserUpdateResponse(80001, "配置只读用户前需首先完成该工作空间编辑用户的配置", gitUser.getWorkspaceId()); + } + String userGitId = DSSGitUtils.getUserIdByUsername(writeGitUser, gitUser.getGitUser()); + gitUser.setGitUserId(userGitId); + if (update) { + List projectGitInfos = workspaceGitMapper.getProjectIdListByWorkspaceId(gitUser.getWorkspaceId()); + for (GitProjectGitInfo projectGitInfo : projectGitInfos) { + // 删除权限 + LOGGER.info("删除用户" + oldGitUser.getGitUser() + "在" + projectGitInfo.getProjectName() + "项目的只读权限"); + DSSGitUtils.removeProjectMember(writeGitUser, oldGitUser.getGitUserId(), projectGitInfo.getGitProjectId()); + LOGGER.info("删除成功"); + // 增加权限 + LOGGER.info("增加用户" + gitUser.getGitUser() + "在" + projectGitInfo.getProjectName() + "项目的只读权限"); + DSSGitUtils.addProjectMember(writeGitUser, userGitId, projectGitInfo.getGitProjectId(), 20); + LOGGER.info("增加成功"); + } + } + } + return null; + } + + public static void insert (GitProjectGitInfo projectGitInfo) { + workspaceGitMapper.insertProjectInfo(projectGitInfo); + } + + public static GitUserInfoResponse selectGitUserInfo(GitUserInfoRequest gitUserInfoRequest) throws DSSErrorException { + if (gitUserInfoRequest == null) { + throw new DSSErrorException(010101, "gitUserCreateRequest is null"); + } + Long workspaceId = gitUserInfoRequest.getWorkspaceId(); + String type = gitUserInfoRequest.getType(); + GitUserEntity gitUserEntity = selectGit(workspaceId, type, gitUserInfoRequest.getDecrypt()); + + + GitUserInfoResponse gitUserInfoResponse = new GitUserInfoResponse(); + gitUserInfoResponse.setGitUser(gitUserEntity); + return gitUserInfoResponse; + } + + public static GitUserEntity selectGit(Long workspaceId, String type, Boolean decrypt) { + try { + GitUserEntity gitUserEntity = workspaceGitMapper.selectByWorkspaceId(workspaceId, type); + if (gitUserEntity == null) { + return null; + } + // 密码 token 解密处理 + if (!StringUtils.isEmpty(gitUserEntity.getGitPassword())) { + String encryptPassword = generateKeys(gitUserEntity.getGitPassword(), Cipher.DECRYPT_MODE); + gitUserEntity.setGitPassword(encryptPassword); + } + if (!StringUtils.isEmpty(gitUserEntity.getGitToken())) { + String encryptToken = generateKeys(gitUserEntity.getGitToken(), Cipher.DECRYPT_MODE); + gitUserEntity.setGitToken(encryptToken); + } + return gitUserEntity; + } catch (Exception e) { + return null; + } + } + + private static String generateKeys(String password, int mode) throws DSSErrorException{ + // 定义一个字符串作为密钥源 + String keyString = GitServerConfig.LINKIS_MYSQL_PRI_KEY.getValue(); + if (keyString.length() < 16) { + throw new DSSErrorException(800001, "密钥长度必须大于16位"); + } + if (StringUtils.isEmpty(password)) { + throw new DSSErrorException(800001, "密码或token为空"); + } + try { + // 确保密钥长度合适，AES 密钥长度为 128 位 (16 字节) + byte[] keyBytes = keyString.substring(0, 16).getBytes(); + Key key = new SecretKeySpec(keyBytes, "AES"); + // 加密Cipher对象 + Cipher encryptCipher = Cipher.getInstance("AES"); + encryptCipher.init(mode, key); + if (mode == Cipher.ENCRYPT_MODE) { + byte[] encrypted = encryptCipher.doFinal(password.getBytes()); + return Base64.getEncoder().encodeToString(encrypted); + } else { + byte[] decode = Base64.getDecoder().decode(password); + byte[] encrypted = encryptCipher.doFinal(decode); + return new String(encrypted); + } + } catch (Exception e) { + throw new DSSErrorException(800001, "加密失败,原因为" + e); + } + } + + public static GitConnectResponse gitTokenTest(GitConnectRequest connectTestRequest)throws DSSErrorException { + // GitLab 令牌 + String token = connectTestRequest.getToken(); + // 期望匹配的用户名 + String expectedUsername = connectTestRequest.getUsername(); + GitUserEntity gitUserEntity = workspaceGitMapper.selectByUser(expectedUsername); + // GitLab API 用户信息接口 + String apiUrl = UrlUtils.normalizeIp(GitServerConfig.GIT_URL_PRE.getValue()) + "/api/v4/user"; + + try (CloseableHttpClient client = HttpClients.createDefault()) { + URIBuilder builder = new URIBuilder(apiUrl); + HttpGet request = new HttpGet(builder.build()); + request.setHeader("PRIVATE-TOKEN", token); + + HttpResponse response = client.execute(request); + String jsonResponse = EntityUtils.toString(response.getEntity()); + + + if (response.getStatusLine().getStatusCode() == 200) { + JSONObject userData = new JSONObject(jsonResponse); + + String actualUsername = userData.getString("username"); + if (actualUsername.equals(expectedUsername)) { + LOGGER.info("Token is valid and matches the username: " + actualUsername); + return new GitConnectResponse(true); + }else { + LOGGER.info("当前token与用户名" + actualUsername + "匹配，与当前用户名" + expectedUsername + "不匹配"); + throw new DSSErrorException(800001, "当前用户名 token 不匹配，请检查"); + } + } else if (response.getStatusLine().getStatusCode() == 401){ + throw new DSSErrorException(800001, "请检查token是否正确"); + } + return new GitConnectResponse(false); + } catch (DSSErrorException e) { + LOGGER.info("Error verifying token: " + e.getMessage()); + throw new DSSErrorException(800001, "校验失败" + e.getMessage()); + }catch (Exception e) { + throw new DSSErrorException(800001, "校验token失败，请确认当前环境git是否可以正常访问" + e); + } + } + + public static GitUserInfoListResponse getGitUserByType(GitUserInfoByRequest infoByRequest) { + List gitUserEntities = workspaceGitMapper.selectGitUser(infoByRequest.getWorkspaceId(), infoByRequest.getType(), infoByRequest.getGitUserName()); + GitUserInfoListResponse gitUserInfoListResponse = new GitUserInfoListResponse(); + gitUserInfoListResponse.setGitUserEntities(gitUserEntities); + return gitUserInfoListResponse; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java new file mode 100644 index 000000000..7079573f3 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.git.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; + +import java.util.concurrent.ExecutionException; + +public interface DSSGitProjectManagerService { + GitCreateProjectResponse create(GitCreateProjectRequest request) throws DSSErrorException; + + GitArchivePorjectResponse archive(GitArchiveProjectRequest request) throws DSSErrorException; + + GitCheckProjectResponse checkProject(GitCheckProjectRequest request) throws DSSErrorException; + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java new file mode 100644 index 000000000..56cfa5401 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.git.service; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public interface DSSGitWorkflowManagerService { + GitDiffResponse diff(GitDiffRequest request) throws DSSErrorException; + + GitCommitResponse commit(GitCommitRequest request) throws DSSErrorException; + + GitSearchResponse search(GitSearchRequest request) throws DSSErrorException; + + GitDeleteResponse delete(GitDeleteRequest request) throws DSSErrorException; + + GitFileContentResponse getFileContent(GitFileContentRequest request) throws DSSErrorException; + + GitHistoryResponse getHistory(GitHistoryRequest request) throws DSSErrorException; + + GitCommitResponse getCurrentCommit(GitCurrentCommitRequest request) throws DSSErrorException; + + GitCommitResponse gitCheckOut(GitRevertRequest request) throws DSSErrorException; + + GitCommitResponse removeFile(GitRemoveRequest request) throws DSSErrorException; + + GitCommitResponse rename(GitRenameRequest request) throws DSSErrorException; + + GitHistoryResponse getHistory(GitCommitInfoBetweenRequest request) throws DSSErrorException; +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java new file mode 100644 index 000000000..784b4ab90 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java @@ -0,0 +1,119 @@ +package com.webank.wedatasphere.dss.git.service.impl; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import com.webank.wedatasphere.dss.git.manage.GitProjectManager; +import com.webank.wedatasphere.dss.git.service.DSSGitProjectManagerService; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@Service +public class DSSGitProjectManagerServiceImpl implements DSSGitProjectManagerService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("projectBmlService") + private BMLService bmlService; + + @Override + public GitCreateProjectResponse create(GitCreateProjectRequest request) throws DSSErrorException{ + Long workspaceId = request.getWorkspaceId(); + GitUserEntity gitUser = GitProjectManager.selectGit(workspaceId, GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", workspaceId); + return null; + } + Repository repository = null; + try { + // Http请求Git，创建project + DSSGitUtils.init(request.getProjectName(), gitUser); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, request.getProjectName(), request.getBmlResource(), request.getUsername(), workspaceId); + FileUtils.removeProject(request.getProjectName(), workspaceId); + FileUtils.unzipBMLResource(request.getProjectName(), workspaceId); + // 本地创建Git项目 + DSSGitUtils.create(request.getProjectName(), gitUser, workspaceId); + // 获取git项目 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), workspaceId); + File repoDir = new File(gitPath); + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + // 关联远端Git + DSSGitUtils.remote(repository, request.getProjectName(), gitUser); + // 提交 + String comment = "init project: " + request.getProjectName() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 首次创建提交项目整体 + List paths = Collections.singletonList("."); + DSSGitUtils.push(repository, request.getProjectName(), gitUser, comment, paths); + // 获取工作空间只读用户 + GitUserEntity readGitUser = GitProjectManager.selectGit(workspaceId, GitConstant.GIT_ACCESS_READ_TYPE, true); + if (readGitUser == null) { + throw new DSSErrorException(80001, "只读用户不能为空，需完成工作空间制度用户设置"); + } + // 获取项目ProjectId + String projectIdByName = DSSGitUtils.getProjectIdByName(gitUser, request.getProjectName()); + DSSGitUtils.addProjectMember(gitUser, readGitUser.getGitUserId(), projectIdByName, 20); + // 存储projectId + GitProjectGitInfo projectGitInfo = new GitProjectGitInfo(); + projectGitInfo.setGitProjectId(projectIdByName); + projectGitInfo.setProjectName(request.getProjectName()); + projectGitInfo.setWorkspaceId(workspaceId); + GitProjectManager.insert(projectGitInfo); + return new GitCreateProjectResponse(); + } catch (Exception e) { + logger.error("create project failed, the reason is: ", e); + throw new DSSErrorException(80001, "create project failed, the reason is: " + e); + } finally { + if (repository != null) { + repository.close(); + } + } + } + + @Override + public GitArchivePorjectResponse archive(GitArchiveProjectRequest request) throws GitErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + // 远程归档 + DSSGitUtils.archive(request.getProjectName(), gitUser); + // 删除本地项目 + DSSGitUtils.archiveLocal(request.getProjectName(), request.getWorkspaceId()); + return new GitArchivePorjectResponse(); + } + + @Override + public GitCheckProjectResponse checkProject(GitCheckProjectRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + String projectPath = gitUser.getGitUser() + "/" + request.getProjectName(); + boolean b = DSSGitUtils.checkIfProjectExists(gitUser, projectPath); + logger.info("checkProjectName result is : {}", b); + return new GitCheckProjectResponse(request.getProjectName(), b); + } + + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java new file mode 100644 index 000000000..2c3baaa71 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java @@ -0,0 +1,631 @@ +package com.webank.wedatasphere.dss.git.service.impl; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.git.common.protocol.GitSearchLine; +import com.webank.wedatasphere.dss.git.common.protocol.GitSearchResult; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import com.webank.wedatasphere.dss.git.manage.GitProjectManager; +import com.webank.wedatasphere.dss.git.service.DSSGitWorkflowManagerService; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +public class DSSGitWorkflowManagerServiceImpl implements DSSGitWorkflowManagerService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("workflowBmlService") + private BMLService bmlService; + @Override + public GitDiffResponse diff(GitDiffRequest request) { + Long workspaceId = request.getWorkspaceId(); + GitUserEntity gitUser = GitProjectManager.selectGit(workspaceId, GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", workspaceId); + return null; + } + GitDiffResponse diff = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), workspaceId); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 解压BML文件到本地 + Map bmlResourceMap = request.getBmlResourceMap(); + List fileList = new ArrayList<>(bmlResourceMap.keySet()); + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + for (Map.Entry entry : bmlResourceMap.entrySet()) { + fileList.add(entry.getKey()); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, entry.getKey(), entry.getValue(), request.getUsername(), workspaceId); + FileUtils.removeFlowNode(entry.getKey(), request.getProjectName(), workspaceId); + FileUtils.unzipBMLResource(entry.getKey(), workspaceId); + } + diff = DSSGitUtils.diff(request.getProjectName(), fileList, workspaceId); + // 重置本地 + DSSGitUtils.reset(repository, request.getProjectName()); + } catch (Exception e) { + logger.error("pull failed, the reason is ",e); + } + return diff; + + } + + @Override + public GitCommitResponse commit(GitCommitRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + GitUserEntity gitUser = GitProjectManager.selectGit(workspaceId, GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", workspaceId); + return null; + } + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), workspaceId); + // 获取git仓库 + File repoDir = new File(gitPath); + + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 解压BML文件到本地 + Map bmlResourceMap = request.getBmlResourceMap(); + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + List paths = new ArrayList<>(); + for (Map.Entry entry : bmlResourceMap.entrySet()) { + paths.add(entry.getKey()); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, entry.getKey(), entry.getValue(), request.getUsername(), workspaceId); + FileUtils.removeFlowNode(entry.getKey(), request.getProjectName(), workspaceId); + FileUtils.unzipBMLResource(entry.getKey(), workspaceId); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + entry.getKey(); + paths.add(metaConfPath); + } + // 提交 + String comment = request.getComment() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + DSSGitUtils.push(repository, request.getProjectName(), gitUser, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + } catch (Exception e) { + logger.error("commit failed, the reason is ",e); + throw new DSSErrorException(8001, "commit workflow failed, the reason is: " + e); + } + return commitResponse; + } + + @Override + public GitSearchResponse search(GitSearchRequest request) { + Long workspaceId = request.getWorkspaceId(); + GitUserEntity gitUser = GitProjectManager.selectGit(workspaceId, GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", workspaceId); + return null; + } + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), workspaceId); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + } catch (Exception e) { + logger.error("pull failed, the reason is ",e); + return new GitSearchResponse(); + } + if (CollectionUtils.isEmpty(request.getTypeList())) { + request.setTypeList(GitConstant.GIT_SERVER_SEARCH_TYPE); + } + String gitDir = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + String gitPathPre = DSSGitConstant.GIT_PATH_PRE + request.getWorkspaceId() + File.separator; + String workTree = gitPathPre + request.getProjectName() ; + List gitCommands = new ArrayList<>(Arrays.asList( + "git", "--git-dir=" + gitDir, "--work-tree=" + workTree, "grep", "-F", "-l", request.getSearchContent() + )); + List workflowNode = request.getWorkflowNameList(); + String fileName = request.getNodeName(); + List typeList = request.getTypeList(); + List path = new ArrayList<>(); + if (!CollectionUtils.isEmpty(workflowNode)) { + if (!StringUtils.isEmpty(fileName)) { + workflowNode = workflowNode.stream().map(s -> s + "*" + fileName + "*").collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(typeList)) { + for (String workflow : workflowNode) { + for (String type : typeList) { + path.add(workflow + "." + type); + } + } + } + }else if (!CollectionUtils.isEmpty(typeList)) { + for (String workflow : workflowNode) { + for (String type : typeList) { + path.add(workflow + "/**/*." + type); + } + } + } + + if (CollectionUtils.isEmpty(path)) { + path = workflowNode; + } + }else if (!StringUtils.isEmpty(fileName)) { + if (!CollectionUtils.isEmpty(typeList)) { + for (String type : typeList) { + path.add("*" + fileName + "*/." + type); + } + }else { + path.add("*" + fileName + "*"); + } + }else if (!CollectionUtils.isEmpty(typeList)) { + for (String type : typeList) { + path.add("*." + type); + } + } + + if (!CollectionUtils.isEmpty(path)) { + gitCommands.add("--"); + gitCommands.addAll(path); + } + logger.info(gitCommands.toString()); + List fileList = process(gitCommands); + List result = new ArrayList<>(); + + if (CollectionUtils.isEmpty(fileList)) { + return new GitSearchResponse(result, 0); + } + + String excludeDirectory = GitServerConfig.GIT_SEARCH_EXCLUDE_DIRECTORY.getValue(); + String excludeFile = GitServerConfig.GIT_SEARCH_EXCLUDE_FILE.getValue(); + + List excludeDirList = StringUtils.isEmpty(excludeDirectory)? new ArrayList<>() : Arrays.asList(excludeDirectory.split(",")); + List excludeFileList = StringUtils.isEmpty(excludeFile)? new ArrayList<>() : Arrays.asList(excludeFile.split(",")); + + Set excludeResult = new HashSet<>(); + for (String file : fileList) { + //排除指定文件夹下的内容 (.metaConf) + for (String excludeDir : excludeDirList) { + if (file.startsWith(excludeDir)) { + excludeResult.add(file); + break; + } + } + // 排除指定文件下的内容(.properties) + for (String exclude : excludeFileList) { + if (file.endsWith(exclude)) { + excludeResult.add(file); + } + } + } + + if (!CollectionUtils.isEmpty(excludeResult)) { + fileList.removeAll(excludeResult); + if (CollectionUtils.isEmpty(fileList)) { + return new GitSearchResponse(result, 0); + } + } + + int start = (request.getPageNow()-1) * request.getPageSize(); + int end = Math.min((start + request.getPageSize()), fileList.size()); + if (request.getPageNow() < 0 || start >= fileList.size()) { + logger.error("当前请求页" + request.getPageNow() + "超出搜索指定范围"); + return new GitSearchResponse(result, 0); + } + // subList 截断的List及子集 不允许List变更 + List subList = new ArrayList<>(fileList.subList(start, end)); + List filePathList = new ArrayList<>(); + for (String file : subList) { + filePathList.add(workTree + File.separator + file); + } + + + + List gitBaseCommand = new ArrayList<>(Arrays.asList( + "git", "--git-dir=" + gitDir, "--work-tree=" + workTree, "grep", "-F", "-n", request.getSearchContent() + )); + + + for (String file : filePathList) { + List gitSearchCommand = new ArrayList<>(gitBaseCommand); + gitSearchCommand.add(file); + logger.info(gitSearchCommand.toString()); + + List searchResult = process(gitSearchCommand); + + List keyLines = new ArrayList<>(); + for (String resultLine : searchResult) { + // 找到第一个冒号的位置 + int colonIndex = resultLine.indexOf(':'); + + // 如果存在冒号，去除它之前的所有内容 /testGit/test/.sql:1:test -> 1:test + if (colonIndex != -1 && colonIndex + 1 < resultLine.length()) { + resultLine = resultLine.substring(colonIndex + 1); + // 找到第二个冒号位置 + int colonIndexSec = resultLine.indexOf(':'); + if (colonIndexSec != -1 && colonIndexSec + 1 < resultLine.length()) { + Integer num = Integer.valueOf(resultLine.substring(0, colonIndexSec)); + String line = resultLine.substring(colonIndexSec + 1); + if (!StringUtils.isEmpty(line)) { + // 只添加符合要求的结果 + GitSearchLine searchLine = new GitSearchLine(num, line); + keyLines.add(searchLine); + } + } + } + } + // 处理文件路径 /data/GitInstall/testGit/test/.sql -> testGit/test/.sql + if (file.startsWith(workTree + File.separator)) { + file = file.substring(workTree.length() + File.separator.length()); + } + + result.add(new GitSearchResult(file, keyLines)); + } + + + return new GitSearchResponse(result, fileList.size()); + } + + private List process(List commands) { + List result = new ArrayList<>(); + ProcessBuilder processBuilder = new ProcessBuilder(commands); + processBuilder.redirectErrorStream(true); // Merge error stream with the standard output stream + Process process = null; + try { + process = processBuilder.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + logger.info(line); + result.add(line); + } + boolean b = process.waitFor(5, TimeUnit.SECONDS); + if (b) { + int exitCode = process.waitFor(); + logger.info("Exit code: " + exitCode); + + }else { + logger.info("search timeout"); + process.destroy(); + } + } catch (IOException | InterruptedException e) { + logger.error("grep failed ,the reason is :", e); + } finally { + if (process != null) { + process.destroy(); + } + } + return result; + } + + @Override + public GitDeleteResponse delete(GitDeleteRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitDeleteResponse deleteResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + List deleteFileList = request.getDeleteFileList(); + for (String path : deleteFileList) { + File file = new File(path); + if (file.exists()) { + if (file.isDirectory()) { + FileUtils.removeDirectory(path); + } else { + FileUtils.removeFiles(path); + } + } + } + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + // 提交 + DSSGitUtils.push(repository, request.getProjectName(), gitUser,"delete " + request.getDeleteFileList(), request.getDeleteFileList()); + } catch (Exception e) { + logger.error("delete failed, the reason is ",e); + throw new DSSErrorException(80001, "delete workflow failed, the reason is: " + e); + } + return null; + } + + @Override + public GitFileContentResponse getFileContent(GitFileContentRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitFileContentResponse contentResponse = new GitFileContentResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + + String content = DSSGitUtils.getTargetCommitFileContent(repository, request.getProjectName(), request.getCommitId(), request.getFilePath()); + String fullpath = File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + request.getWorkspaceId() + File.separator + FileUtils.normalizePath(request.getFilePath()); + File file = new File(fullpath); + String fileName = file.getName(); + // todo 透传 + BmlResource bmlResource = FileUtils.uploadResourceToBML(bmlService, gitUser.getGitUser(), content, fileName, request.getProjectName()); + logger.info("upload success, the fileName is : {}", request.getFilePath()); + contentResponse.setBmlResource(bmlResource); + return contentResponse; + } catch (Exception e) { + throw new DSSErrorException(80001, "getFileContent failed, the reason is: " + e); + } + } + + @Override + public GitHistoryResponse getHistory(GitHistoryRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitHistoryResponse response = new GitHistoryResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + + List fileList = Collections.singletonList(request.getFilePath()); + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getFilePath(), null); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get Commit failed, the reason is null"); + }else { + response.setResponses(latestCommit); + } + + } catch (Exception e) { + throw new DSSErrorException(80001, "getHistory failed, the reason is: " + e); + } + return response; + } + + private Repository getRepository(File repoDir, String projectName, GitUserEntity gitUser) throws DSSErrorException { + Repository repository = null; + try { + // 当前机器不存在就新建 + if (repoDir.exists()) { + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + } else { + // 本地创建Git项目 + DSSGitUtils.create(projectName, gitUser, gitUser.getWorkspaceId()); + // 获取git项目 + String gitPath = DSSGitUtils.generateGitPath(projectName, gitUser.getWorkspaceId()); + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + DSSGitUtils.remote(repository, projectName, gitUser); + DSSGitUtils.pull(repository, projectName, gitUser); + } + } catch (Exception e) { + logger.info("get repository failed, the reason is: ", e); + throw new DSSErrorException(80001, "get repository failed, the reason is: " + e); + } + return repository; + } + + @Override + public GitCommitResponse getCurrentCommit(GitCurrentCommitRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser);){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getFilepath(), 1); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get latestCommit failed, the reason is null"); + } else { + commitResponse = latestCommit.get(0); + return commitResponse; + } + + } catch (Exception e) { + logger.error("getCurrentCommit, the reason is ",e); + throw new DSSErrorException(80001, "getCurrentCommit failed, the reason is: " + e); + } + return null; + } + + @Override + public GitCommitResponse gitCheckOut(GitRevertRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + // 回滚 + DSSGitUtils.checkoutTargetCommit(repository, request); + // push + List paths = Collections.singletonList(request.getPath()); + DSSGitUtils.push(repository, request.getProjectName(), gitUser, "revert "+ DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(), paths); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getPath(), 1); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get latestCommit failed, the reason is null"); + } else { + return latestCommit.get(0); + } + + } catch (Exception e) { + logger.error("checkOut failed, the reason is ",e); + throw new DSSErrorException(80001, "checkOut failed, the reason is: " + e); + } + return null; + } + + @Override + public GitCommitResponse removeFile(GitRemoveRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + List paths = new ArrayList<>(); + // 同步删除对应节点 + for (String path : request.getPath()) { + FileUtils.removeFlowNode(path, request.getProjectName(), request.getWorkspaceId()); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + path; + paths.add(metaConfPath); + paths.add(path); + } + // 提交 + String comment = "delete workflowNode " + request.getPath().toString() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + DSSGitUtils.push(repository, request.getProjectName(), gitUser, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + + } catch (Exception e) { + logger.error("removeFile failed, the reason is ",e); + throw new DSSErrorException(80001, "removeFile failed, the reason is: " + e); + } + return commitResponse; + } + + @Override + public GitCommitResponse rename(GitRenameRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 同步删除对应节点 eg: /data/GitInstall/224/testGit/flowGitOld -> /data/GitInstall/224/testGit/flowGitNew + String projectPath = generateProjectPath(request.getWorkspaceId(), request.getProjectName()) + File.separator; + String olfFilePath = projectPath + FileUtils.normalizePath(request.getOldName()); + String filePath = projectPath + FileUtils.normalizePath(request.getName()); + // 同步删除对应节点 eg: /data/GitInstall/224/testGit/.metaConf/flowGitOld -> /data/GitInstall/224/testGit/.metaConf/flowGitNew + String metaPath = FileUtils.normalizePath(GitConstant.GIT_SERVER_META_PATH) + File.separator + FileUtils.normalizePath(request.getOldName()); + String oldFileMetaPath = projectPath + metaPath; + String oldMetaPath = FileUtils.normalizePath(GitConstant.GIT_SERVER_META_PATH) + File.separator + FileUtils.normalizePath(request.getName()); + String fileMetaPath = projectPath + oldMetaPath; + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + FileUtils.renameFile(olfFilePath, filePath); + FileUtils.renameFile(oldFileMetaPath, fileMetaPath); + // 提交 + String comment = "rename workflowNode " + request.getName() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + List paths = new ArrayList<>(); + paths.add(FileUtils.normalizePath(request.getOldName())); + paths.add(FileUtils.normalizePath(request.getName())); + + paths.add(metaPath); + paths.add(oldMetaPath); + DSSGitUtils.push(repository, request.getProjectName(), gitUser, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + + } catch (Exception e) { + logger.error("rename failed, the reason is ",e); + throw new DSSErrorException(80001, "rename failed, the reason is: " + e); + } + return commitResponse; + } + + private String generateProjectPath(Long workspaceId, String projectName) { + return DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + projectName; + } + + @Override + public GitHistoryResponse getHistory(GitCommitInfoBetweenRequest request) throws DSSErrorException { + GitUserEntity gitUser = GitProjectManager.selectGit(request.getWorkspaceId(), GitConstant.GIT_ACCESS_WRITE_TYPE, true); + if (gitUser == null) { + logger.error("the workspace : {} don't associate with git", request.getWorkspaceId()); + return null; + } + GitHistoryResponse response = new GitHistoryResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(request.getProjectName(), request.getWorkspaceId()); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, request.getProjectName(), gitUser)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, request.getProjectName(), gitUser); + if (StringUtils.isEmpty(request.getOldCommitId())) { + // 去掉上线 + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getDirName(), null); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get Commit failed, the reason is null"); + }else { + response.setResponses(latestCommit); + } + } else { + response = DSSGitUtils.listCommitsBetween(repository, request.getOldCommitId(), request.getNewCommitId(), request.getDirName()); + } + } catch (Exception e) { + logger.error("getHistory failed, the reason is ",e); + throw new DSSErrorException(80001, "getHistory failed, the reason is: " + e); + } + return response; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java new file mode 100644 index 000000000..90a0a8fd0 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.git.thread; + +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; + + +public class GitServerThreadFactory { + private static GitServerThreadPool gitServerThreadPool = new GitServerThreadPool(GitServerConfig.GIT_THREAD_NUM.getValue()); + + public static GitServerThreadPool getGitServerThreadPool() { + return gitServerThreadPool; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java new file mode 100644 index 000000000..9a1c5a4a6 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.git.thread; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; + +public class GitServerThreadPool { + private final Map taskExecutors = new HashMap<>(); + private final int POOL_SIZE; + + public GitServerThreadPool(int num) { + this.POOL_SIZE = num; + // 初始化每个任务的ExecutorService + for (int i = 0; i < POOL_SIZE; i++) { + taskExecutors.put("TaskExecutor_" + i, Executors.newSingleThreadExecutor()); + } + } + + public Future submitTask(String taskName, Callable task) { + // 根据任务名称选择对应的ExecutorService，这里简化为根据hashCode选取 + int index = Math.abs(taskName.hashCode()) % POOL_SIZE; + ExecutorService executorService = taskExecutors.get("TaskExecutor_" + index); + return executorService.submit(task); + } + + public void shutdown() { + // 关闭所有ExecutorService + taskExecutors.values().forEach(ExecutorService::shutdown); + } + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java new file mode 100644 index 000000000..7918aa76c --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java @@ -0,0 +1,767 @@ +package com.webank.wedatasphere.dss.git.utils; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonElement; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import org.apache.http.client.methods.HttpDelete; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRevertRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitDiffResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.errors.CheckoutConflictException; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.*; + +public class DSSGitUtils { + private static final Logger logger = LoggerFactory.getLogger(DSSGitUtils.class); + + public static void init(String projectName, GitUserEntity gitUserDO) throws Exception, GitErrorException{ + String projectPath = gitUserDO.getGitUser() + "/" + projectName; + if (!checkIfProjectExists(gitUserDO, projectPath)) { + String url = UrlUtils.normalizeIp(gitUserDO.getGitUrl()) + "/" + GitServerConfig.GIT_RESTFUL_API_CREATE_PROJECTS.getValue(); + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost post = new HttpPost(url); + post.addHeader("PRIVATE-TOKEN", gitUserDO.getGitToken()); + post.addHeader("Content-Type", "application/json"); + String jsonInputString = String.format("{\"name\": \"%s\", \"description\": \"%s\"}", projectName, projectName); + post.setEntity(new StringEntity(jsonInputString)); + + try (CloseableHttpResponse response = httpClient.execute(post)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 201) { + logger.info("init success"); + } else { + throw new GitErrorException(80001, "创建Git项目失败，请检查工作空间token是否过期"); + } + } + } + } else { + throw new GitErrorException(80101, "git init failed, the reason is: projectName " + projectName +" already exists"); + } + } + + public static void remote(Repository repository, String projectName, GitUserEntity gitUser)throws GitErrorException { + // 拼接git remote Url + String remoteUrl = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/" +gitUser.getGitUser() + File.separator + projectName + ".git"; + try { + Git git = new Git(repository); + + // 添加远程仓库引用 + git.remoteAdd() + .setName("origin") + .setUri(new URIish(remoteUrl)) + .call(); + + logger.info("remote success"); + } catch (URISyntaxException e) { + throw new GitErrorException(80102, "connect Uri " + remoteUrl +" failed, the reason is: ", e); + } catch (GitAPIException e) { + throw new GitErrorException(80102, "remote git failed, the reason is: ", e); + } + + } + + + public static void create(String projectName, GitUserEntity gitUserDO, Long workspaceId) throws GitErrorException{ + logger.info("start success"); + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName); // 指定仓库的目录 + File respo = new File(generateGitPath(projectName, workspaceId)); + if (!respo.exists()) { + try { + // 初始化仓库 + Git git = Git.init() + .setDirectory(repoDir) + .call(); + + logger.info("Initialized empty Git repository in " + git.getRepository().getDirectory()); + + git.close(); // 不再需要时关闭Git对象 + } catch (GitAPIException e) { + throw new GitErrorException(80103, "git Create failed, the reason is: ", e); + } + } else { + logger.info(respo + " already exists"); + } + } + + public static void pull(Repository repository, String projectName, GitUserEntity gitUser)throws GitErrorException { + try { + Git git = new Git(repository); + + int i = 1; + while (true) { + i += 1; + // 拉取远程仓库更新至本地 + PullCommand pullCmd = git.pull().setCredentialsProvider(new UsernamePasswordCredentialsProvider(gitUser.getGitUser(), gitUser.getGitToken())); + PullResult result = pullCmd.call(); + + // 成功直接返回，失败清空本地修改重试最多3次 + if (result.isSuccessful()) { + logger.info("Pull successful!"); + break; + } else if (i <= 3){ + logger.info("Pull failed : " + result.toString()); + // 冲突时以远程仓库为准 + if (result.getMergeResult().getConflicts() != null) { + logger.info("Conflicts occurred. Resolving with remote as priority..."); + // 丢失本地修改，处理冲突 + reset(repository, projectName); + } + }else { + throw new GitErrorException(80104, "git pull failed"); + } + } + } catch (Exception e) { + // 丢失本地修改，处理冲突 + reset(repository, projectName); + } + } + + public static void pullTargetFile(Repository repository, String projectName, GitUserEntity gitUser, List paths) throws Exception { + try { + if (CollectionUtils.isEmpty(paths)) { + return ; + } + + // Opening the repository + Git git = new Git(repository); + // 拉取最新提交记录，但不进行合并 + git.fetch().call(); + // 设置需要更新的文件或文件夹路径 + try { + git.checkout() + .setStartPoint("origin/master") + .addPaths(paths) + .call(); + } catch (CheckoutConflictException e) { + logger.info("Conflicts occurred, resetting local changes to match remote..."); + git.reset() + .setMode(ResetCommand.ResetType.HARD) + .setRef("refs/remotes/origin/master") + .call(); + logger.info("reset success"); + } + + } catch (GitAPIException e) { + e.printStackTrace(); + } + } + + public static GitDiffResponse diff(String projectName, List fileList, Long workspaceId)throws GitErrorException{ + + Set status = status(projectName, fileList, workspaceId); + GitTree root = new GitTree(""); + for (String statu : status) { + root.addChild(statu); + } + // 打印树形结构 + printTree("", root); + return new GitDiffResponse(root); + } + + // 打印树结构 + static void printTree(String prefix, GitTree tree) { + logger.info(prefix + tree.getName()); + for (GitTree child : tree.getChildren().values()) { + printTree(prefix + " ", child); + } + } + + + + public static void push(Repository repository, String projectName, GitUserEntity gitUser, String comment, List paths) throws GitErrorException{ + + try { + Git git = new Git(repository); + // 添加新增、更改到暂存区 + for (String path : paths) { + // 添加所有更改 + git.add().addFilepattern(path).call(); + // 添加删除到暂存区 + git.add().setUpdate(true).addFilepattern(path).call(); + } + + // 创建新的提交 + git.commit() + .setMessage(comment) + .call(); + + logger.info("Changes committed to local repository."); + + // 推送到远程仓库 + git.push() + .setCredentialsProvider(new UsernamePasswordCredentialsProvider(gitUser.getGitUser(), gitUser.getGitToken())) + .call(); + + logger.info("Changes pushed to remote repository."); + } catch (GitAPIException e) { + reset(repository, projectName); + throw new GitErrorException(80105, "提交失败，请重试或检查token是否过期", e); + } + } + + + public static void reset(Repository repository, String projectName)throws GitErrorException { + try { + Git git = new Git(repository); + + git.reset().setMode(ResetCommand.ResetType.HARD).call(); + + git.clean().setForce(true).setCleanDirectories(true).call(); + + logger.info("git reset success : " + projectName); + } catch (GitAPIException e) { + throw new GitErrorException(80106, "git reset failed, the reason is: ", e); + } + } + + public static void checkoutTargetCommit(Repository repository, GitRevertRequest request) throws GitAPIException, IOException, GitErrorException { + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + request.getWorkspaceId() + File.separator + request.getProjectName()+ File.separator + ".git"); + String commitId = request.getCommitId(); // 替换为目标commit的完整哈希值 + + try { + Git git = new Git(repository); + // 检出（回滚）指定commit的文件版本 + git.checkout() + .setStartPoint(commitId) + .addPath(request.getPath()) + .call(); + + logger.info("git check out success"); + logger.info("File " + repoDir.getAbsolutePath() + " has been rolled back to the version at commit: " + commitId); + + } catch (GitAPIException e) { + reset(repository, request.getProjectName()); + throw new GitErrorException(80107, "git check out failed, the reason is: ", e); + } + } + + public static boolean checkIfProjectExists(GitUserEntity gitUser, String projectPath) throws GitErrorException { + String url = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/projects/" + projectPath.replace("/", "%2F"); + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(url); + request.addHeader("PRIVATE-TOKEN", gitUser.getGitToken()); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + return true; + } else if (statusCode == 404) { + return false; + } else { + String responseBody = EntityUtils.toString(response.getEntity()); + logger.info("Unexpected response status: " + statusCode); + logger.info("Response body: " + responseBody); + return false; + } + } + } catch (Exception e) { + throw new GitErrorException(80108, "检查项目名称失败，请检查工作空间token是否过期", e); + } + } + + public static String getUserIdByUsername(GitUserEntity gitUser, String username) throws GitErrorException, IOException { + String url = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/users?username=" + username; + BufferedReader in = null; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(url); + request.addHeader("PRIVATE-TOKEN", gitUser.getGitToken()); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + in = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + + String responseBody = content.toString(); + logger.info("Response Body: " + responseBody); + + JsonArray jsonArray = JsonParser.parseString(responseBody).getAsJsonArray(); + + if (jsonArray.size() > 0) { + JsonObject userObject = jsonArray.get(0).getAsJsonObject(); + return userObject.get("id").toString(); + } else { + throw new GitErrorException(80109, "获取userId失败，请检查该用户是否为git用户并激活"); + } + } else { + throw new GitErrorException(80109, "获取userId失败，请检查编辑用户token是否过期或git服务是否正常"); + } + } + } catch (Exception e) { + throw new GitErrorException(80109, "获取该git用户Id失败，原因为", e); + } finally { + if (in != null) { + in.close(); + } + } + } + + public static String getProjectIdByName(GitUserEntity gitUser, String projectName) throws GitErrorException{ + String urlString = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/projects?search=" + projectName; + + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(urlString); + request.setHeader("PRIVATE-TOKEN", gitUser.getGitToken()); + + CloseableHttpResponse response = httpClient.execute(request); + System.out.println("Response Status Line: " + response.getStatusLine()); + + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader in = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + + in.close(); + + String responseBody = content.toString(); + logger.info("Response Body: " + responseBody); + + JsonArray jsonArray = JsonParser.parseString(responseBody).getAsJsonArray(); + + if (jsonArray.size() > 0) { + for (JsonElement element : jsonArray) { + JsonObject projectObject = element.getAsJsonObject(); + if (projectObject.get("name").getAsString().equals(projectName)) { + return projectObject.get("id").toString(); + } + } + } else { + throw new GitErrorException(80110, "项目创建失败，请稍后重试"); + } + } else { + throw new GitErrorException(80110, "请检查编辑用户token是否过期或git服务是否正常"); + } + } catch (Exception e) { + throw new GitErrorException(80110, "获取该项目git Id 失败，原因为", e); + } + return null; + } + + public static boolean addProjectMember(GitUserEntity gitUser, String userId, String projectId, int accessLevel) throws GitErrorException, IOException { + String url = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/projects/" + projectId + "/members"; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(url); + request.addHeader("PRIVATE-TOKEN", gitUser.getGitToken()); + request.addHeader("Content-Type", "application/json"); + + String json = String.format("{\"user_id\": \"%s\", \"access_level\": \"%d\"}", userId, accessLevel); + request.setEntity(new StringEntity(json)); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + if (statusCode == 201) { + return true; + } else { + throw new GitErrorException(80111, "添加用户失败，请检查只读用户是否存在或编辑用户token是否过期"); + } + } + } catch (Exception e) { + throw new GitErrorException(80111, "添加用户失败，请检查编辑用户token是否过期或git服务是否正常"); + } + } + + public static boolean removeProjectMember(GitUserEntity gitUser, String userId, String projectId) throws GitErrorException { + String urlString = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/projects/" + projectId + "/members/" + userId; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpDelete request = new HttpDelete(urlString); + request.addHeader("PRIVATE-TOKEN", gitUser.getGitToken()); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode == 204) { + return true; + } else { + throw new GitErrorException(80112, "请检查工作空间Git只读用户是否存在"); + } + } + } catch (IOException e) { + throw new GitErrorException(80112, "更新用户权限失败", e); + } + } + + public static List getAllProjectName(GitUserEntity gitUserDO) throws DSSErrorException { + int page = 1; + List allProjectNames = new ArrayList<>(); + + List projectNames = new ArrayList<>(); + do { + // 修改为GitLab实例的URL + String gitLabUrl = UrlUtils.normalizeIp(gitUserDO.getGitUrl()) + "/api/v4/projects?per_page=100&page=" + page; + // 创建HttpClient实例 + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + // 创建HttpGet请求 + HttpGet request = new HttpGet(gitLabUrl); + // 添加认证头部 + request.addHeader("PRIVATE-TOKEN", gitUserDO.getGitToken()); + // 执行请求 + try (CloseableHttpResponse response = httpClient.execute(request)) { + // 获取响应实体 + HttpEntity entity = response.getEntity(); + // 将响应实体转换为字符串 + String result = EntityUtils.toString(entity); + // 解析项目名称 + projectNames = parseProjectNames(result); + // 打印项目名称 + logger.info("projectNames is: {}", projectNames.toString()); + // 添加到总项目列表中 + allProjectNames.addAll(projectNames); + } + } catch (IOException e) { + throw new GitErrorException(80113, "检查项目名称失败，请检查工作空间token是否过期", e); + } catch (Exception e) { + throw new GitErrorException(80113, "检查项目名称时解析JSON失败，请确认git当前是否可访问 ", e); + } + page++; + } while (projectNames.size() > 0); + + return allProjectNames; + } + + public static List parseProjectNames(String json) throws org.json.JSONException { + JSONArray projects = new JSONArray(json); + List projectNames = new ArrayList<>(); + + for (int i = 0; i < projects.length(); i++) { + JSONObject project = projects.getJSONObject(i); + projectNames.add(project.getString("name")); + } + + return projectNames; + } + + public static Set status(String projectName, List fileList, Long workspaceId)throws GitErrorException { + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName + File.separator +".git"); // 修改为你的仓库路径 + + try (Repository repository = new FileRepositoryBuilder().setGitDir(repoDir).build()) { + Git git = new Git(repository); + StatusCommand statusCommand = git.status(); + // 仅关注当前diff改动涉及的文件夹 + for (String file : fileList) { + statusCommand.addPath(file); + } + + Status status = statusCommand.call(); + + logger.info("Modified files:"); + logger.info("Modified files: {} , \nUntracked files: {}, \nAdded to index: {}, \nChanged files: {}, \nRemoved files: {}, \nMissing files: {}, \nConflicting files: {} ", + status.getModified().toString(), + status.getUntracked().toString(), + status.getAdded().toString(), + status.getChanged().toString(), + status.getRemoved().toString(), + status.getMissing().toString(), + status.getConflicting().toString() + ); + + Set tree = new HashSet<>(); + tree.addAll(status.getModified()); + tree.addAll(status.getUntracked()); + tree.addAll(status.getAdded()); + tree.addAll(status.getChanged()); + tree.addAll(status.getRemoved()); + tree.addAll(status.getMissing()); + tree.addAll(status.getConflicting()); + + return tree; + } catch (IOException | GitAPIException e) { + throw new GitErrorException(80114, "git status failed, the reason is : ", e); + } + } + + public static void archive(String projectName, GitUserEntity gitUserDO) throws GitErrorException { + try { + String projectUrlEncoded = java.net.URLEncoder.encode(gitUserDO.getGitUser() + "/" + projectName, "UTF-8"); + URL url = new URL(gitUserDO.getGitUrl() + "/api/v4/projects/" + projectUrlEncoded + "/archive"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("PRIVATE-TOKEN", gitUserDO.getGitToken()); + + int responseCode = conn.getResponseCode(); + logger.info("Response Code: " + responseCode); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuilder response = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + + // 打印结果 + logger.info(response.toString()); + + } catch (Exception e) { + throw new GitErrorException(80115, "归档失败，请检查当前token是否过期 ", e); + } + } + + public static void archiveLocal(String projectName, Long workspaceId) throws GitErrorException{ + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName + File.separator + ".git"); + if (!repoDir.exists()) { + logger.info("file {} not exists", repoDir.getAbsolutePath()); + return ; + } + try (Repository repository = new FileRepositoryBuilder().setGitDir(repoDir).build()) { + Git git = new Git(repository); + // 删除名为"origin"的远程仓库配置 + git.remoteRemove().setRemoteName("origin").call(); + // 删除本地文件 + FileUtils.removeDirectory(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName); + logger.info("Remote 'origin' removed successfully."); + } catch (GitAPIException e) { + throw new GitErrorException(80116, "git archive failed, the reason is : ", e); + } catch (IOException e) { + throw new GitErrorException(80116, "archive failed, the reason is : ", e); + } + } + + public static String getTargetCommitFileContent(Repository repository, String projectName, String commitId, String filePath) throws GitErrorException { + String content = ""; + try { + // 获取最新的commitId + ObjectId lastCommitId = repository.resolve(commitId); + // 获取提交记录 + try (RevWalk revWalk = new RevWalk(repository)) { + RevCommit commit = revWalk.parseCommit(lastCommitId); + RevTree tree = commit.getTree(); + logger.info("Having tree: " + tree); + // 遍历获取最近提交记录 + try (TreeWalk treeWalk = new TreeWalk(repository)) { + treeWalk.addTree(tree); + treeWalk.setRecursive(true); + treeWalk.setFilter(PathFilter.create(filePath)); + if (!treeWalk.next()) { + throw new IllegalStateException("Did not find expected file '" + filePath + "'"); + } + + ObjectId objectId = treeWalk.getObjectId(0); + try { + ObjectLoader loader = repository.open(objectId); + byte[] bytes = loader.getBytes(); + content = new String(bytes); + logger.info("File content: " + content); + } catch (Exception e) { + logger.error("getFileContent Failed, the reason is: ", e); + } + } + revWalk.dispose(); + } + } catch (IOException e) { + throw new GitErrorException(80117, "getFileContent failed, the reason is : ", e); + } + return content; + } + + public static void getCommitId(Repository repository, String projectName, int num, Long workspaceId)throws GitErrorException { + // 获取当前CommitId， + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName + File.separator +".git"); + + try { + Git git = new Git(repository); + Iterable commits = git.log().setMaxCount(num).call(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + for (RevCommit commit : commits) { + PersonIdent authorIdent = commit.getAuthorIdent(); + String commitHash = commit.getName(); // Commit hash + String commitTime = sdf.format(authorIdent.getWhen()); // Commit time + String commitMessage = commit.getShortMessage(); // Commit message + String commitAuthor = authorIdent.getName(); // Commit author + + logger.info("Commit Hash: " + commitHash); + logger.info("Commit Time: " + commitTime); + logger.info("Commit Message: " + commitMessage); + logger.info("Commit Author: " + commitAuthor); + } + } catch (Exception e) { + throw new GitErrorException(80118, "git log failed, the reason is : ", e); + } + } + + public static GitCommitResponse getCurrentCommit(Repository repository) throws GitErrorException{ + GitCommitResponse commitResponse = new GitCommitResponse(); + try { + // 获取HEAD引用 + Ref head = repository.exactRef("HEAD"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + // 使用RevWalk解析当前的提交 + try (RevWalk walk = new RevWalk(repository)) { + RevCommit commit = walk.parseCommit(head.getObjectId()); + walk.dispose(); + commitResponse.setCommitId(commit.getId().getName()); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + + getUserName(shortMessage, commitResponse, commit); + // 返回commitId字符串 + return commitResponse; + } + } catch (IOException e) { + throw new GitErrorException(80119, "get current commit failed, the reason is : ", e); + } + } + + public static void getUserName(String shortMessage, GitCommitResponse commitResponse, RevCommit commit) { + if (StringUtils.isEmpty(shortMessage)) { + return ; + } + + int lastIndexOf = shortMessage.lastIndexOf(DSSGitConstant.GIT_USERNAME_FLAG); + + if (lastIndexOf != -1) { + String username = shortMessage.substring(lastIndexOf + DSSGitConstant.GIT_USERNAME_FLAG.length()); + String comment = shortMessage.substring(0, lastIndexOf); + commitResponse.setCommitUser(username); + commitResponse.setComment(comment); + } else { + commitResponse.setCommitUser(commit.getCommitterIdent().getName()); + commitResponse.setComment(shortMessage); + } + } + + public static List getLatestCommit(Repository repository, String filePath, Integer num) throws GitErrorException{ + List commitResponseList = new ArrayList<>(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + try (Git git = new Git(repository)) { + Iterable commits = null; + if (num == null) { + commits = git.log().addPath(filePath).call(); + } else { + commits = git.log().addPath(filePath).setMaxCount(num).call(); + } + for (RevCommit commit : commits) { + GitCommitResponse commitResponse = new GitCommitResponse(); + commitResponse.setCommitId(commit.getId().getName()); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + getUserName(shortMessage, commitResponse, commit); + logger.info("提交ID: " + commit.getId().getName()); + commitResponseList.add(commitResponse); + } + + return commitResponseList; + } catch (GitAPIException e) { + throw new GitErrorException(80120, "get latestCommitId failed, the reason is : ", e); + } + } + + public static GitHistoryResponse listCommitsBetween(Repository repository, String oldCommitId, String newCommitId, String path) throws Exception { + List gitCommitResponseList = new ArrayList<>(); + Set commitIdSet = new HashSet<>(); + + + try (RevWalk walk = new RevWalk(repository)) { + Git git = new Git(repository); + ObjectId commitIdNow = null; + if (newCommitId == null) { + ObjectId head = repository.resolve("HEAD"); + commitIdNow = walk.parseCommit(head); + } else { + commitIdNow = repository.resolve(newCommitId); + } + // 代码改动 + gitLogHistory(git, repository, oldCommitId, commitIdNow, path, gitCommitResponseList, commitIdSet); + // 元数据改动 + gitLogHistory(git, repository, oldCommitId, commitIdNow, GitConstant.GIT_SERVER_META_PATH + File.separator + path, gitCommitResponseList, commitIdSet); + } catch (Exception e) { + throw new GitErrorException(80121, "get log between " + oldCommitId + " and " + newCommitId + "failed, the reason is : ", e); + } + GitHistoryResponse historyResponse = new GitHistoryResponse(); + historyResponse.setResponses(gitCommitResponseList); + return historyResponse; + } + + private static void gitLogHistory(Git git, Repository repository, String oldCommitId, ObjectId commitIdNow, String path, List gitCommitResponseList, Set commitIdSet) throws IOException, GitAPIException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Iterable commits = git.log() + .addRange(repository.resolve(oldCommitId), commitIdNow) + .addPath(path) + .call(); + + for (RevCommit commit : commits) { + PersonIdent authorIdent = commit.getAuthorIdent(); // 获取提交人信息 + GitCommitResponse commitResponse = new GitCommitResponse(); + String commitId = commit.getId().getName(); + if (!commitIdSet.contains(commitId)) { + commitIdSet.add(commitId); + commitResponse.setCommitId(commitId); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + getUserName(shortMessage, commitResponse, commit); + gitCommitResponseList.add(commitResponse); + logger.info("Commit Hash: " + commit.getName()); // 提交的Hash值 + logger.info("Commit Time: " + authorIdent.getWhen()); // 提交时间 + logger.info("Commit Message: " + commit.getFullMessage()); // 提交信息 + logger.info("Author: " + authorIdent.getName() + " <" + authorIdent.getEmailAddress() + ">"); // 提交人 + } + } + } + + + + public static String generateGitPath(String projectName, Long workspaceId) { + // eg ： /data/GitInstall/224/testGit/.git + return DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + projectName + File.separator + DSSGitConstant.GIT_PATH_SUFFIX; + } + + + + + + + + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java new file mode 100644 index 000000000..770ef720c --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java @@ -0,0 +1,299 @@ +package com.webank.wedatasphere.dss.git.utils; + + +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.util.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(DSSGitUtils.class); + + public static void addFiles(String projectName, Long workspaceId) { + + String filePath = "/" + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + "/" + workspaceId + File.separator + projectName +"/file1.txt"; + List lines = Arrays.asList("The first line", "The second line"); + + try { + removeFiles(filePath); + + // 确保父目录存在 + Files.createDirectories(Paths.get(filePath).getParent()); + + // 向文件写入内容 + Files.write(Paths.get(filePath), lines, StandardOpenOption.CREATE); + } catch (Exception e) { + logger.error("add Files Failed, the reason is: ", e); + } + } + + public static void removeFiles (String filePath) { + try { + File file = new File(filePath); + if (file.exists()) { + file.delete(); + } + } catch (Exception e) { + logger.error("remove Files Failed, the reason is: ", e); + } + } + + public static void addDirectory (String directory, Long workspaceId) { + String filePath = "/" + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + workspaceId + File.separator + "/testGit1/file1.txt"; + List lines = Arrays.asList("The first line", "The second line"); + + try { + removeFiles(filePath); + + // 确保父目录存在 + Files.createDirectories(Paths.get(filePath).getParent()); + + // 向文件写入内容 + Files.write(Paths.get(filePath), lines, StandardOpenOption.CREATE); + } catch (Exception e) { + logger.error("add Files Failed, the reason is: ", e); + } + } + + public static void removeDirectory (String removeDirectoryPath) { + try { + File file = new File(removeDirectoryPath); + if (!file.exists()) { + logger.info("file {} not exist", removeDirectoryPath); + return; + } + Path dirToBeDeleted = Paths.get(removeDirectoryPath); + // 使用Files.walk收集所有路径，然后按照逆序排序，确保文件/子文件夹在其父文件夹之前被删除 + Files.walk(dirToBeDeleted) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.deleteIfExists(path); // 删除每个路径，如果存在的话 + logger.info("{} 删除成功", path); + } catch (IOException e) { + logger.info("无法删除路径: "+ path, e); + } + }); + logger.info("{} Directory and all its content deleted successfully.", removeDirectoryPath); + } catch (IOException e) { + logger.error("remove Directory Failed, the reason is: ", e); + } + } + + public static BmlResource uploadResourceToBML(BMLService bmlService, String userName, String content, String fileName, String projectName) { + Map bmlReturnMap = bmlService.upload(userName, content, fileName, projectName); + + BmlResource bmlResource = new BmlResource(); + bmlResource.setResourceId(bmlReturnMap.get("resourceId").toString()); + bmlResource.setVersion(bmlReturnMap.get("version").toString()); + + return bmlResource; + } + + public static String unzipFile(String zipFile) { + logger.info("-------=======================beginning to uznip testGit1=======================-------{}", zipFile); + + String longZipFilePath = ""; + try { + longZipFilePath = unzip(zipFile, true); + } catch (Exception e) { + logger.error("unzip failed, the reason is "); + } + return longZipFilePath; + } + + public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException { + File destFile = new File(destinationDir, zipEntry.getName()); + + String destDirPath = destinationDir.getCanonicalPath(); + String destFilePath = destFile.getCanonicalPath(); + + if (!destFilePath.startsWith(destDirPath + File.separator)) { + throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); + } + + return destFile; + } + + public static void removeFlowNode(String path, String projectName, Long workspaceId) { + // 删除node节点 + String flowNodePathPre = FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName ; + String flowNodePath = File.separator + FileUtils.normalizePath(flowNodePathPre) + File.separator + path; + String flowNodeMetaPath = File.separator + FileUtils.normalizePath(flowNodePathPre) + File.separator + FileUtils.normalizePath(GitConstant.GIT_SERVER_META_PATH) + File.separator + path; + // 1.删除工作流节点代码 + removeDirectory(flowNodePath); + // 2. 删除工作流节点对应.metaConf文件 + removeDirectory(flowNodeMetaPath); + } + + public static void removeProject(String path, Long workspaceId) { + // 删除node节点 + String projectPath = File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + path ; + // 1.删除项目 + removeDirectory(projectPath); + } + + public static void downloadBMLResource(BMLService bmlService, String path, BmlResource bmlResource, String username, Long workspaceId) { + //下载到本地处理 + String dirPath = "/" + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()); + String importFile = dirPath + File.separator + workspaceId + File.separator + path + ".zip"; + logger.info("import zip file locate at {}",importFile); + + try{ + // 确保父目录存在 + Files.createDirectories(Paths.get(importFile).getParent()); + //下载压缩包 + bmlService.downloadToLocalPath(username, bmlResource.getResourceId(), bmlResource.getVersion(), importFile); + }catch (Exception e){ + logger.error("download failed, the reason is :", e); + throw new DSSRuntimeException("upload file format error(导入包格式错误)"); + } + } + + public static void unzipBMLResource(String path, Long workspaceId) throws DSSErrorException { + //下载到本地处理 + String dirPath = DSSGitConstant.GIT_PATH_PRE + workspaceId ; + String importFile= dirPath + File.separator + path + ".zip"; + //解压 + ZipHelper.unzipFile(importFile, dirPath, true); + logger.info("import unzip file locate at {}",importFile); + } + + private static String readImportZipProjectName(String zipFilePath) throws IOException { + try(ZipFile zipFile =new ZipFile(zipFilePath)){ + Enumeration entries =zipFile.entries(); + if(entries.hasMoreElements()){ + String projectName=entries.nextElement().getName(); + while (projectName.endsWith(File.separator)){ + projectName = projectName.substring(0, projectName.length() - 1); + } + return projectName; + } + } + throw new IOException(); + } + + public static String normalizePath(String path) { + path = path.trim(); + while(path.startsWith(File.separator)) { + path = path.substring(File.separator.length()); + } + while(path.endsWith(File.separator)) { + path = path.substring(0, path.length()-File.separator.length()); + } + + return path; + } + + public static String unzip(String dirPath,boolean deleteOriginZip)throws DSSErrorException { + File file = new File(dirPath); + if(!file.exists()){ + logger.error("{} 不存在, 不能解压zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not unzip"); + } + //先用简单的方法，调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = new File(dirPath).getName(); + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add("unzip"); + list.add("-o"); + String longZipFilePath = dirPath.replace(".zip",""); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90007,errMsg.toString()); + } + if(deleteOriginZip){ + logger.info("开始删除目录 {}", file); + if (file.delete()){ + logger.info("结束删除目录 {} 成功", file); + }else{ + logger.info("删除目录 {} 失败", file); + } + } + }catch(final Exception e){ + logger.error( file + " 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + + logger.info("生成解压目录{}", longZipFilePath); + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + } + return longZipFilePath; + } + + public static void renameFile(String oldFileName, String fileName) { + File oldFile = new File(oldFileName); + File file = new File(fileName); + // 检查原文件夹是否存在 + if (oldFile.exists()) { + // 尝试重命名文件夹 + if (oldFile.renameTo(file)) { + logger.info("Folder renamed successfully."); + } else { + logger.error("Failed to rename folder."); + } + } else { + logger.error("Folder does not exist."); + } + } + + public static List getLocalProjectName(Long workspaceId) throws IOException { + String path = DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator; + Path dir = Paths.get(path); + List localProjectList = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path entry : stream) { + // 获取基本文件属性 + BasicFileAttributes attrs = Files.readAttributes(entry, BasicFileAttributes.class); + if (attrs.isDirectory()) { + localProjectList.add(entry.getFileName().toString()); + } + } + } catch (IOException | DirectoryIteratorException e) { + logger.error("get Local Project Failed", e); + throw e; + } + return localProjectList; + } +} diff --git a/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala new file mode 100644 index 000000000..3a0407a02 --- /dev/null +++ b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.git.receiver + +import com.webank.wedatasphere.dss.git.common.protocol.request.{GitArchiveProjectRequest, GitBaseRequest, GitCheckProjectRequest, GitCommitInfoBetweenRequest, GitCommitRequest, GitConnectRequest, GitCreateProjectRequest, GitCurrentCommitRequest, GitDeleteRequest, GitDiffRequest, GitFileContentRequest, GitHistoryRequest, GitRemoveRequest, GitRenameRequest, GitRevertRequest, GitSearchRequest, GitUserInfoByRequest, GitUserInfoRequest, GitUserUpdateRequest} +import com.webank.wedatasphere.dss.git.service.{DSSGitProjectManagerService, DSSGitWorkflowManagerService} + +import javax.annotation.PostConstruct +import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + + +@Component +class DSSGitChooser extends ReceiverChooser { + + @Autowired + var gitProjectManagerService: DSSGitProjectManagerService = _ + + @Autowired + var gitWorkflowManagerService: DSSGitWorkflowManagerService = _ + + var receiver: Option[DSSGitReceiver] = _ + + @PostConstruct + def init(): Unit = receiver = Some(new DSSGitReceiver(gitProjectManagerService, gitWorkflowManagerService)) + + override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { + case _: GitCreateProjectRequest => receiver + case _: GitArchiveProjectRequest => receiver + case _: GitCheckProjectRequest => receiver + case _: GitCommitRequest => receiver + case _: GitDiffRequest => receiver + case _: GitSearchRequest => receiver + case _: GitFileContentRequest => receiver + case _: GitDeleteRequest => receiver + case _: GitHistoryRequest => receiver + case _: GitCommitInfoBetweenRequest => receiver + case _: GitUserUpdateRequest => receiver + case _: GitUserInfoRequest => receiver + case _: GitCurrentCommitRequest => receiver + case _: GitRevertRequest => receiver + case _: GitRemoveRequest => receiver + case _: GitRenameRequest => receiver + case _: GitConnectRequest => receiver + case _: GitUserInfoByRequest => receiver + case _ => None + } +} \ No newline at end of file diff --git a/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala new file mode 100644 index 000000000..a300286cd --- /dev/null +++ b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.git.receiver + +import com.webank.wedatasphere.dss.git.common.protocol.request.{GitArchiveProjectRequest, GitCheckProjectRequest, GitCommitInfoBetweenRequest, GitCommitRequest, GitConnectRequest, GitCreateProjectRequest, GitCurrentCommitRequest, GitDeleteRequest, GitDiffRequest, GitFileContentRequest, GitHistoryRequest, GitRemoveRequest, GitRenameRequest, GitRevertRequest, GitSearchRequest, GitUserInfoByRequest, GitUserInfoRequest, GitUserUpdateRequest} +import com.webank.wedatasphere.dss.git.manage.GitProjectManager +import com.webank.wedatasphere.dss.git.service.{DSSGitProjectManagerService, DSSGitWorkflowManagerService} +import org.apache.linkis.rpc.{Receiver, Sender} +import org.slf4j.{Logger, LoggerFactory} +import org.springframework.stereotype.Component + +import java.util +import scala.concurrent.duration.Duration + +class DSSGitReceiver(gitProjectManagerService: DSSGitProjectManagerService, gitWorkflowManagerService: DSSGitWorkflowManagerService) extends Receiver { + + override def receive(message: Any, sender: Sender): Unit = {} + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + case gitCreateProjectRequest: GitCreateProjectRequest => + gitProjectManagerService.create(gitCreateProjectRequest) + case gitArchiveProjectRequest: GitArchiveProjectRequest => + gitProjectManagerService.archive(gitArchiveProjectRequest) + case gitCheckProjectRequest: GitCheckProjectRequest => + gitProjectManagerService.checkProject(gitCheckProjectRequest) + case gitDiffRequest: GitDiffRequest => + gitWorkflowManagerService.diff(gitDiffRequest) + case gitCommitRequest: GitCommitRequest => + gitWorkflowManagerService.commit(gitCommitRequest) + case gitSearchRequest: GitSearchRequest => + gitWorkflowManagerService.search(gitSearchRequest) + case gitDeleteRequest: GitDeleteRequest => + gitWorkflowManagerService.delete(gitDeleteRequest) + case gitFileContentRequest: GitFileContentRequest => + gitWorkflowManagerService.getFileContent(gitFileContentRequest) + case gitHistoryRequest: GitHistoryRequest => + gitWorkflowManagerService.getHistory(gitHistoryRequest) + case gitCommitInfoBetweenRequest: GitCommitInfoBetweenRequest => + gitWorkflowManagerService.getHistory(gitCommitInfoBetweenRequest) + case gitUserUpdateRequest: GitUserUpdateRequest => + GitProjectManager.associateGit(gitUserUpdateRequest) + case gitUserInfoRequest: GitUserInfoRequest => + GitProjectManager.selectGitUserInfo(gitUserInfoRequest) + case gitCurrentCommitRequest: GitCurrentCommitRequest => + gitWorkflowManagerService.getCurrentCommit(gitCurrentCommitRequest) + case gitRevertRequest: GitRevertRequest => + gitWorkflowManagerService.gitCheckOut(gitRevertRequest) + case gitRemoveRequest: GitRemoveRequest => + gitWorkflowManagerService.removeFile(gitRemoveRequest) + case gitRenameRequest: GitRenameRequest => + gitWorkflowManagerService.rename(gitRenameRequest) + case gitConnectTestRequest: GitConnectRequest => + GitProjectManager.gitTokenTest(gitConnectTestRequest) + case gitUserInfoByTypeRequest: GitUserInfoByRequest => + GitProjectManager.getGitUserByType(gitUserInfoByTypeRequest) + case _ => None + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} +} diff --git a/dss-git/pom.xml b/dss-git/pom.xml new file mode 100644 index 000000000..92a744c81 --- /dev/null +++ b/dss-git/pom.xml @@ -0,0 +1,19 @@ + + + + dss + com.webank.wedatasphere.dss + 1.5.0-SNAPSHOT + + 4.0.0 + + dss-git + pom + + dss-git-common + dss-git-server + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-common/pom.xml b/dss-orchestrator/dss-orchestrator-common/pom.xml index 10b97e09d..167fc88f3 100644 --- a/dss-orchestrator/dss-orchestrator-common/pom.xml +++ b/dss-orchestrator/dss-orchestrator-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java index 2035ae912..61776ecc7 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java @@ -86,6 +86,16 @@ public class DSSOrchestratorInfo implements DSSOrchestration { */ private Date updateTime; + /** + * 是否默认引用资源参数模板 + */ + private String isDefaultReference; + + /** + * 工作流状态：save-已保存 push-已提交 publish-已发布 + */ + private String status; + public DSSOrchestratorInfo() { } @@ -277,6 +287,14 @@ public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + @Override public String toString() { return "DSSOrchestratorInfo{" + @@ -293,6 +311,15 @@ public String toString() { ", secondaryType='" + secondaryType + '\'' + ", linkedAppConnNames=" + linkedAppConnNames + ", comment='" + comment + '\'' + + ", status='" + status + '\'' + '}'; } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } } diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java new file mode 100644 index 000000000..2d3275ddd --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + + +import java.util.List; + +public class DSSOrchestratorInfoList { + private List orchestratorInfos; + + public DSSOrchestratorInfoList(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } + + public DSSOrchestratorInfoList() { + } + + public List getOrchestratorInfos() { + return orchestratorInfos; + } + + public void setOrchestratorInfos(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java index 1175731d9..3c7b4c112 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java @@ -72,6 +72,10 @@ public class DSSOrchestratorVersion { * 有效标示 0:无效；1：有效，默认是有效 */ private Integer validFlag; + /** + * 项目发布当前版本的commitId + */ + private String commitId; public Long getId() { @@ -205,6 +209,15 @@ public String toString() { ", content='" + content + '\'' + ", contextId='" + contextId + '\'' + ", validFlag='" + validFlag + '\'' + + ", commitId='" + commitId + '\'' + '}'; } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } } diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java new file mode 100644 index 000000000..f6e6b29d6 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java @@ -0,0 +1,69 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.Date; + +public class OrchestratorSubmitJob { + private Long id; + private Long orchestratorId; + private Date createTime; + private Date updateTime; + private String instanceName; + private String status; + private String errorMsg; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getInstanceName() { + return instanceName; + } + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java new file mode 100644 index 000000000..24c197d6b --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +public class RequestQuertByAppIdOrchestrator { + private Long appId; + + public RequestQuertByAppIdOrchestrator(Long appId) { + this.appId = appId; + } + + public RequestQuertByAppIdOrchestrator() { + } + + public Long getAppId() { + return appId; + } + + public void setAppId(Long appId) { + this.appId = appId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java new file mode 100644 index 000000000..89eb86b6c --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + + +public class RequestSubmitOrchestratorSync { + private Long flowId; + private LabelRouteVO labels; + private String projectName; + private String comment; + private Long orchestratorId; + private String username; + private Workspace workspace; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java index 6504ef7c6..9cbac4344 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java @@ -7,6 +7,10 @@ */ public interface OrchestratorRefConstant { + String ORCHESTRATOR_ID_LIST_KEY = "orchestratorIdList"; + String ORCHESTRATOR_VERSION_ID_KEY = "orchestratorVersionId"; + String ORCHESTRATOR_ADD_VERSION_FLAG_KEY = "addOrcVersionFlag"; + String DSS_ORCHESTRATOR_INFO_KEY = "dssOrchestratorInfo"; /*************************************************************/ @@ -34,5 +38,19 @@ public interface OrchestratorRefConstant { //workflow node suffix input by the user during workflow replication. String ORCHESTRATION_NODE_SUFFIX = "nodeSuffix"; + /** + * 工作流状态：save-已发布 push-已提交 publish-已保存 + */ + public static final String FLOW_STATUS_SAVE = "save"; + + public static final String FLOW_STATUS_PUSHING = "running"; + + public static final String FLOW_STATUS_PUSH_FAILED = "failed"; + + public static final String FLOW_STATUS_PUSH_SUCCESS = "success"; + + public static final String FLOW_STATUS_PUSH = "push"; + + public static final String FLOW_STATUS_PUBLISH = "publish"; } diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml index 3335bfbee..00b9f19ff 100644 --- a/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml @@ -21,7 +21,7 @@ dss-orchestrator com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-core/pom.xml b/dss-orchestrator/dss-orchestrator-core/pom.xml index 7d2ffcb84..53e304cd2 100644 --- a/dss-orchestrator/dss-orchestrator-core/pom.xml +++ b/dss-orchestrator/dss-orchestrator-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-db/pom.xml b/dss-orchestrator/dss-orchestrator-db/pom.xml index fd75675ec..4c1550ccc 100644 --- a/dss-orchestrator/dss-orchestrator-db/pom.xml +++ b/dss-orchestrator/dss-orchestrator-db/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java index b95f7df7c..413e955b9 100644 --- a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java @@ -126,4 +126,11 @@ DSSOrchestratorVersion getOrcVersionByIdAndOrcVersionId(@Param("orchestratorId") List getHistoryOrcVersion(@Param("remainVersion") int remainVersion); void batchUpdateOrcInfo(@Param("list") List historyOrcVersion); + + void updateOrchestratorSubmitJobStatus(@Param("orchestratorId") Long orchestratorId, @Param("status") String status, @Param("errMsg") String errMsg); + + void insertOrchestratorSubmitJob(OrchestratorSubmitJob orchestratorSubmitJob); + + OrchestratorSubmitJob selectSubmitJobStatus(@Param("orchestratorId") Long orchestratorId); + } diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml index 77e8669fd..412cea964 100644 --- a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml @@ -22,11 +22,11 @@ id,`name`,`type`,`desc`,`creator`,`create_time`,`project_id`,`uses`,`appconn_name`,`uuid`,`secondary_type` , - `workspace_id`,`orchestrator_mode`,`orchestrator_way`,`orchestrator_level`,`update_user`,`update_time` + `workspace_id`,`orchestrator_mode`,`orchestrator_way`,`orchestrator_level`,`update_user`,`update_time`,`is_default_reference` - id,`orchestrator_id`,`app_id`,`source`,`version`,`comment`,`update_time`,`updater`,`project_id`,`content`,`context_id`,`valid_flag` + id,`orchestrator_id`,`app_id`,`source`,`version`,`comment`,`update_time`,`updater`,`project_id`,`content`,`context_id`,`valid_flag`, `commit_id` @@ -35,12 +35,16 @@ `orchestrator_level`=#{orchestratorLevel}, `update_user`=#{updateUser}, `update_time`=#{updateTime}, + `is_default_reference`=#{isDefaultReference}, WHERE id = #{id} @@ -80,7 +86,7 @@ parameterType="com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion"> INSERT INTO dss_orchestrator_version_info () VALUES - (#{id},#{orchestratorId},#{appId},#{source},#{version},#{comment},#{updateTime},#{updater},#{projectId},#{content},#{contextId},#{validFlag}) + (#{id},#{orchestratorId},#{appId},#{source},#{version},#{comment},#{updateTime},#{updater},#{projectId},#{content},#{contextId},#{validFlag}, #{commitId}) @@ -103,7 +109,7 @@ SELECT - * + FROM `dss_orchestrator_version_info` WHERE @@ -128,7 +134,7 @@ - SELECT * FROM dss_orchestrator_version_info + SELECT + , commit_id + FROM dss_orchestrator_version_info WHERE orchestrator_id = #{orchestratorId} and valid_flag = 1 and context_id != '' ORDER BY id DESC @@ -181,10 +189,13 @@ + @@ -200,7 +211,9 @@ @@ -213,7 +226,9 @@ - select * + select + from dss_orchestrator_ref_orchestration_relation where orchestrator_id = #{orchestratorId} @@ -274,4 +290,24 @@ #{item.id} + + + insert into dss_orchestrator_submit_job_info (`orchestrator_id`, `create_time`, `update_time`, `instance_name`, `status`) values + (#{orchestratorId}, Now(), Now(), #{instanceName}, #{status}) + + + + update dss_orchestrator_submit_job_info + + `error_msg` = #{errMsg} ,`update_time` = Now() , `status` = #{status} + + where `orchestrator_id` = #{orchestratorId} + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-loader/pom.xml b/dss-orchestrator/dss-orchestrator-loader/pom.xml index 1f4a05191..7fdf098e2 100644 --- a/dss-orchestrator/dss-orchestrator-loader/pom.xml +++ b/dss-orchestrator/dss-orchestrator-loader/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml index 1ac625b49..4449cf600 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala index be502890b..22962a037 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala @@ -17,12 +17,12 @@ package com.webank.wedatasphere.dss.flow.execution.entrance.execution import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} - import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils import org.apache.linkis.common.utils.Logging +import com.google.common.util.concurrent.ThreadFactoryBuilder import org.springframework.stereotype.Service import scala.collection.JavaConversions._ @@ -35,7 +35,8 @@ class DefaultFlowExecution extends FlowExecution with Logging { private val nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] = new LinkedBlockingQueue[NodeRunner]() - private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue) + private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue, + new ThreadFactoryBuilder().setNameFormat("Dss-Node-Execute-Status-Poll-Thread-%d").build) private var pollerCount = 0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala index f97475fb2..611abd395 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala @@ -171,7 +171,7 @@ class FlowEntranceJob(persistManager:PersistenceManager) extends EntranceExecuti } - - + override def clear(): Unit = { + } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala index 00a3d65a7..f7f482d34 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala @@ -40,6 +40,23 @@ import scala.collection.JavaConverters.asScalaBufferConverter object FlowExecutionUtils { def isSkippedNode(node: WorkflowNode, paramsMap: java.util.Map[String, Any]): Boolean = { + val skip = Option(node.getDSSNode) + .flatMap(node => Option(node.getParams)) + .flatMap(params => Option(params.get("configuration"))) + .flatMap { + case config: java.util.Map[_, _] => Option(config.get("special")) + case _ => None + } + .flatMap { + case special: java.util.Map[_, _] => Option(special.get("auto.disabled")) + case _ => None + } + .map(_.toString) + .exists("true".equalsIgnoreCase) + + if (skip) { + return true + } val executeStrategy = paramsMap.get("executeStrategy") if (executeStrategy != null) { return StrategyFactory.getNodeSkipStrategy(executeStrategy.toString).isSkippedNode(node, paramsMap, isReversedChoose = false) diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml index d88e327d6..9f9acfb9e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml @@ -23,7 +23,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../../pom.xml diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java index e4eb53887..e839001e3 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java @@ -139,7 +139,7 @@ public JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorEx //是否复用引擎，不复用就为空 - if(!isReuseEngine(job.getParams())){ + if(!isAppconnJob(job) && !isReuseEngine(job.getParams())){ labels.put("executeOnce", ""); } Map paramMapCopy = (HashMap) SerializationUtils.clone(new HashMap(job.getParams())); @@ -189,6 +189,13 @@ public boolean isReuseEngine(Map params) { return true; } + /** + * 是否为appconnjob + */ + private boolean isAppconnJob(Job job){ + return APPCONN.equals(job.getEngineType()); + } + /** * spark自定义参数配置输入，例如spark.sql.shuffle.partitions=10。多个参数使用分号分隔。 * @@ -210,6 +217,8 @@ private void replaceSparkConfParams(Map paramMapCopy) throws Lin startupMap.remove("spark.executor.instances"); startupMap.remove("wds.linkis.engineconn.java.driver.memory"); startupMap.remove("spark.conf"); + startupMap.remove("mapreduce.job.running.map.limit"); + startupMap.remove("mapreduce.job.running.reduce.limit"); logger.info("after remove startup map:{}",startupMap.keySet()); } Map configurationMap = TaskUtils.getMap(paramMapCopy, TaskConstant.PARAMS_CONFIGURATION); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml index e74991312..b4b2f9b91 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java index 53721e040..3745b9839 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java @@ -30,4 +30,6 @@ public interface NodeParser { String updateSubFlowID(String nodeJson, long subflowId) throws IOException; String getNodeValue(String key, String nodeJson) throws IOException; List getNodeResource(String nodeJson); + + Map getNodeJobContent(String nodeJson) throws IOException; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java new file mode 100644 index 000000000..e581e2b42 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import javax.servlet.http.Cookie; + +public class RequestLockWorkflow { + public RequestLockWorkflow() { + } + + private String username; + private Cookie[] cookies; + private Long flowId; + + public RequestLockWorkflow(String username, Cookie[] cookies, Long flowId) { + this.username = username; + this.cookies = cookies; + this.flowId = flowId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Cookie[] getCookies() { + return cookies; + } + + public void setCookies(Cookie[] cookies) { + this.cookies = cookies; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java index 492eb467b..bae910f39 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java @@ -1,16 +1,27 @@ package com.webank.wedatasphere.dss.workflow.common.protocol; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + public class RequestUnlockWorkflow { private String username; private Long flowId; private Boolean confirmDelete; + private Workspace workspace; + public RequestUnlockWorkflow(String username, Long flowId, boolean confirmDelete) { this.username = username; this.flowId = flowId; this.confirmDelete = confirmDelete; } + public RequestUnlockWorkflow(String username, Long flowId, Boolean confirmDelete, Workspace workspace) { + this.username = username; + this.flowId = flowId; + this.confirmDelete = confirmDelete; + this.workspace = workspace; + } + public String getUsername() { return username; } @@ -34,4 +45,12 @@ public Boolean getConfirmDelete() { public void setConfirmDelete(Boolean confirmDelete) { this.confirmDelete = confirmDelete; } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java new file mode 100644 index 000000000..6b951aa17 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +public class ResponseLockWorkflow { + public static final int LOCK_SUCCESS = 0; + public static final int LOCK_FAILED = 1; + + private int unlockStatus; + private String lockOwner; + + public ResponseLockWorkflow(int unlockStatus, String lockOwner) { + this.unlockStatus = unlockStatus; + this.lockOwner = lockOwner; + } + + public ResponseLockWorkflow() { + } + + public int getUnlockStatus() { + return unlockStatus; + } + + public void setUnlockStatus(int unlockStatus) { + this.unlockStatus = unlockStatus; + } + + public String getLockOwner() { + return lockOwner; + } + + public void setLockOwner(String lockOwner) { + this.lockOwner = lockOwner; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml index fb2f99ce6..7c7b7f806 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml @@ -21,7 +21,7 @@ dss-workflow com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml index 7bf055d48..d39926163 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml @@ -21,7 +21,7 @@ dss-workflow com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml index fd970e1d4..d9968c4e9 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml @@ -21,13 +21,25 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../../pom.xml 4.0.0 dss-workflow-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + provided + com.webank.wedatasphere.dss dss-common diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java index 29b649069..726e82831 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java @@ -43,9 +43,7 @@ import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; -import com.webank.wedatasphere.dss.workflow.common.protocol.RequestSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseUnlockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.*; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; @@ -68,15 +66,15 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import javax.servlet.http.Cookie; import java.io.IOException; import java.io.InputStream; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import static com.webank.wedatasphere.dss.common.utils.IoUtils.FLOW_META_DIRECTORY_NAME; import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; @Component @@ -190,17 +188,27 @@ public void deleteWorkflow(String userName, Long flowId) throws DSSErrorExceptio } @Override - public ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete) throws DSSErrorException { + public ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete, Workspace workspace) throws DSSErrorException { DSSFlowEditLock editLock = lockMapper.getFlowEditLockByID(flowId); if (editLock == null) { return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.NONEED_UNLOCK, null); } else if (!Boolean.TRUE.equals(confirmDelete)) { return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.NEED_SECOND_CONFIRM, editLock.getUsername()); } - DSSFlowEditLockManager.deleteLock(editLock.getLockContent()); + DSSFlowEditLockManager.deleteLock(editLock.getLockContent(), workspace); return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.UNLOCK_SUCCESS, null); } - + @Override + public BmlResource exportWorkflowNew(String userName, Long flowId, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + DSSFlow dssFlow = flowService.getFlowByID(flowId); + String exportPath = workFlowExportService.exportFlowInfoNew(dssProjectId, projectName, flowId, userName, workspace, dssLabels,exportExternalNodeAppConnResource); + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + BmlResource bmlResource = bmlService.upload(userName, inputStream, dssFlow.getName() + ".export", projectName); + logger.info("export workflow success. flowId:{},bmlResource:{} .",flowId,bmlResource); + return bmlResource; + } @Override public BmlResource exportWorkflow(String userName, Long flowId, Long dssProjectId, String projectName, Workspace workspace, @@ -212,6 +220,59 @@ public BmlResource exportWorkflow(String userName, Long flowId, Long dssProjectI logger.info("export workflow success. flowId:{},bmlResource:{} .",flowId,bmlResource); return bmlResource; } + @Override + public List importWorkflowNew(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws DSSErrorException, IOException { + + //todo download workflow bml file contains flowInfo and flowRelationInfo + String projectName = dssFlowImportParam.getProjectName(); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu + String tempPath = IoUtils.generateTempIOPath(userName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx.zip + String inputZipPath = IoUtils.addFileSeparator(tempPath, projectName + ".zip"); + bmlService.downloadToLocalPath(userName, resourceId, bmlVersion, inputZipPath); + try{ + String originProjectName=readImportZipProjectName(inputZipPath); + if(!projectName.equals(originProjectName)){ + String msg=String.format("target project name must be same with origin project name.origin project name:%s,target project name:%s(导入的目标工程名必须与导出时源工程名保持一致。源工程名：%s，目标工程名：%s)" + ,originProjectName,projectName,originProjectName,projectName); + throw new DSSRuntimeException(msg); + } + }catch (IOException e){ + throw new DSSRuntimeException("upload file format error(导入包格式错误)"); + } + String projectPath = ZipHelper.unzip(inputZipPath,true); + String flowName = IoUtils.getSubdirectoriesNames(projectPath).stream().filter(name -> !name.startsWith(".")) + .findFirst().orElseThrow(() -> new DSSRuntimeException("import package has no flow（未导入任何工作流，请检查导入包格式）")); + String flowMetaPath=IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME, flowName); + ImmutablePair,List> meta= metaInputService.inputFlowNew(flowMetaPath); + //导入工作流数据库信息 + List dssFlows = meta.getKey(); + //导入工作流关系信息 + List dwsFlowRelations = meta.getValue(); + + List dwsFlowRelationList = workFlowInputService.persistenceFlow(dssFlowImportParam.getProjectID(), + dssFlowImportParam.getUserName(), + dssFlows, + dwsFlowRelations); + //这里其实只会有1个元素 + List rootFlows = dwsFlowRelationList.stream().filter(DSSFlow::getRootFlow).collect(Collectors.toList()); + for (DSSFlow rootFlow : rootFlows) { + String flowCodePath0=IoUtils.addFileSeparator(projectPath, flowName); + String flowMetaPath0=IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME, flowName); + workFlowInputService.inputWorkFlowNew(dssFlowImportParam.getUserName(), + rootFlow, + projectName, + flowCodePath0, + flowMetaPath0,null, dssFlowImportParam.getWorkspace(), dssFlowImportParam.getOrcVersion(), + dssFlowImportParam.getContextId(), dssLabels); + } + logger.info("import workflow success.orcVersion:{},context Id:{}", dssFlowImportParam.getOrcVersion(), dssFlowImportParam.getContextId()); + return rootFlows; + } @Override public List importWorkflow(String userName, @@ -358,11 +419,11 @@ private ResponseOperateOrchestrator convert(RequestConvertOrchestrations request } private String readImportZipProjectName(String zipFilePath) throws IOException { - try(ZipFile zipFile =new ZipFile(zipFilePath)){ - Enumeration entries =zipFile.entries(); - if(entries.hasMoreElements()){ - String name=entries.nextElement().getName(); - if(name.endsWith("\\")||name.endsWith("/")){ + try (ZipFile zipFile = new ZipFile(zipFilePath)) { + Enumeration entries = zipFile.entries(); + if (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (name.endsWith("\\") || name.endsWith("/")) { name = name.substring(0, name.length() - 1); } return name; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java index 281e3405a..89c4cd90d 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java @@ -25,15 +25,12 @@ import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; -import com.webank.wedatasphere.dss.workflow.common.protocol.RequestSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseUnlockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.*; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; import org.apache.linkis.common.exception.ErrorException; import java.io.IOException; import java.util.List; -import java.util.Map; public interface WorkFlowManager { @@ -95,8 +92,26 @@ void updateWorkflow(String userName, void deleteWorkflow(String userName, Long flowID) throws DSSErrorException; - ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete) throws DSSErrorException; - + ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete, Workspace workspace) throws DSSErrorException; + /** + * 导出工作流 + * @param userName + * @param flowID + * @param dssProjectId + * @param projectName + * @param workspace + * @param dssLabels + * @param exportExternalNodeAppConnResource 是否导出第三方节点的物料 + * @return 导出的工作流，以Bml资源的形式返回 + * @throws Exception + */ + BmlResource exportWorkflowNew(String userName, + Long flowID, + Long dssProjectId, + String projectName, + Workspace workspace, + List dssLabels, + boolean exportExternalNodeAppConnResource) throws Exception; /** * 导出工作流 * @param userName @@ -115,6 +130,11 @@ BmlResource exportWorkflow(String userName, Workspace workspace, List dssLabels) throws Exception; + List importWorkflowNew(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws Exception; List importWorkflow(String userName, String resourceId, String bmlVersion, @@ -139,4 +159,5 @@ List importWorkflow(String userName, */ ResponseSubFlowContextIds getSubFlowContextIdsByFlowIds(RequestSubFlowContextIds requestSubFlowContextIds) throws ErrorException; + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java index d85658d7e..a0c2c8a34 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java @@ -60,4 +60,5 @@ public class DSSWorkFlowConstant { * 仅仅用于兼容老的、已经创建的工作量，用于自动路由到一个默认的调度系统。 */ public static final CommonVars DEFAULT_SCHEDULER_APP_CONN = CommonVars.apply("wds.dss.workflow.scheduler.default", "schedulis"); + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java index c2f8ab5e3..4cc11f5a1 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java @@ -19,6 +19,8 @@ import org.apache.ibatis.annotations.Param; import org.springframework.dao.DuplicateKeyException; +import java.util.List; + public interface LockMapper { @@ -48,4 +50,18 @@ Boolean flowNotExistEditLock(@Param("flowID") Long flowID, void clearExpire(@Param("expireTime") String expireTime, @Param("flowId") long flowId); void deleteALL(); + + + void deleteExpectAfterSave(@Param("list") List list); + + + void insertFlowStatus(@Param("flowID") Long flowID, @Param("status") String status); + + void updateOrchestratorStatus(@Param("id") Long id, @Param("status") String status); + + List selectOrchestratorByStatus( @Param("status") String status); + + String selectOrchestratorStatus(@Param("id") Long id); + + void updateOrchestratorVersionCommitId(@Param("commitId")String commitId, @Param("flowID")Long flowID); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml index 41e45c40c..727424854 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml @@ -73,4 +73,42 @@ delete from dss_workflow_edit_lock + + + delete from dss_workflow_edit_lock where flow_id not in + + #{index} + + + + + INSERT INTO dss_workflow_status (`flow_id`, `status`, `create_time`, `update_time`) + VALUES + (#{flowID}, #{status}, NOW(), NOW()) + + + + update dss_orchestrator_info + + `status` = #{status} + + where `id` = #{id} + + + + + + + + update dss_orchestrator_version_info + + `commit_id` = #{commitId} + + where `app_id` = #{flowID} + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java index 87216241d..2181f14bb 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.workflow.io.export; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; @@ -26,5 +27,6 @@ public interface MetaExportService { void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException; + void exportFlowBaseInfoNew(OrchestratorVo orchestratorVo, List allDSSFlows, List allFlowRelations, String savePath) throws IOException; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java index 8aa063b23..f472e96df 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java @@ -25,4 +25,6 @@ public interface NodeExportService { void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath); void downloadAppConnResourceToLocal(String userName, Long projectId, String projectName, DSSNode dwsNode, String savePath, Workspace workspace, List dssLabels) throws Exception; + void downloadNodeResourceToLocalNew(String userName, DSSNode dwsNode, String flowCodePath); + void downloadAppConnResourceToLocalNew(String userName, Long projectId, String projectName, DSSNode dwsNode, String flowCodePath, Workspace workspace, List dssLabels) throws Exception; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java index 55eef0064..e7047f6c1 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java @@ -22,6 +22,21 @@ import java.util.List; public interface WorkFlowExportService { + /** + * 将一个工作流导出成压缩包，存放到本地磁盘，并返回压缩包的路径 + * @param dssProjectId 所属工作空间id + * @param projectName 所属项目名称 + * @param rootFlowId 工作流根节点id + * @param userName 用户名 + * @param workspace 所属工作空间 + * @param dssLabels label列表 + * @param exportExternalNodeAppConnResource 是否导出第三方节点的物料 + * @return 导出的工作流压缩包的文件地址 + * @throws Exception + */ + String exportFlowInfoNew(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception; + /** * 将一个工作流导出成压缩包，存放到本地磁盘，并返回压缩包的路径 * @param dssProjectId 所属工作空间id diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java index 263c2d05d..019ea3ec0 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java @@ -16,26 +16,51 @@ package com.webank.wedatasphere.dss.workflow.io.export.impl; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.io.export.MetaExportService; import com.webank.wedatasphere.dss.workflow.io.export.MetaWriter; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.*; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Service public class MetaExportServiceImpl implements MetaExportService { private Logger logger = LoggerFactory.getLogger(this.getClass()); - + public static final String FLOW_META_FILE_NAME = ".flowmeta"; + public static final String FLOW_META_KEY = "dss_flow"; + public static final String FLOW_RELATION_META_KEY = "dss_flow_relation"; + public static final String ORCHESTRATOR_META_KEY = "dss_orchestrator"; + public static final String ORCHESTRATOR_VERSION_META_KEY = "dss_orchestrator_version"; private final String fileName = "meta.txt"; + @Override + public void exportFlowBaseInfoNew(OrchestratorVo orchestratorVo, List allDSSFlows, List allFlowRelations, String savePath) throws IOException { + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(savePath + File.separator + FLOW_META_FILE_NAME) + ) { + Map flowMetaMap = new HashMap<>(2); + flowMetaMap.put(FLOW_META_KEY, allDSSFlows); + flowMetaMap.put(FLOW_RELATION_META_KEY, allFlowRelations); + flowMetaMap.put(ORCHESTRATOR_META_KEY, orchestratorVo.getDssOrchestratorInfo()); + flowMetaMap.put(ORCHESTRATOR_VERSION_META_KEY, orchestratorVo.getDssOrchestratorVersion()); + String flowMetaStr = new GsonBuilder().setPrettyPrinting().create().toJson(flowMetaMap); + org.apache.commons.io.IOUtils.write(flowMetaStr,outputStream,"UTF-8"); + } + } @Override public void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java index 0e4aca71e..bc9ed04e5 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java @@ -40,11 +40,13 @@ import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.Optional; @Service public class NodeExportServiceImpl implements NodeExportService { + public static final String APPCONN_FILE_NAME = ".appconnre"; private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -55,6 +57,61 @@ public class NodeExportServiceImpl implements NodeExportService { @Autowired private WorkflowNodeService workflowNodeService; + @Override + public void downloadNodeResourceToLocalNew(String userName, DSSNode dwsNode, String flowCodePath) { + List resources = dwsNode.getResources(); + String scriptName = Optional.ofNullable(dwsNode.getJobContent()).map(jobContent -> jobContent.get("script")) + .map(Object::toString).orElse(null); + if (resources != null) { + resources.forEach(x -> { + // TODO: 2020/6/9 防止前台传的 resources：{{}],后期要去掉 + if (x.getResourceId() != null && x.getFileName() != null && x.getVersion() != null) { + String nodePath = flowCodePath + File.separator + dwsNode.getName(); + // 如果是scriptis节点，且资源名和jobContent中的script相同，则下载后的文件名为script名的后缀，如.sql。 否则说明是用户自定义文件，直接用filename + String resourceName = x.getFileName(); + if(resourceName.equals(scriptName)){ + resourceName = resourceName.substring(resourceName.lastIndexOf('.')); + } + String nodeResourcePath = nodePath + File.separator + resourceName; + bmlService.downloadToLocalPath(userName, x.getResourceId(), x.getVersion(), nodeResourcePath); + } else { + LOGGER.warn("Illegal resource information"); + LOGGER.warn("username:{},nodeId:{},nodeName:{},fileName:{},version:{},resourceId:{}", userName, dwsNode.getId(), dwsNode.getName(), x.getFileName(), x.getVersion(), x.getResourceId()); + } + }); + } + } + + @Override + public void downloadAppConnResourceToLocalNew(String userName, Long projectId, String projectName, DSSNode dwsNode, + String flowCodePath, Workspace workspace, List dssLabels) throws Exception { + CommonAppConnNode node = new CommonAppConnNode(); + node.setJobContent(dwsNode.getJobContent()); + node.setProjectId(projectId); + node.setWorkspace(workspace); + node.setDssLabels(dssLabels); + node.setNodeType(dwsNode.getNodeType()); + node.setName(dwsNode.getName()); + node.setId(dwsNode.getId()); + ExportResponseRef responseRef = workflowNodeService.exportNode(userName, node); + Map resourceMap = responseRef.getResourceMap(); + String nodePath = flowCodePath + File.separator + dwsNode.getName(); + String nodeResourcePath = nodePath + File.separator + APPCONN_FILE_NAME; + if(responseRef.isLinkisBMLResources()) { + String resourceId = resourceMap.get(ImportRequestRef.RESOURCE_ID_KEY).toString(); + String version = resourceMap.get(ImportRequestRef.RESOURCE_VERSION_KEY).toString(); + bmlService.downloadToLocalPath(userName, resourceId, version, nodeResourcePath); + } else { + InputStream inputStream = (InputStream) resourceMap.get(ImportRequestRef.INPUT_STREAM_KEY); + Closeable closeable = (Closeable) resourceMap.get(ImportRequestRef.CLOSEABLE_KEY); + try { + FileUtils.copyInputStreamToFile(inputStream, new File(nodeResourcePath)); + } finally { + IOUtils.closeQuietly(inputStream); + IOUtils.closeQuietly(closeable); + } + } + } @Override public void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath) { List resources = dwsNode.getResources(); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java index 7417a682f..b6b503233 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java @@ -17,16 +17,27 @@ package com.webank.wedatasphere.dss.workflow.io.export.impl; +import com.google.gson.*; import com.webank.wedatasphere.dss.common.entity.IOType; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; import com.webank.wedatasphere.dss.common.entity.node.DSSNode; import com.webank.wedatasphere.dss.common.entity.node.Node; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQuertByAppIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQueryByIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; @@ -42,6 +53,7 @@ import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -49,6 +61,8 @@ import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; +import java.io.OutputStream; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.FutureTask; @@ -64,6 +78,9 @@ public class WorkFlowExportServiceImpl implements WorkFlowExportService { private Logger logger = LoggerFactory.getLogger(this.getClass()); + public static final String FLOW_FILE_NAME = ".flow"; + public static final String NODE_PARAMS_FILE_NAME = ".properties"; + @Autowired @Qualifier("workflowBmlService") private BMLService bmlService; @@ -87,6 +104,63 @@ public class WorkFlowExportServiceImpl implements WorkFlowExportService { private DSSFlowService flowService; + + @Override + public String exportFlowInfoNew(Long dssProjectId, String projectName, long rootFlowId, String userName, + Workspace workspace, List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + //获取rootFlow,和所有子Flow + DSSFlow rootFlow = flowMapper.selectFlowByID(rootFlowId); + List dssFlowList = new ArrayList<>(); + //生成rootflow及所有子flow + dssFlowList.add(rootFlow); + getAllDssFlowsByRootflowId(rootFlow, dssFlowList); + //生成rootflow及所有子flow的Relations + List flowIds = dssFlowList.stream().map(DSSFlow::getId).collect(Collectors.toList()); + List flowRelations = flowIds.isEmpty() ? new ArrayList<>() : flowMapper.listFlowRelation(flowIds); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx + String projectPath = IoUtils.generateProjectIOPath(userName, projectName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/.flowmeta/flow_all_type_node/ + List dssFlows = new ArrayList<>(); + for (DSSFlow dssFlow : dssFlowList) { + if (dssFlow.getRootFlow()) { + // 生成rootflow orchestrator信息 + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabels); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(dssFlow.getId())), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String flowMetaPath = IoUtils.generateFlowMetaIOPath(projectPath, dssOrchestratorInfo.getName()); + metaExportService.exportFlowBaseInfoNew(orchestratorVo, dssFlowList, flowRelations, flowMetaPath); + logger.info(userName + "-开始导出Flow：" + dssOrchestratorInfo.getName()); + String flowMetaFilePath = IoUtils.addFileSeparator(flowMetaPath, FLOW_FILE_NAME); + //导出工作流json文件 + String flowJson = bmlService.readTextFromBML(userName, dssFlow.getResourceId(), dssFlow.getBmlVersion()); + if (!dssFlow.getHasSaved()) { + logger.info("工作流{}从未保存过，忽略", dssOrchestratorInfo.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/flow_all_type_node/ + String flowCodePath = IoUtils.generateFlowCodeIOPath(projectPath, dssOrchestratorInfo.getName()); + exportFlowResourcesNew(userName, dssProjectId, projectName, flowCodePath, flowJson, dssOrchestratorInfo.getName(), workspace, dssLabels,exportExternalNodeAppConnResource); + exportAllSubFlowsNew(userName, dssFlow, dssProjectId, projectName, flowCodePath,flowMetaPath, workspace, dssLabels,exportExternalNodeAppConnResource); + String flowJsonWithoutParams = extractAndExportParams(flowJson, flowCodePath); + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(flowMetaFilePath ) + ) { + org.apache.commons.io.IOUtils.write(flowJsonWithoutParams,outputStream,"UTF-8"); + } + dssFlows.add(dssFlow); + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90033, warnMsg); + } + } + } + if (dssFlows.isEmpty()) { + throw new DSSErrorException(90037, "该工程没有可以导出的工作流,请检查工作流是否都为空"); + } + //打包导出工程 + return ZipHelper.zip(projectPath); + } @Override public String exportFlowInfo(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, List dssLabels) throws Exception { //获取rootFlow,和所有子Flow @@ -132,7 +206,39 @@ public String exportFlowInfo(Long dssProjectId, String projectName, long rootFlo return ZipHelper.zip(flowExportSaveBasePath); } + private void exportAllSubFlowsNew(String userName, DSSFlow dssFlowParent, Long projectId, String projectName, + String parentFlowCodePath, String parentFlowMetaPath, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + List subFlows = dssFlowParent.getChildren(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + String subFlowCodePath = parentFlowCodePath + File.separator + subFlow.getName(); + String subFlowMetaPath = parentFlowMetaPath + File.separator + subFlow.getName(); + + String subFlowMetaSavePath = subFlowMetaPath+ File.separator + FLOW_FILE_NAME; + //导出子flow的json文件 + String flowJson = bmlService.readTextFromBML(userName, subFlow.getResourceId(), subFlow.getBmlVersion()); + if (!subFlow.getHasSaved()) { + logger.info("工作流{}从未保存过，忽略", subFlow.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + exportFlowResourcesNew(userName, projectId, projectName, subFlowCodePath, flowJson, subFlow.getName(), workspace, dssLabels,exportExternalNodeAppConnResource); + exportAllSubFlowsNew(userName, subFlow, projectId, projectName, subFlowCodePath, subFlowMetaPath, workspace, dssLabels,exportExternalNodeAppConnResource); + String subFlowWithoutParams = extractAndExportParams(flowJson, subFlowCodePath); + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(subFlowMetaSavePath ) + ) { + org.apache.commons.io.IOUtils.write(subFlowWithoutParams,outputStream,"UTF-8"); + } + + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, subFlow.getName(), subFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90014, warnMsg); + } + } + } + } private void exportAllSubFlows(String userName, DSSFlow dssFlowParent, Long projectId, String projectName, String projectExportBasePath, Workspace workspace, List dssLabels) throws Exception { List subFlows = dssFlowParent.getChildren(); @@ -159,7 +265,84 @@ private void exportAllSubFlows(String userName, DSSFlow dssFlowParent, Long proj private String genWorkFlowExportDir(String projectExportPath, String flowName) { return projectExportPath + File.separator + flowName; } + /** + * 导出工作流中的各种资源，放到flowCodePath中。 + * 工作流中的资源包括工作流资源和节点资源 + * @param flowCodePath 保存工作流代码的目录 + * @param flowJson 工作流元信息 + * @param flowName 工作流明 + * @param dssLabels label列表 + * @param exportExternalNodeAppConnResource 是否导出第三方节点的appconn资源 + * @throws Exception + */ + private void exportFlowResourcesNew(String userName, Long projectId, String projectName, + String flowCodePath, String flowJson, String flowName, + Workspace workspace, List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + if (StringUtils.isNotEmpty(flowCodePath)) { + //导出工作流资源文件 + List resources = workFlowParser.getWorkFlowResources(flowJson); + if (resources != null) { + resources.forEach(resource -> { + downloadFlowResourceFromBmlNew(userName, resource, flowCodePath); + }); + } + + //导出工作流节点资源文件,工作流节点appconn文件 + List nodes = workFlowParser.getWorkFlowNodes(flowJson); + if (nodes != null) { + for (DSSNode node : nodes) { + nodeExportService.downloadNodeResourceToLocalNew(userName, node, flowCodePath); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); + if(nodeInfo==null){ + String msg = String.format("%s note type not exist,please check appconn install successfully", node.getNodeType()); + logger.error(msg); + throw new DSSRuntimeException(msg); + } + if (exportExternalNodeAppConnResource&&Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("node.getJobContent() is :{}", node.getJobContent()); + nodeExportService.downloadAppConnResourceToLocalNew(userName, projectId, projectName, node, flowCodePath, workspace, dssLabels); + } + } + } + + } else { + throw new DSSErrorException(90067, "工作流导出生成路径为空"); + } + } + + /** + * 从flowjson中分离各个节点的params参数，以及flow本身的参数，把参数写到磁盘，并返回分离后的flow + * @param flowJson 要分离参数的flow + * @param flowCodePath 工作流代码导出目录 + * @return 分离后的flow + */ + private String extractAndExportParams(String flowJson,String flowCodePath) throws IOException { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(flowJson).getAsJsonObject(); + JsonArray nodeJsonArray = jsonObject.getAsJsonArray("nodes"); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + if (nodeJsonArray == null) { + return gson.toJson(jsonObject); + } + for (JsonElement element : nodeJsonArray) { + JsonObject node = element.getAsJsonObject(); + JsonElement params = node.remove("params"); + String nodeName = Optional.ofNullable(node.get("title")).map(JsonElement::getAsString).orElse(null); + String nodeParamsPath = IoUtils.addFileSeparator(flowCodePath, nodeName, NODE_PARAMS_FILE_NAME); + String paramsJson = gson.toJson(params); + try ( + + OutputStream outputStream = IoUtils.generateExportOutputStream(nodeParamsPath ) + ) { + + org.apache.commons.io.IOUtils.write(paramsJson,outputStream,"UTF-8"); + } + + } + return gson.toJson(jsonObject); + + } @Override public void exportFlowResources(String userName, Long projectId, String projectName, String projectSavePath, String flowJson, String flowName, @@ -300,7 +483,10 @@ public void exportFlowResources_for_multi_thread(String userName, Long projectId public String downloadFlowJsonFromBml(String userName, String resourceId, String version, String savePath) { return bmlService.downloadAndGetText(userName, resourceId, version, savePath); } - + private String downloadFlowResourceFromBmlNew(String userName, Resource resource, String flowCodePath) { + String flowResourcePath = IoUtils.addFileSeparator(flowCodePath,resource.getFileName()); + return bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); + } private String downloadFlowResourceFromBml(String userName, Resource resource, String savePath) { String flowResourcePath = savePath + File.separator + resource.getResourceId() + ".re"; return bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java index 506de99e4..114052642 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java @@ -19,6 +19,7 @@ import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import org.apache.linkis.protocol.util.ImmutablePair; import java.io.IOException; import java.util.List; @@ -26,6 +27,7 @@ public interface MetaInputService { List inputFlow(String basePath) throws IOException; + ImmutablePair,List> inputFlowNew(String flowMetaPath) throws IOException; List inputFlowRelation(String basePath) throws IOException; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java index e01c1f092..108de00b3 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java @@ -29,9 +29,14 @@ public interface NodeInputService { String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException; + String uploadResourceToBmlNew(String userName, String nodeJson, String nodePath, String projectName) throws IOException; + String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, String nodeJson, String flowContextId, String appConnResourcePath, Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException,IOException; + String uploadAppConnResourceNew(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String nodePath, + Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException,IOException; String updateNodeSubflowID(String nodeJson, long subflowID) throws IOException; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java index b954d8d28..0b7ef1470 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java @@ -28,6 +28,10 @@ public interface WorkFlowInputService { + void inputWorkFlowNew(String userName, DSSFlow dssFlow, String projectName, + String flowCodePath, String flowMetaPath, Long parentFlowId, Workspace workspace, + String orcVersion, String contextId, List dssLabels) throws DSSErrorException,IOException; + /** * * @param userName diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java index 0d181ebdc..f0971bed8 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java @@ -16,19 +16,24 @@ package com.webank.wedatasphere.dss.workflow.io.input.impl; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.io.input.MetaInputService; import com.webank.wedatasphere.dss.workflow.io.input.MetaReader; +import org.apache.linkis.protocol.util.ImmutablePair; import org.springframework.stereotype.Service; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.lang.reflect.Type; import java.util.List; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.MetaExportServiceImpl.*; + @Service public class MetaInputServiceImpl implements MetaInputService { @@ -37,6 +42,20 @@ public class MetaInputServiceImpl implements MetaInputService { private final String fileName = "meta.txt"; + @Override + public ImmutablePair,List> inputFlowNew(String flowMetaPath) throws IOException { + File flowMetaFile = new File(flowMetaPath + File.separator + FLOW_META_FILE_NAME); + JsonParser jsonParser = new JsonParser(); + Gson gson = new Gson(); + try (FileReader reader = new FileReader(flowMetaFile)) { + Type flowListType = new TypeToken>() {}.getType(); + Type flowRelationListType = new TypeToken>() {}.getType(); + JsonObject jsonObject = jsonParser.parse(reader).getAsJsonObject(); + List flows = gson.fromJson(jsonObject.get(FLOW_META_KEY), flowListType); + List flowRelations = gson.fromJson(jsonObject.get(FLOW_RELATION_META_KEY), flowRelationListType); + return new ImmutablePair<>(flows, flowRelations); + } + } @Override public List inputFlow(String basePath) throws IOException { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java index df2869718..27d8c8a1e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java @@ -22,6 +22,7 @@ import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -47,6 +48,8 @@ import java.util.*; import java.util.function.Supplier; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.NodeExportServiceImpl.APPCONN_FILE_NAME; + @Service public class NodeInputServiceImpl implements NodeInputService { @Autowired @@ -63,7 +66,33 @@ public class NodeInputServiceImpl implements NodeInputService { private final Logger logger = LoggerFactory.getLogger(this.getClass()); - + @Override + public String uploadResourceToBmlNew(String userName, String nodeJson, String nodePath, String projectName) throws IOException { + List resources = nodeParser.getNodeResource(nodeJson); + Map jobContent = nodeParser.getNodeJobContent(nodeJson); + String scriptName = Optional.ofNullable(jobContent).map(e->e.get("script")).map(Object::toString).orElse(null); + if (resources != null && resources.size() > 0) { + resources.forEach(resource -> { + if (resource.getVersion() != null && resource.getFileName() != null && resource.getResourceId() != null) { + //需要区分代码节点和非代码节点。非代码节点直接根据filename上传即可 + String fileName=resource.getFileName(); + if(fileName.equals(scriptName)){ + fileName = fileName.substring(fileName.lastIndexOf('.')); + } + String filePath = IoUtils.addFileSeparator(nodePath, fileName); + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, filePath); + BmlResource bmlReturnMap = bmlService.upload(userName, + resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.getResourceId()); + resource.setVersion(bmlReturnMap.getVersion()); + } else { + logger.warn("Illegal resource information"); + logger.warn("username:{},fileName:{},version:{},resourceId:{}", userName, resource.getFileName(), resource.getVersion(), resource.getResourceId()); + } + }); + } + return nodeParser.updateNodeResource(nodeJson, resources); + } @Override public String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException { List resources = nodeParser.getNodeResource(nodeJson); @@ -88,7 +117,65 @@ private InputStream readResource(String userName, Resource resource, String flow String readPath = flowResourcePath + resource.getResourceId() + "_" + resource.getVersion() + ".re"; return bmlService.readLocalResourceFile(userName, readPath); } + @Override + public String uploadAppConnResourceNew(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String nodePath, + Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException, IOException { + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + String nodeType = nodeJsonMap.get("jobType").toString(); + String nodeId = nodeJsonMap.get("id").toString(); + String nodeResourcePath = IoUtils.addFileSeparator(nodePath,APPCONN_FILE_NAME) ; + Map nodeContent = (LinkedHashMap) nodeJsonMap.get("jobContent"); + CommonAppConnNode appConnNode = new CommonAppConnNode(); + appConnNode.setId(nodeId); + appConnNode.setName((String) nodeJsonMap.get("title")); + appConnNode.setDssLabels(dssLabels); + appConnNode.setNodeType(nodeType); + appConnNode.setJobContent(nodeContent); + appConnNode.setFlowId(dssFlow.getId()); + appConnNode.setProjectId(dssFlow.getProjectId()); + appConnNode.setWorkspace(workspace); + appConnNode.setContextId(flowContextId); + + Map nodeExportContent = null; + logger.info("nodeResourcePath:{}", nodeResourcePath); + File file = new File(nodeResourcePath); + if (file.exists()) { + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, nodeResourcePath); + Supplier> bmlResourceMap = () -> { + BmlResource resource = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", + projectName); + return ImmutableMap.of( + "resourceId", resource.getResourceId(), + "version", resource.getVersion() + ); + }; + Supplier> streamResourceMap = () -> MapUtils.newCommonMap(ImportRequestRef.INPUT_STREAM_KEY, resourceInputStream); + try { + nodeExportContent = nodeService.importNode(userName, appConnNode, bmlResourceMap, streamResourceMap, orcVersion); + } catch (ExternalOperationFailedException e) { + logger.error("failed to import node.", e); + throw new DSSErrorException(e.getErrCode(), e.getMessage()); + } catch (Exception e) { + logger.error("failed to import node.", e); + throw new DSSErrorException(90011, e.getMessage()); + } + if (nodeExportContent != null) { + if (nodeExportContent.get("project_id") != null) { + Long newProjectId = Long.parseLong(nodeExportContent.get("project_id").toString()); + logger.warn(String.format("new appConn node add into dss,dssProjectId: %s,newProjectId: %s", appConnNode.getProjectId(), newProjectId)); + nodeExportContent.remove("project_id"); + } + nodeJsonMap.replace("jobContent", nodeExportContent); + appConnNode.setJobContent(nodeExportContent); + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeJsonMap); + } + } else { + logger.warn("appConn node resource file does not exists. nodeId: {}" + nodeId); + } + return nodeJson ; + } @Override public String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, String nodeJson, String flowContextId, String appConnResourcePath, diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java index e2e85305e..d33ae1fa4 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java @@ -23,6 +23,7 @@ import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -57,6 +58,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.WorkFlowExportServiceImpl.FLOW_FILE_NAME; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.WorkFlowExportServiceImpl.NODE_PARAMS_FILE_NAME; + @Service public class WorkFlowInputServiceImpl implements WorkFlowInputService { @@ -79,6 +83,56 @@ public class WorkFlowInputServiceImpl implements WorkFlowInputService { private FlowMapper flowMapper; private static ContextService contextService = ContextServiceImpl.getInstance(); + @Override + public void inputWorkFlowNew(String userName, + DSSFlow dssFlow, + String projectName, + String flowCodePath, + String flowMetaPath, + Long parentFlowId, + Workspace workspace, + String orcVersion, + String contextId, + List dssLabels) throws DSSErrorException, IOException { + String flowMetaFilePath = IoUtils.addFileSeparator(flowMetaPath, FLOW_FILE_NAME); + String flowJson = bmlService.readLocalTextFile(userName, flowMetaFilePath); + + // TODO: 2020/7/31 优化update方法里面的saveContent + flowJson = updateFlowContextIdAndVersion(userName, + workspace.getWorkspaceName(), + projectName, + flowJson, + dssFlow, + parentFlowId, + contextId, + orcVersion); + flowJson = inputWorkFlowNodesNew(userName, projectName, flowJson, dssFlow, + flowCodePath, workspace, orcVersion, dssLabels); + //如果包含subflow,需要一同导入subflow内容，并更新parentflow的json内容 + List subFlows = dssFlow.getChildren(); + List templateIds = new ArrayList<>(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + String subCodePath = IoUtils.addFileSeparator(flowCodePath, subFlow.getName()); + String subMetaPath = IoUtils.addFileSeparator(flowMetaPath, subFlow.getName()); + inputWorkFlowNew(userName, subFlow, projectName, subCodePath,subMetaPath, dssFlow.getId(), + workspace, orcVersion, contextId, dssLabels); + templateIds.addAll(subFlow.getFlowIdParamConfTemplateIdTuples()); + } + } + + flowJson = uploadFlowResourceToBmlNew(userName, flowJson, flowCodePath, projectName); + + DSSFlow updateDssFlow = uploadFlowJsonToBml(userName, projectName, dssFlow, flowJson); + List tempIds = workFlowParser.getParamConfTemplate(flowJson); + List templateIdsInRoot = tempIds.stream() + .map(e->new String[]{updateDssFlow.getId().toString(),e}) + .collect(Collectors.toList()); + templateIds.addAll(templateIdsInRoot); + updateDssFlow.setFlowIdParamConfTemplateIdTuples(templateIds); + contextService.checkAndSaveContext(flowJson, String.valueOf(parentFlowId)); + flowMapper.updateFlowInputInfo(updateDssFlow) ; + } @Override public void inputWorkFlow(String userName, DSSFlow dssFlow, @@ -93,7 +147,7 @@ public void inputWorkFlow(String userName, String flowInputPath = inputProjectPath + File.separator + dssFlow.getName(); String flowJsonPath = flowInputPath + File.separator + dssFlow.getName() + ".json"; String flowJson = bmlService.readLocalTextFile(userName, flowJsonPath); - //生成新的节点key。 + //生成新的节点key。为了解决复制工程后，新工程出现原有工程工作流的历史执行记录 Set nodeKeys=findFlowNodeKeys(flowJson); Map oldNewNodeKeyMap = nodeKeys.stream().collect(Collectors.toMap(Function.identity(), v -> UUID.randomUUID().toString(),(e1,e2)->e1)); String updateFlowJson=flowJson; @@ -195,6 +249,80 @@ private String updateFlowContextIdAndVersion(String userName, } return workFlowParser.updateFlowJsonWithMap(flowJson, MapUtils.newCommonMap(CSCommonUtils.CONTEXT_ID_STR, contextId, DSSJobContentConstant.ORC_VERSION_KEY, orcVersion)); } + /** + * 导入工作流节点 + * @param userName + * @param projectName + * @param flowJson + * @param dssFlow + * @param workspace + * @param orcVersion + * @param dssLabels + * @return + * @throws DSSErrorException + * @throws IOException + */ + private String inputWorkFlowNodesNew(String userName, String projectName, String flowJson, + DSSFlow dssFlow, String flowCodePath, Workspace workspace, + String orcVersion, List dssLabels) throws DSSErrorException, IOException { + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点，导入失败 " + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + List> nodeJsonListRes = new ArrayList<>(); + if (nodeJsonList.size() > 0) { + for (String nodeJson : nodeJsonList) { + String nodeName = nodeParser.getNodeValue("title", nodeJson); + String nodePath = IoUtils.addFileSeparator(flowCodePath, nodeName); + //上传节点资源到bml，实现节点资源导入，返回新的节点json + String updateNodeJson = nodeInputService.uploadResourceToBmlNew(userName, nodeJson, nodePath, projectName); + //导入第三方节点 + updateNodeJson = nodeInputService.uploadAppConnResourceNew(userName, projectName, + dssFlow, updateNodeJson, updateContextId, nodePath, + workspace, orcVersion, dssLabels); + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + String nodeParamsJson = readNodeParam(userName, nodePath); + + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + if ("workflow.subflow".equals(nodeType) && CollectionUtils.isNotEmpty(subflows)) { + String subFlowName = nodeJsonMap.get("title").toString(); + logger.info("subflows:{}", subflows); + List dssFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + } else { + logger.error("工程内未能找到子工作流节点，导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点，导入失败" + subFlowName); + } + } + if (nodeParamsJson != null && !"null".equalsIgnoreCase(nodeParamsJson)) { + nodeJsonMap.put("params", BDPJettyServerHelper.jacksonJson().readValue(nodeParamsJson, Map.class)); + } + nodeJsonListRes.add(nodeJsonMap); + + } + } + + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + } + + private String readNodeParam(String userName,String nodePath){ + String paramsFilePath = IoUtils.addFileSeparator(nodePath, NODE_PARAMS_FILE_NAME); + return bmlService.readLocalTextFile(userName, paramsFilePath); + } private String inputWorkFlowNodes(String userName, String projectName, String flowJson, DSSFlow dssFlow, String flowPath, Workspace workspace, @@ -254,7 +382,23 @@ private String inputWorkFlowNodes(String userName, String projectName, String fl return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); } + private String uploadFlowResourceToBmlNew(String userName, String flowJson, String flowCodePath, String projectName) throws IOException { + List resourceList = workFlowParser.getWorkFlowResources(flowJson); + //上传文件获取resourceId和version save应该是已经有 + if (resourceList != null) { + resourceList.forEach(resource -> { + InputStream resourceInputStream = readFlowResourceNew(userName, resource, flowCodePath); + BmlResource bmlReturnMap = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.getResourceId()); + resource.setVersion(bmlReturnMap.getVersion()); + }); + if (resourceList.size() == 0) { + return flowJson; + } + } + return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); + } private String uploadFlowResourceToBml(String userName, String flowJson, String flowResourcePath, String projectName) throws IOException { List resourceList = workFlowParser.getWorkFlowResources(flowJson); @@ -272,7 +416,10 @@ private String uploadFlowResourceToBml(String userName, String flowJson, String } return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); } - + private InputStream readFlowResourceNew(String userName, Resource resource, String flowCodePath) { + String readPath = IoUtils.addFileSeparator(flowCodePath,resource.getFileName()); + return bmlService.readLocalResourceFile(userName, readPath); + } private InputStream readFlowResource(String userName, Resource resource, String flowResourcePath) { // TODO: 2020/3/20 和导出统一,资源都放resouce 如有问题,后再修改 String readPath = flowResourcePath + File.separator + "resource" + File.separator + resource.getResourceId() + ".re"; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java index 2bb3298aa..495d1e59e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java @@ -16,23 +16,39 @@ package com.webank.wedatasphere.dss.workflow.lock; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCommitRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQuertByAppIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQueryByIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestSubmitOrchestratorSync; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; import org.apache.linkis.DataWorkCloudApplication; import org.apache.linkis.common.utils.Utils; import org.apache.commons.lang.StringUtils; -import org.apache.poi.util.StringUtil; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; +import org.springframework.util.CollectionUtils; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -49,6 +65,8 @@ public class DSSFlowEditLockManager { private static LockMapper lockMapper; + private static FlowMapper flowMapper; + private static final DelayQueue unLockEvents = new DelayQueue<>(); private static final ThreadLocal sdf = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); @@ -60,8 +78,14 @@ protected DSSFlowEditLockManager() { LOGGER.info("unLockEvents移除定时线程开启..."); LOGGER.info("编辑锁超时时间为：{} ms", DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue()); init(); - //程序重启时，删除所有编辑锁 - lockMapper.deleteALL(); + // 对于工作流状态为已保存的，不自动解锁，该工作流的锁永不过期 + List saveList = lockMapper.selectOrchestratorByStatus(OrchestratorRefConstant.FLOW_STATUS_SAVE); + if (!CollectionUtils.isEmpty(saveList)) { + lockMapper.deleteExpectAfterSave(saveList); + }else { + //程序重启时，删除所有编辑锁 + lockMapper.deleteALL(); + } Utils.defaultScheduler().scheduleAtFixedRate(() -> { try { UnLockEvent pop = unLockEvents.poll(); @@ -168,12 +192,25 @@ private static String generateLock(Long flowID, String username, String owner) t } } - public static void deleteLock(String flowEditLock) throws DSSErrorException { + public static void deleteLock(String flowEditLock, Workspace workspace) throws DSSErrorException { try { if (StringUtils.isNotBlank(flowEditLock)) { DSSFlowEditLock dssFlowEditLock = lockMapper.getFlowEditLockByLockContent(flowEditLock); if (dssFlowEditLock != null) { - lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), dssFlowEditLock.getFlowID()); + Long flowID = dssFlowEditLock.getFlowID(); + // 获取当前项目信息 + DSSFlow dssFlow = flowMapper.selectFlowByID(flowID); + DSSProject projectInfo = getProjectInfo(dssFlow.getProjectId()); + // 对于接入Git的项目，工作流解锁加入额外处理 + if (projectInfo.getAssociateGit()) { + OrchestratorVo orchestratorVo = getOrchestratorInfo(flowID); + DSSOrchestratorInfo orchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String status = lockMapper.selectOrchestratorStatus(orchestratorInfo.getId()); + if (!StringUtils.isEmpty(status) && OrchestratorRefConstant.FLOW_STATUS_SAVE.equals(status)) { + submitOrchestrator(orchestratorInfo.getId(), flowID, workspace, projectInfo.getName()); + } + } + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), flowID); } } } catch (Exception e) { @@ -182,6 +219,46 @@ public static void deleteLock(String flowEditLock) throws DSSErrorException { } } + public static DSSProject getProjectInfo(Long projectId) throws DSSErrorException { + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectInfoRequest), DSSProject.class, ProjectInfoRequest.class); + if (dssProject == null) { + throw new DSSErrorException(90001, "工程不存在"); + } + return dssProject; + } + + public static OrchestratorVo getOrchestratorInfo(Long flowID) throws DSSErrorException{ + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(flowID)), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + if (orchestratorVo == null) { + throw new DSSErrorException(800001, "编排不存在"); + } + return orchestratorVo; + } + + public static void submitOrchestrator(Long orchestratorId, Long flowId, Workspace workspace, String projectName) { + LOGGER.info("-------=======================begin to add testGit1=======================-------"); + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + // 同步提交编排至BML以及push到git + RequestSubmitOrchestratorSync submitOrchestratorSync = new RequestSubmitOrchestratorSync(); + submitOrchestratorSync.setOrchestratorId(orchestratorId); + submitOrchestratorSync.setFlowId(flowId); + submitOrchestratorSync.setComment("force unlock"); + submitOrchestratorSync.setProjectName(projectName); + LabelRouteVO labelRouteVO = new LabelRouteVO(); + labelRouteVO.setRoute("dev"); + submitOrchestratorSync.setLabels(labelRouteVO); + submitOrchestratorSync.setUsername("system"); + submitOrchestratorSync.setWorkspace(workspace); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(orcSender.ask(submitOrchestratorSync), GitCommitResponse.class, RequestSubmitOrchestratorSync.class); + LOGGER.info("-------=======================End to add submit=======================-------: {}", responseWorkflowValidNode); + } + + public static String updateLock(String lock) throws DSSErrorException { if (StringUtils.isBlank(lock)) { throw new DSSErrorException(60066, "update workflow failed because you do not have flowEditLock!"); @@ -205,7 +282,13 @@ public static String updateLock(String lock) throws DSSErrorException { } } - public static boolean isLockExpire(DSSFlowEditLock flowEditLock) { + public static boolean isLockExpire(DSSFlowEditLock flowEditLock) throws DSSErrorException{ + OrchestratorVo orchestratorVo = getOrchestratorInfo(flowEditLock.getFlowID()); + DSSOrchestratorInfo orchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String status = lockMapper.selectOrchestratorStatus(orchestratorInfo.getId()); + if (!StringUtils.isEmpty(status) && OrchestratorRefConstant.FLOW_STATUS_SAVE.equals(status)) { + return false; + } return System.currentTimeMillis() - flowEditLock.getUpdateTime().getTime() >= DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue(); } @@ -214,6 +297,7 @@ public static void init() { synchronized (DSSFlowEditLockManager.class) { if (!isInit) { lockMapper = DataWorkCloudApplication.getApplicationContext().getBean(LockMapper.class); + flowMapper = DataWorkCloudApplication.getApplicationContext().getBean(FlowMapper.class); isInit = true; } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java index bcaf0dce7..391c34723 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,14 @@ public String getNodeValue(String key, String nodeJson) throws IOException { } + + @Override + public Map getNodeJobContent(String nodeJson) throws IOException { + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + return (LinkedHashMap) nodeJsonMap.get("jobContent"); + } + + @Override public List getNodeResource(String nodeJson) { Gson gson = new Gson(); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java index 156b18124..ab1c5f504 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java @@ -29,6 +29,7 @@ import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; @@ -85,10 +86,6 @@ public class FlowRestfulApi { @Autowired private HttpServletRequest httpServletRequest; - @PostConstruct - public void init() { - AppConnManagerUtils.autoLoadAppConnManager(); - } /** * 添加subflow节点 @@ -335,9 +332,10 @@ public Message saveFlow( @RequestBody SaveFlowRequest saveFlowRequest) throws IO if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); } + version = flowService.saveFlow(flowID, jsonFlow, null, userName, workspaceName, projectName); - AuditLogUtils.printLog( username,workspace.getWorkspaceId(), workspaceName,TargetTypeEnum.WORKFLOW, - flowID,dssFlow.getName(),OperateTypeEnum.UPDATE,saveFlowRequest); + AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspaceName, TargetTypeEnum.WORKFLOW, + flowID, dssFlow.getName(), OperateTypeEnum.UPDATE, saveFlowRequest); return Message.ok().data("flowVersion", version); } @@ -366,7 +364,9 @@ public Message getExtraToolBars( @RequestBody GetExtraToolBarsRequest getExtraTo @RequestMapping(value = "/deleteFlowEditLock/{flowEditLock}", method = RequestMethod.POST) public Message deleteFlowEditLock(HttpServletRequest req, @PathVariable("flowEditLock") String flowEditLock) throws DSSErrorException { - DSSFlowEditLockManager.deleteLock(flowEditLock); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + DSSFlowEditLockManager.deleteLock(flowEditLock, workspace); return Message.ok(); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java index 0c2c3ff4f..95c824289 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java @@ -27,16 +27,24 @@ import com.webank.wedatasphere.dss.common.entity.node.DSSNode; import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; import com.webank.wedatasphere.dss.common.entity.node.Node; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQuertByAppIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQueryByIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.WorkFlowManager; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.common.parser.NodeParser; @@ -47,12 +55,14 @@ import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; import com.webank.wedatasphere.dss.workflow.lock.Lock; import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; @@ -65,6 +75,7 @@ import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.cs.client.utils.SerializeHelper; import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +100,8 @@ public class DSSFlowServiceImpl implements DSSFlowService { @Autowired private FlowMapper flowMapper; @Autowired + private LockMapper lockMapper; + @Autowired private NodeInfoMapper nodeInfoMapper; @Autowired private NodeInputService nodeInputService; @@ -108,10 +121,17 @@ public class DSSFlowServiceImpl implements DSSFlowService { @Autowired private SaveFlowHook saveFlowHook; + @Autowired + private WorkFlowManager workFlowManager; + private static ContextService contextService = ContextServiceImpl.getInstance(); private static final String TOKEN = CommonVars.apply("wds.dss.workspace.token", "BML-AUTH").getValue(); + protected Sender getOrchestratorSender() { + return DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + } + @Override public DSSFlow getFlowByID(Long id) { return flowMapper.selectFlowByID(id); @@ -263,7 +283,7 @@ public String saveFlow(Long flowID, String comment, String userName, String workspaceName, - String projectName) throws IOException{ + String projectName) throws IOException { DSSFlow dssFlow = flowMapper.selectFlowByID(flowID); String creator = dssFlow.getCreator(); @@ -304,9 +324,47 @@ public String saveFlow(Long flowID, } saveFlowHook.afterSave(jsonFlow,dssFlow,parentFlowID); String version = bmlReturnMap.get("version").toString(); + // 对子工作流,需更新父工作流状态，以便提交 + Long updateFlowId = parentFlowID == null? dssFlow.getId():parentFlowID; + updateTOSaveStatus(dssFlow.getProjectId(), updateFlowId); + return version; } + private void updateTOSaveStatus(Long projectId, Long flowID) { + try { + DSSProject projectInfo = DSSFlowEditLockManager.getProjectInfo(projectId); + //仅对接入Git的项目 更新状态为 保存 + if (projectInfo.getAssociateGit() != null && projectInfo.getAssociateGit()) { + Long rootFlowId = getRootFlowId(flowID); + if (rootFlowId != null) { + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestQuertByAppIdOrchestrator(rootFlowId)), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + lockMapper.updateOrchestratorStatus(orchestratorVo.getDssOrchestratorInfo().getId(), OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + } + } catch (DSSErrorException e) { + logger.error("getProjectInfo failed by:", e); + throw new DSSRuntimeException(e.getErrCode(),"更新工作流状态失败，您可以尝试重新保存工作流！原因：" + ExceptionUtils.getRootCauseMessage(e),e); + } + } + + private Long getRootFlowId(Long flowId) { + if (flowId == null) { + return null; + } + DSSFlow dssFlow = flowMapper.selectFlowByID(flowId); + if (dssFlow == null) { + return null; + } + if (dssFlow.getRootFlow()) { + return dssFlow.getId(); + } else { + Long parentFlowID = flowMapper.getParentFlowID(flowId); + return getRootFlowId(parentFlowID); + } + } + /** * 当数据库的父子工作流依赖关系和json中不一致时，该方法会删除数据库中的脏数据 * diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java index 92bdc280b..0d951306c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java @@ -23,15 +23,18 @@ import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestrationStatus; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCurrentCommitRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import com.webank.wedatasphere.dss.workflow.service.PublishService; import org.apache.commons.lang.StringUtils; @@ -42,7 +45,7 @@ import java.util.Map; -import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.*; public class PublishServiceImpl implements PublishService { @@ -50,6 +53,8 @@ public class PublishServiceImpl implements PublishService { private DSSFlowService dssFlowService; @Autowired private WorkFlowParser workFlowParser; + @Autowired + private LockMapper lockMapper; public void setDssFlowService(DSSFlowService dssFlowService) { this.dssFlowService = dssFlowService; @@ -84,6 +89,31 @@ public String submitPublish(String convertUser, Long workflowId, if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); } + //仅对接入Git的项目更新状态为 发布-publish + GitCommitResponse gitCommitResponse = null; + OrchestratorVo orchestratorVo = null; + Long orchestratorId = null; + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + orchestratorVo = RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestQuertByAppIdOrchestrator(workflowId)), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + if (orchestratorVo == null) { + throw new DSSErrorException(800001, "编排不存在"); + } + orchestratorId = orchestratorVo.getDssOrchestratorInfo().getId(); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (OrchestratorRefConstant.FLOW_STATUS_SAVE.equals(status)) { + throw new DSSErrorException(800001, "发布前请先提交工作流"); + } + + // 获取当前文件Commit + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitCurrentCommitRequest currentCommitRequest = new GitCurrentCommitRequest(workspace.getWorkspaceId(), dssProject.getName(), convertUser, dssFlow.getName()); + gitCommitResponse = RpcAskUtils.processAskException(sender.ask(currentCommitRequest), GitCommitResponse.class, GitCurrentCommitRequest.class); + // 更新commitId + lockMapper.updateOrchestratorVersionCommitId(gitCommitResponse.getCommitId(), orchestratorId); + // 更新工作流状态 + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUBLISH); + } String schedulerAppConnName = workFlowParser.getValueWithKey(dssFlow.getFlowJson(), DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME); if (StringUtils.isBlank(schedulerAppConnName)) { // 向下兼容老版本 @@ -97,6 +127,7 @@ public String submitPublish(String convertUser, Long workflowId, if (response.getResponse().isFailed()) { throw new DSSErrorException(50311, response.getResponse().getMessage()); } + return response.getId(); } catch (DSSErrorException e) { throw e; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala index 7501faf90..aca280966 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala @@ -19,7 +19,7 @@ package com.webank.wedatasphere.dss.workflow.receiver import com.webank.wedatasphere.dss.common.protocol.{RequestDeleteWorkflow, RequestExportWorkflow, RequestQueryWorkFlow, RequestUpdateWorkflow} import com.webank.wedatasphere.dss.orchestrator.common.protocol._ import com.webank.wedatasphere.dss.workflow.WorkFlowManager -import com.webank.wedatasphere.dss.workflow.common.protocol.{RequestCopyWorkflow, RequestCreateWorkflow, RequestDeleteBmlSource, RequestImportWorkflow, RequestSubFlowContextIds, RequestUnlockWorkflow} +import com.webank.wedatasphere.dss.workflow.common.protocol.{RequestCopyWorkflow, RequestCreateWorkflow, RequestDeleteBmlSource, RequestImportWorkflow, RequestLockWorkflow, RequestSubFlowContextIds, RequestUnlockWorkflow} import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} import javax.annotation.PostConstruct diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala index 954166b0d..c727ed718 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala @@ -56,16 +56,17 @@ class DSSWorkflowReceiver(workflowManager: WorkFlowManager) extends Receiver { new ResponseDeleteWorkflow(JobStatus.Success) case reqUnlockWorkflow: RequestUnlockWorkflow => - workflowManager.unlockWorkflow(reqUnlockWorkflow.getUsername, reqUnlockWorkflow.getFlowId, reqUnlockWorkflow.getConfirmDelete) + workflowManager.unlockWorkflow(reqUnlockWorkflow.getUsername, reqUnlockWorkflow.getFlowId, reqUnlockWorkflow.getConfirmDelete, reqUnlockWorkflow.getWorkspace) case reqExportFlow: RequestExportWorkflow => - val dssExportFlowResource: BmlResource = workflowManager.exportWorkflow( + val dssExportFlowResource: BmlResource = workflowManager.exportWorkflowNew( reqExportFlow.userName, reqExportFlow.flowID, reqExportFlow.projectId, reqExportFlow.projectName, DSSCommonUtils.COMMON_GSON.fromJson(reqExportFlow.workspaceStr, classOf[Workspace]), - reqExportFlow.dssLabelList) + reqExportFlow.dssLabelList, + reqExportFlow.exportExternalNodeAppConnResource) ResponseExportWorkflow(dssExportFlowResource.getResourceId, dssExportFlowResource.getVersion, reqExportFlow.flowID) @@ -77,7 +78,7 @@ class DSSWorkflowReceiver(workflowManager: WorkFlowManager) extends Receiver { dssFlowImportParam.setOrcVersion(requestImportWorkflow.getOrcVersion) dssFlowImportParam.setWorkspace(requestImportWorkflow.getWorkspace) dssFlowImportParam.setContextId(requestImportWorkflow.getContextId) - val dssFlows = workflowManager.importWorkflow(requestImportWorkflow.getUserName, + val dssFlows = workflowManager.importWorkflowNew(requestImportWorkflow.getUserName, requestImportWorkflow.getResourceId, requestImportWorkflow.getBmlVersion, dssFlowImportParam, requestImportWorkflow.getDssLabels) diff --git a/dss-orchestrator/orchestrators/dss-workflow/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/pom.xml index f15cbb371..af1a01d91 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/pom.xml b/dss-orchestrator/pom.xml index dc9f4c6fa..7ea53314f 100644 --- a/dss-orchestrator/pom.xml +++ b/dss-orchestrator/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-server/pom.xml b/dss-server/pom.xml index ed7e68cf7..e0ff8c349 100644 --- a/dss-server/pom.xml +++ b/dss-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 @@ -49,6 +49,11 @@ + + com.webank.wedatasphere.dss + dss-git-server + ${dss.version} + com.webank.wedatasphere.dss dss-framework-project-server diff --git a/dss-standard/development-standard/development-process-standard-execution/pom.xml b/dss-standard/development-standard/development-process-standard-execution/pom.xml index 40bb6c080..3fc48d637 100644 --- a/dss-standard/development-standard/development-process-standard-execution/pom.xml +++ b/dss-standard/development-standard/development-process-standard-execution/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/development-standard/development-process-standard/pom.xml b/dss-standard/development-standard/development-process-standard/pom.xml index f9f761d5b..f0b2371f3 100644 --- a/dss-standard/development-standard/development-process-standard/pom.xml +++ b/dss-standard/development-standard/development-process-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/dss-standard-common/pom.xml b/dss-standard/dss-standard-common/pom.xml index ee9a581af..ce08eda46 100644 --- a/dss-standard/dss-standard-common/pom.xml +++ b/dss-standard/dss-standard-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-standard/pom.xml b/dss-standard/pom.xml index a136fb2e9..898819ee8 100644 --- a/dss-standard/pom.xml +++ b/dss-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml index 592b59d6c..70ba0ac2e 100644 --- a/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml +++ b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml index 5fdab0942..b13ae4660 100644 --- a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml +++ b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/sso-integration-standard/pom.xml b/dss-standard/sso-standard/sso-integration-standard/pom.xml index 2ac6ce59e..825fed1a6 100644 --- a/dss-standard/sso-standard/sso-integration-standard/pom.xml +++ b/dss-standard/sso-standard/sso-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-project-plugin/pom.xml b/dss-standard/structure-standard/dss-project-plugin/pom.xml index 745e6b013..35809ef6f 100644 --- a/dss-standard/structure-standard/dss-project-plugin/pom.xml +++ b/dss-standard/structure-standard/dss-project-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-role-plugin/pom.xml b/dss-standard/structure-standard/dss-role-plugin/pom.xml index e5a562694..671821628 100644 --- a/dss-standard/structure-standard/dss-role-plugin/pom.xml +++ b/dss-standard/structure-standard/dss-role-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml index 00ce1d53a..545f490c7 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml +++ b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java index 64e68a2bc..1a9cefb37 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java @@ -1,9 +1,14 @@ package com.webank.wedatasphere.dss.standard.app.structure.project.ref; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import java.util.Collections; +import java.util.List; + /** * @author enjoyyin * @date 2022-03-13 @@ -34,6 +39,27 @@ default R setDSSProjectPrivilege(DSSProjectPrivilege dssProjectPrivilege) { return (R) this; } + /** + * DSS 工程的全量最新权数据源列表，包含了 DSS 工程所有的数据源 + * @return DSSProjectPrivilege + */ + default List getDSSProjectDataSources() { + String json = (String) this.getParameter("dssProjectDataSources"); + if(json != null) { + return new Gson().fromJson(json, new TypeToken>(){}.getType()); + } + return Collections.emptyList(); + } + + + default R setDSSProjectDataSources(List dssProjectDataSources) { + //为了让第三方组件可以不用升级dss依赖包，转成json string再存储 + if(dssProjectDataSources != null) { + setParameter("dssProjectDataSources", new Gson().toJson(dssProjectDataSources)); + } + return (R) this; + } + class DSSProjectContentRequestRefImpl extends StructureRequestRefImpl implements DSSProjectContentRequestRef { diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java new file mode 100644 index 000000000..2d8556ace --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import java.util.Date; + +/** + * Author: xlinliu + * Date: 2024/6/6 + */ +public class DSSProjectDataSource { + private String dataSourceName; + private String dataSourceDesc; + private Date createTime; + private Date modifyTime; + private String createUser; + private String dataSourceType; + + public String getDataSourceName() { + return dataSourceName; + } + + public void setDataSourceName(String dataSourceName) { + this.dataSourceName = dataSourceName; + } + + public String getDataSourceDesc() { + return dataSourceDesc; + } + + public void setDataSourceDesc(String dataSourceDesc) { + this.dataSourceDesc = dataSourceDesc; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getDataSourceType() { + return dataSourceType; + } + + public void setDataSourceType(String dataSourceType) { + this.dataSourceType = dataSourceType; + } +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java index 618b93edf..b33a4b6e3 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java @@ -1,7 +1,12 @@ package com.webank.wedatasphere.dss.standard.app.structure.project.ref; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import java.util.Collections; +import java.util.List; + /** * @author enjoyyin * @date 2022-03-13 @@ -46,7 +51,26 @@ default R setRemovedDSSProjectPrivilege(DSSProjectPrivilege removedDSSProjectPri setParameter("removedDSSProjectPrivilege", removedDSSProjectPrivilege); return (R) this; } + /** + * DSS 工程的全量最新权数据源列表，包含了 DSS 工程所有的数据源 + * 第三方组件需要根据这个列表来判断哪些数据源是新增的，哪些数据源是删除的，时刻保持同步 + * @return DSSProjectPrivilege + */ + default List getDSSProjectDataSources() { + String json = (String) this.getParameter("dssProjectDataSources"); + if(json != null) { + return new Gson().fromJson(json, new TypeToken>(){}.getType()); + } + return Collections.emptyList(); + } + + default R setDSSProjectDataSources(List dssProjectDataSources) { + //为了让第三方组件可以不用升级dss依赖包，转成json string再存储 + String json = new Gson().toJson(dssProjectDataSources); + setParameter("dssProjectDataSources", json); + return (R) this; + } class ProjectUpdateRequestRefImpl extends StructureRequestRefImpl implements ProjectUpdateRequestRef {} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml index 645724e18..f6259acff 100644 --- a/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/next-web/.commitlintrc b/next-web/.commitlintrc new file mode 100644 index 000000000..cda374c83 --- /dev/null +++ b/next-web/.commitlintrc @@ -0,0 +1,49 @@ +{ + "extends": [ + "@commitlint/config-conventional" + ], + "rules": { + "type-enum": [ + 2, + "always", + [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test" + ] + ], + "type-case": [ + 0 + ], + "type-empty": [ + 0 + ], + "scope-empty": [ + 0 + ], + "scope-case": [ + 0 + ], + "subject-full-stop": [ + 0, + "never" + ], + "subject-case": [ + 0, + "never" + ], + "header-max-length": [ + 0, + "always", + 72 + ] + } +} diff --git a/next-web/.env b/next-web/.env new file mode 100644 index 000000000..903fa8e22 --- /dev/null +++ b/next-web/.env @@ -0,0 +1 @@ +VITE_REQUEST_PREFIX=/api/rest_j/v1/ \ No newline at end of file diff --git a/next-web/.env.development b/next-web/.env.development new file mode 100644 index 000000000..903fa8e22 --- /dev/null +++ b/next-web/.env.development @@ -0,0 +1 @@ +VITE_REQUEST_PREFIX=/api/rest_j/v1/ \ No newline at end of file diff --git a/next-web/.eslintignore b/next-web/.eslintignore new file mode 100644 index 000000000..76add878f --- /dev/null +++ b/next-web/.eslintignore @@ -0,0 +1,2 @@ +node_modules +dist \ No newline at end of file diff --git a/next-web/.eslintrc.json b/next-web/.eslintrc.json new file mode 100644 index 000000000..68b1f7f95 --- /dev/null +++ b/next-web/.eslintrc.json @@ -0,0 +1,23 @@ +{ + "parser": "vue-eslint-parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module", + "jsx": true, + "parser": "@typescript-eslint/parser" + }, + + "plugins": ["@typescript-eslint", "prettier"], + + "extends": [ + "plugin:@typescript-eslint/recommended", + "plugin:vue/vue3-recommended", + "plugin:prettier/recommended" + ], + + "rules": { + "@typescript-eslint/ban-ts-comment": "warn", + "vue/multi-word-component-names": "off", + "no-unused-vars": "off" + } +} diff --git a/next-web/.gitignore b/next-web/.gitignore new file mode 100644 index 000000000..8e1d0f2da --- /dev/null +++ b/next-web/.gitignore @@ -0,0 +1,23 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* +package-lock.json +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/next-web/.jsconfig.json b/next-web/.jsconfig.json new file mode 100644 index 000000000..eb1b272dc --- /dev/null +++ b/next-web/.jsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@dataspherestudio": ["./packages/"] + } + } +} diff --git a/next-web/.prettierrc b/next-web/.prettierrc new file mode 100644 index 000000000..78075da4e --- /dev/null +++ b/next-web/.prettierrc @@ -0,0 +1,12 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": true, + "overrides": [ + { + "files": [".prettierrc",".commitlintrc"], + "options": { "parser": "json" } + } + ] +} diff --git a/next-web/README.md b/next-web/README.md new file mode 100644 index 000000000..3e6e4de7e --- /dev/null +++ b/next-web/README.md @@ -0,0 +1,43 @@ +# DSS Template + +vue3、vite、fes-design，是否使用 TS + +新版 DSS 初始化项目，原 DSS 项目已有功能逐步迁移 + +## 本地开发及接口调试： + +``` +lerna bootstrap +wnpm run prepare +wnpm run dev +``` + +### 方式一 + +本地开发请求后端接口通过配置 vite proxy 来进行，cookie 可以从 dss 登录接口后 set-cookie 里取 + +### 方式二 + +浏览器插件 SwitchyOmega 配置规则，非接口的资源转发至 http://localhost:5173，开发时访问目标域名即可 +网址正则示例： +^http://sit.dss.bdap.weoa.com(?!/api)._ +^http://***REMOVED***:8088(?!/api)._ + +## 代码提交规范 + +请执行 wnpm run prepare 配置好提交前 eslint 检测及 commit 信息规范校验的 git hook + +## 项目结构 + +``` +├─dist # 构建后静态资源 +├─node_modules +├─exts # 扩展插件 +└─packages # 各应用模块 + ├─dss + ├─shared + └─workspace # 工作空间管理 + +``` + +### 其它 diff --git a/next-web/config/dss.js b/next-web/config/dss.js new file mode 100644 index 000000000..d3f59a53a --- /dev/null +++ b/next-web/config/dss.js @@ -0,0 +1,30 @@ +export default { + apps: { + workspace: { + routes: 'workspace/router', + module: 'workspace/module', + i18n: { + en: 'workspace/i18n/en.json', + 'zh-CN': 'workspace/i18n/zh.json', + }, + }, + }, + exts: {}, + conf: { + app_name: 'DataSphere Studio', + // app_logo: 'src/assets/images/dssLogo.png', + user_guide: '', + // hide_view_tb_detail: true, + // hide_view_db_detail: true, + watermark: { + show: false, + template: '${username} ${time}', + timeupdate: 60000, + }, + // lsp_service: { + // sql: "${protocol}//${host}/server", + // py: "${protocol}//${host}/python", + // }, + }, + version: '1.1.12', +}; diff --git a/next-web/config/workspace.js b/next-web/config/workspace.js new file mode 100644 index 000000000..ff8b4c563 --- /dev/null +++ b/next-web/config/workspace.js @@ -0,0 +1 @@ +export default {}; diff --git a/next-web/index.html b/next-web/index.html new file mode 100644 index 000000000..e8974504c --- /dev/null +++ b/next-web/index.html @@ -0,0 +1,13 @@ + + + + + + + + + +
+ + + diff --git a/next-web/package.json b/next-web/package.json new file mode 100644 index 000000000..01eea0971 --- /dev/null +++ b/next-web/package.json @@ -0,0 +1,78 @@ +{ + "name": "dataspherestudio", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "prepare": "cd .. && husky install next-web/.husky", + "lint": "eslint . --ext .vue,.js,.ts,.jsx,.tsx", + "lint:fix": "eslint . --ext .vue,.js,.ts,.jsx,.tsx --fix", + "lint-staged": "lint-staged", + "format": "prettier --write ./**/*.{vue,ts,tsx,js,jsx,css,less,scss,json,md}", + "commit": "git-cz", + "typecheck": "vue-tsc --noEmit" + }, + "dependencies": { + "@fesjs/fes-design": "^0.7.28", + "@fesjs/traction-widget": "1.9.1", + "@vueuse/core": "^10.2.0", + "dev-ip": "^1.0.1", + "lodash-es": "^4.17.21", + "vue": "^3.2.47", + "vue-i18n": "^9.3.0-beta.19", + "vue-router": "^4.2.2" + }, + "devDependencies": { + "@commitlint/config-conventional": "^15.0.0", + "@types/node": "^20.3.1", + "@typescript-eslint/eslint-plugin": "^5.59.11", + "@typescript-eslint/parser": "^5.59.11", + "@vitejs/plugin-vue": "^4.1.0", + "@vue/tsconfig": "^0.4.0", + "commitlint": "^15.0.0", + "cz-conventional-changelog": "^3.3.0", + "eslint": "^8.41.0", + "eslint-config-prettier": "^8.8.0", + "eslint-plugin-prettier": "^4.2.1", + "eslint-plugin-vue": "^9.14.1", + "husky": "^7.0.4", + "less": "^4.1.2", + "less-loader": "^10.2.0", + "lint-staged": "^12.1.2", + "prettier": "^2.8.8", + "typescript": "^5.1.3", + "vite": "^4.3.9", + "vite-plugin-eslint": "^1.8.1", + "vue-eslint-parser": "^9.3.0", + "vue-tsc": "^1.6.5" + }, + "config": { + "commitizen": { + "path": "./next-web/node_modules/cz-conventional-changelog" + } + }, + "lint-staged": { + "*.{js,jsx,vue,ts,tsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{scss,less,css,html,md,json}": [ + "prettier --write" + ] + }, + "husky": { + "hooks": { + "pre-commit": "npm run lint-staged", + "commit-msg": "commitlint -E $HUSKY_GIT_PARAMS" + } + }, + "workspaces": [ + "packages/cyeditor", + "packages/dss", + "packages/shared", + "packages/workspace" + ] +} diff --git a/next-web/packages/dss/package.json b/next-web/packages/dss/package.json new file mode 100644 index 000000000..a1be21712 --- /dev/null +++ b/next-web/packages/dss/package.json @@ -0,0 +1,7 @@ +{ + "name": "@dataspherestudio/dss", + "version": "0.0.1", + "dependencies": {}, + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/dss/router.js b/next-web/packages/dss/router.js new file mode 100644 index 000000000..e69de29bb diff --git a/next-web/packages/shared/index.ts b/next-web/packages/shared/index.ts new file mode 100644 index 000000000..23e56d7ec --- /dev/null +++ b/next-web/packages/shared/index.ts @@ -0,0 +1,3 @@ +import request from './requst'; + +export { request }; diff --git a/next-web/packages/shared/package.json b/next-web/packages/shared/package.json new file mode 100644 index 000000000..49c6bd6fb --- /dev/null +++ b/next-web/packages/shared/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dataspherestudio/shared", + "version": "0.0.1", + "dependencies": { + "axios": "1.6.4" + }, + "main": "./index.js", + "module": "./index.js", + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/shared/requst.ts b/next-web/packages/shared/requst.ts new file mode 100644 index 000000000..949ad3eaa --- /dev/null +++ b/next-web/packages/shared/requst.ts @@ -0,0 +1,238 @@ +import type { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'; +import axios from 'axios'; + +const pending: { u: string; c: AbortController }[] = []; +const removePending = (config: AxiosRequestConfig) => { + for (let p = 0; p < pending.length; p++) { + const key = getReqestKey(config); + // 如果存在则执行取消操作 + if (pending[p].u === key) { + pending[p].c.abort(); //执行取消操作 + pending.splice(p, 1); // 移除记录 + } + } +}; + +function getReqestKey(config: AxiosRequestConfig) { + const params = JSON.stringify(config.params); + return config.url + '&' + config.method + '&' + params; +} + +function cutReq(config: AxiosRequestConfig) { + for (let p = 0; p < pending.length; p++) { + if (pending[p].u === getReqestKey(config)) { + return true; + } + } +} + +const instance = axios.create({ + baseURL: + import.meta.env.VITE_REQUEST_PREFIX || + `${location.protocol}//${window.location.host}`, + timeout: 600000, + withCredentials: true, + headers: { + 'Content-Type': 'application/json;charset=UTF-8', + }, +}); + +// request拦截器 +instance.interceptors.request.use( + (config) => { + const flag = cutReq(config); + if (flag === true) { + // 取消第二次相同请求 + removePending(config); + return config; + } + const abortController = new AbortController(); + const key = getReqestKey(config); + pending.push({ u: key, c: abortController }); + config.signal = abortController.signal; + return config; + }, + (err) => { + return Promise.reject(err); + } +); +// response拦截器 +instance.interceptors.response.use( + (response) => { + removePending(response.config); + return response; + }, + (error) => { + if (axios.isCancel(error)) { + return {}; + } + return Promise.reject(error); + } +); + +// 参数处理 +const param = function (url: string, data?: object, option?: FetchOptions) { + if (typeof option === 'string') { + option = { + method: option, + }; + } + if (option && option.method) { + if ( + option.method == 'get' || + option.method == 'delete' || + option.method == 'head' || + option.method == 'options' + ) { + option.params = data; + } + if ( + option.method == 'post' || + option.method == 'put' || + option.method == 'patch' + ) { + option.data = data; + } + } else { + option = { + method: 'get', + params: data, + }; + } + + option.url = url; + return instance.request(option); +}; + +// 请求返回数据处理 +const success = function (response: AxiosResponse) { + let data; + if (response) { + if (response.status === 401 && api.error[response.status]) { + api.error[response.status](response); + } + if (typeof response.data === 'string' && response.data) { + try { + data = JSON.parse(response.data); + } catch (err) { + throw new Error('接口返回数据格式错误'); + } + } else if (response.data) { + data = response.data; + } + } else { + throw new Error('接口返回数据格式错误'); + } + if (data && data.status !== 0) { + if (api.error[data.status]) { + api.error[data.status](response); + throw new Error(''); + } else { + throw new Error(data.message || ''); + } + } + return data || {}; +}; + +// 错误处理 +const fail = function (error: { response: any; code?: any }) { + const response = error.response; + if (error?.code == 'ERR_CANCELED') return; + if (response && api.error[response.status]) { + api.error[response.status](response); + } else { + if (response && response.data) { + // + } + } + throw error; +}; + +/** + * 请求方法 + * @param {*} url 接口路径 + * @param {*} data 请求参数 + * @param {*} option 配置 + * @returns promise + * usages： + * fetch('https://dss-open.wedatasphere.com/api/rest_j/v1/user/publicKey') + * fetch('https://dss-open.wedatasphere.com/api/rest_j/v1/user/publicKey', {}, 'get') + * fetch('/user/XX', {}, 'post') + * fetch('/user/XX', {}, {axios config options}) + */ +function fetch(url: string, data?: object, option?: FetchOptions) { + return param(url, data, option) + .then(success, fail) + .then(function (response) { + return response; + }) + .catch(function (error) { + api.showErrorMsg(error); + throw error; + }); +} + +type FetchOptions = string | AxiosRequestConfig; + +/** + * 定义接口返回的固定格式 + */ +export interface ResponseResult { + status: number; + method: string; + message: string; + data: T; +} + +export type ApiRequest = { + instance: AxiosInstance; + setError: (option: any) => void; + fetch: ( + url: string, + data?: any, + option?: FetchOptions + ) => Promise>; + error: Record void>; + showErrorMsg: (error: Error) => void; +}; + +const api: ApiRequest = { + instance: instance, + error: { + '-1': function (res) { + if (res.data && res.data.enableSSO && res.data.SSOURL) { + return window.location.replace(res.data.SSOURL); + } + if (window.location.href.indexOf('/#/login') < 0) { + if (self != top) { + window.parent.location.href = '/#/login'; + } else { + window.location.href = '/#/login'; + } + } + }, + '401': function () { + if (window.location.href.indexOf('/#/login') < 0) { + if (self != top) { + window.parent.location.href = '/#/login'; + } else { + window.location.href = '/#/login'; + } + } + }, + }, + setError: function ({ error, showErrorMsg }) { + if (error && typeof error === 'object') { + Object.assign(api.error, error); + } + if (showErrorMsg) { + api.showErrorMsg = showErrorMsg; + } + }, + showErrorMsg: function (err) { + // empty + }, + fetch, +}; + +export default api; diff --git a/next-web/packages/workspace/i18n/en.json b/next-web/packages/workspace/i18n/en.json new file mode 100644 index 000000000..e4abbac3b --- /dev/null +++ b/next-web/packages/workspace/i18n/en.json @@ -0,0 +1,65 @@ +{ + "common": { + "search": "Search", + "reset": "Reset", + "moreAction": "More Action", + "more": "More", + "createUser": "Create User", + "createTime": "Create Time", + "updateUser": "Update User", + "updateTime": "Update Time", + "action": "Action", + "edit": "Edit", + "delete": "Delete", + "pleaseInput": "Please Input", + "pleaseSelect": "Please Select", + "pleaseFillIn": "Please FillIn", + "userName": "UserName", + "save": "Save" + }, + "workspace": { + "tmlName": "Template Name", + "engineType": "Engine Type", + "relateUser": "Relate User", + "createTml": "Create Template", + "tmlApplyRule": "Template Apply Rule", + "visibility": "Visibility", + "visibleUserCount": "Visible User Count", + "visibleUser": "Visible User", + "allUser": "All User", + "specifyUser": "Specify User", + "resParams": "Resource Params", + "tmlDes": "Template Description", + "workFlowRefList": "WorkFlow Reference List", + "appRefList": "APP Reference List", + "overwriteUser": "Overwrite User", + "appName": "App Name", + "overwriteScope": "Overwrite Scope", + "appRefTable": "App Ref Table", + "addTemplate": "Add Template", + "editTemplate": "Edit Template", + "projectName": "Project Name", + "workFlowName": "WorkFlow Name" + }, + "announcement": { + "addAnnouncement": "Add announcement", + "expired": "Expired", + "inEffect": " In effect", + "notActivated": "Not activated", + "delete": "Delete", + "announcementMessageManage": "Announcement message management", + "announcementId": "Announcement ID", + "announcementContent": "Announcement content", + "announcementStartTime": "Announcement start time", + "announcementEndTime": "Announcement end time", + "announcementStatus": "Announcement status", + "creator": "Create user", + "createTime": "Creation time", + "operation": "Operation", + "pleaseEnter": "Please enter", + "pleaseSelect": "Please select", + "startTime": "Start time", + "endTime": "End time", + "addAnnouncementMessage": "The announcement was added successfully!" + } +} diff --git a/next-web/packages/workspace/i18n/zh.json b/next-web/packages/workspace/i18n/zh.json new file mode 100644 index 000000000..7da431a48 --- /dev/null +++ b/next-web/packages/workspace/i18n/zh.json @@ -0,0 +1,66 @@ +{ + "common": { + "search": "查询", + "reset": "重置", + "moreAction": "更多操作", + "more": "更多", + "createUser": "创建人", + "createTime": "创建时间", + "updateUser": "更新人", + "updateTime": "更新时间", + "action": "操作", + "edit": "编辑", + "delete": "删除", + "pleaseInput": "请输入", + "pleaseSelect": "请选择", + "pleaseFillIn": "请填写", + "userName": "用户名", + "save": "保存", + "notEmpty": "不能为空" + }, + "workSpace": { + "tmlName": "模板名称", + "engineType": "引擎类型", + "relateUser": "关联用户", + "createTml": "创建模板", + "tmlApplyRule": "模板应用规则", + "visibility": "可见范围", + "visibleUserCount": "可见用户数", + "visibleUser": "可见用户", + "allUser": "全部用户", + "specifyUser": "指定用户", + "resParams": "资源参数", + "tmlDes": "模板描述", + "workFlowRefList": "工作流引用列表", + "appRefList": "应用引用列表", + "overwriteUser": "覆盖用户", + "appName": "应用名称", + "overwriteScope": "覆盖范围", + "appRefTable": "应用引用列表", + "addTemplate": "新建模板", + "editTemplate": "编辑模板", + "projectName": "项目名称", + "workFlowName": "工作流名称" + }, + "announcement": { + "addAnnouncement": "新增公告", + "expired": "已过期", + "inEffect": "生效中", + "notActivated": "未生效", + "delete": "删除", + "announcementMessageManage": "公告消息管理", + "announcementId": "公告ID", + "announcementContent": "公告内容", + "announcementStartTime": "公告起始时间", + "announcementEndTime": "公告结束时间", + "announcementStatus": "公告状态", + "creator": "创建人", + "createTime": "创建时间", + "operation": "操作", + "pleaseEnter": "请输入", + "pleaseSelect": "请选择", + "startTime": "起始时间", + "endTime": "结束时间", + "addAnnouncementMessage": "新增公告成功！" + } +} diff --git a/next-web/packages/workspace/module/announcement/api.ts b/next-web/packages/workspace/module/announcement/api.ts new file mode 100644 index 000000000..034cad0c2 --- /dev/null +++ b/next-web/packages/workspace/module/announcement/api.ts @@ -0,0 +1,10 @@ +export default { + getNoticeList: 'dss/framework/workspace/notice/list', // 获取公告接口 + deleteNotice: 'dss/framework/workspace/notice/delete', // 删除公告 + createNotice: 'dss/framework/workspace/notice/create', //新增公告 + scripts: { + getNoticeList: 'dss/scriptis/notice/list', // 获取公告接口 + deleteNotice: 'dss/scriptis/notice/delete', // 删除公告 + createNotice: 'dss/scriptis/notice/create', //新增公告 + }, +}; diff --git a/next-web/packages/workspace/module/announcement/editAnnouncement.vue b/next-web/packages/workspace/module/announcement/editAnnouncement.vue new file mode 100644 index 000000000..30504ca86 --- /dev/null +++ b/next-web/packages/workspace/module/announcement/editAnnouncement.vue @@ -0,0 +1,129 @@ + + diff --git a/next-web/packages/workspace/module/announcement/index.vue b/next-web/packages/workspace/module/announcement/index.vue new file mode 100644 index 000000000..59ac85847 --- /dev/null +++ b/next-web/packages/workspace/module/announcement/index.vue @@ -0,0 +1,318 @@ + + + diff --git a/next-web/packages/workspace/module/components/NavHeader.vue b/next-web/packages/workspace/module/components/NavHeader.vue new file mode 100644 index 000000000..30f10cf8b --- /dev/null +++ b/next-web/packages/workspace/module/components/NavHeader.vue @@ -0,0 +1,89 @@ + + + diff --git a/next-web/packages/workspace/module/hooks/usePagination.ts b/next-web/packages/workspace/module/hooks/usePagination.ts new file mode 100644 index 000000000..abfd46705 --- /dev/null +++ b/next-web/packages/workspace/module/hooks/usePagination.ts @@ -0,0 +1,98 @@ +import { ref, computed, type Ref, type ComputedRef, onMounted } from 'vue'; +import { useDateFormat } from '@vueuse/core'; + +export type PaginationAndParams = { + pageNow?: number; + pageSize?: number; + totalCount?: number; + totalPage?: number; + [index: string]: any; +}; + +export const usePagination = ( + param?: Ref, + cb?: (param: PaginationAndParams) => Promise +) => { + // 分页信息初始化 + const init = () => ({ + pageNow: 1, + pageSize: 10, + totalCount: 0, + totalPage: 0, + }); + const pagination = ref(init()); + const isLoading = ref(false); // 加载中 + + // 查询参数 + const parameter: ComputedRef = computed(() => ({ + ...param?.value, + pageNow: pagination.value.pageNow, + pageSize: pagination.value.pageSize, + })); + + // 设置分页 + const setPagination = (params: { [index: string]: unknown } = {}): void => { + Object.keys(params).forEach((key: string) => { + pagination.value[key] = params[key]; + }); + }; + + // 回调函数 + const callback = async () => { + try { + if (cb) { + isLoading.value = true; + const pagedata = await cb(parameter.value); + isLoading.value = false; + setPagination(pagedata); + } + } catch (error) { + isLoading.value = false; + console.log(error); + } + }; + + // 查询表单 + const handleInit = async () => { + pagination.value.pageNow = 1; + await callback(); + }; + + // 查询表单 + const handleCurrent = async () => { + await callback(); + }; + + // 分页事件 + const handleCurrentChange = async (currentPage: number, pageSize: number) => { + pagination.value.pageSize = pageSize; + pagination.value.pageNow = currentPage; + await callback(); + }; + + onMounted(() => { + pagination.value.pageSize = param?.value?.pageSize || 10; + pagination.value.pageNow = param?.value?.pageNow || 1; + }); + + // 表格内容filter + const fillText = (row: { [index: string]: unknown }) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: { [index: string]: unknown }) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + return { + isLoading, + pagination, + handleInit, + handleCurrent, + handleCurrentChange, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/hooks/useTableConfig.ts b/next-web/packages/workspace/module/hooks/useTableConfig.ts new file mode 100644 index 000000000..fcd1421b4 --- /dev/null +++ b/next-web/packages/workspace/module/hooks/useTableConfig.ts @@ -0,0 +1,21 @@ +import { ref, computed, type Ref } from 'vue'; +type ObjectType = Record; +export const useTableConfig = (origins: Ref) => { + // 用于接收当前需要展示的表头的列 + const headers = ref([]); + const originHeaders = computed(() => + (origins.value as ObjectType[]).filter( + (v) => !['action', 'selection'].includes(v.type as string) + ) + ); + const showTableHeaderConfig = ref(false); + // 判断表头是否展示 + const checkTColShow = (col: string) => + headers.value.map((item: ObjectType) => item.prop).includes(col); + return { + headers, + originHeaders, + showTableHeaderConfig, + checkTColShow, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/api.ts b/next-web/packages/workspace/module/resourceTmlManage/api.ts new file mode 100644 index 000000000..f0667f19e --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/api.ts @@ -0,0 +1,267 @@ +import { request } from '@dataspherestudio/shared'; + +// 工作流引用列表获取工作流名称下拉框 +export function fetchworkFlowNames(templateId: string) { + return request + .fetch( + 'dss/framework/orchestrator/getTemplateflowNames?templateId=' + templateId + ) + .then((res: any) => { + if (!res) { + return {}; + } + return { + orchestratorNames: res.data.orchestratorNames, + }; + }) + .catch((error: any) => { + // + }); +} +// 工作流引用列表获取项目名称下拉框 +export function fetchworkFlowProjectNames(templateId: string) { + return request + .fetch( + 'dss/framework/orchestrator/getTemplateProjectNames?templateId=' + + templateId + ) + .then((res: any) => { + if (!res) { + return {}; + } + return { + projectNames: res.data.projectNames, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定模板资源参数 +export function fetchResParams(paramString: string): Promise<{ conf: any }> { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateParamDetail?' + + paramString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ conf: {} }); + return; + } + resolve({ + conf: res.data.conf, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} +// 模板列表数据 +export function fetchTemplateTableData(paramsString = {}) { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateList?' + paramsString; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { data: {} }; + } + return res; + }) + .catch((error: any) => { + // + }); +} + +// 引擎类型下拉框数据 +export function fetchEngineTypes() { + return request + .fetch('dss/framework/workspace/engineconf/getEngineTypeList') + .then((res: any) => { + if (!res) { + return { engineTypes: {} }; + } + return { + engineTypes: res.data.engineTypes, + }; + }) + .catch((error: any) => { + // + }); +} + +// 获取所有部门科室 +export function fetchDepartments() { + const url = 'dss/framework/workspace/getAllDepartments'; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { departments: [] }; + } + return { + departments: res.data.departments, + }; + }) + .catch((error: any) => { + // + }); +} + +export function fetchRelateUsersDepts(workspaceId: string) { + const url = + 'dss/framework/workspace/getAllWorkspaceUsersWithDepartment?workspaceId=' + + workspaceId; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { users: {} }; + } + return { + users: res.data.users, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定工作空间下所有用户下拉框数据 +export function fetchRelateUsers(workspaceId: string) { + const url = + 'dss/framework/workspace/getAllWorkspaceUsers?workspaceId=' + workspaceId; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { users: {} }; + } + return { + users: res.data.users, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定模板下可见用户列表数据 表格和下拉框均用该接口 +export function fetchTemplateUser(paramsString: string) { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateUserList?' + + paramsString; + return request.fetch(url); +} +// 查询工作流引用列表数据 +export function fetchWorkFlows(paramsString: string): Promise<{ data: any }> { + const url = + 'dss/framework/orchestrator/getTemplateWorkflowPageInfo?' + paramsString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} +// 查询应用引用列表数据 +export function fetchApps(paramsString: string): Promise<{ data: any }> { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateApplyInfo?' + + paramsString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 删除模板 +export function deleteTemplate(templateId: string) { + const url = + 'dss/framework/workspace/engineconf/deleteConfTemplate/' + templateId; + return new Promise((resolve, reject) => { + request + .fetch(url, {}, { method: 'post' }) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 模板名称唯一性校验 +export function fetchUniqueName(name: string) { + const url = + 'dss/framework/workspace/engineconf/checkConfTemplateName?name=' + name; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 创建或编辑模板 +export function fetchSaveTemplate(params: object) { + const url = 'dss/framework/workspace/engineconf/saveConfTemplate'; + return new Promise((resolve, reject) => { + request + .fetch(url, JSON.stringify(params), { method: 'put' }) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts new file mode 100644 index 000000000..6bfd11724 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts @@ -0,0 +1,18 @@ +export default { + // 公共接口 + getApplicationList: 'dss/framework/workspace/engineconf/getApplicationList', // 加载关联应用 + getEngineTypeList: 'dss/framework/workspace/engineconf/getEngineTypeList', // 加载引擎类型 + getEngineNameList: 'dss/framework/workspace/engineconf/getEngineNameList', // 加载引擎名 + getConfTemplateList: 'dss/framework/workspace/engineconf/getConfTemplateList', // 加载模板名称 + getConfTemplateUserList: + 'dss/framework/workspace/engineconf/getConfTemplateApplyRuleUserList', // 加载覆盖用户 + // 获取应用规则接口 + getConfTemplateApplyRuleList: + 'dss/framework/workspace/engineconf/getConfTemplateApplyRuleList', // 查询模板应用规则 + saveConfTemplateApplyRule: + 'dss/framework/workspace/engineconf/saveConfTemplateApplyRule', // 新增模板应用规则 + deleteConfTemplateApplyRule: + 'dss/framework/workspace/engineconf/deleteConfTemplateApplyRule', // 删除模板应用规则 + getConfTemplateApplyHistory: + 'dss/framework/workspace/engineconf/getConfTemplateApplyHistory', // 查询模板应用执行记录 +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue new file mode 100644 index 000000000..0909e52a3 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue @@ -0,0 +1,382 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue new file mode 100644 index 000000000..baa75e325 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue @@ -0,0 +1,230 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts new file mode 100644 index 000000000..fda07fa53 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts @@ -0,0 +1,130 @@ +import { ref, type ComputedRef } from 'vue'; +import api from '../api'; +import { useDateFormat } from '@vueuse/core'; +import { request } from '@dataspherestudio/shared'; +import { FMessage } from '@fesjs/fes-design'; +const maxPageSize = 10000; + +type ObjectType = Record; + +export const useDataList = (workspaceId?: ComputedRef) => { + const bindApplications = ref([]); // 关联应用 + const engineTypes = ref([]); // 引擎类型 + const engineNames = ref([]); // 引擎名 + const templateNames = ref([]); // 模板名称 + const ruleTypes = ref([ + { value: '1', label: '工作空间新用户规则' }, + { value: '0', label: '临时规则' }, + ]); // 规则类型 + const overlayAreas = ref([ + { value: '0', label: '全部用户' }, + { value: '1', label: '指定用户' }, + { value: '2', label: '工作空间新用户' }, + ]); // 覆盖范围 + + // 加载关联应用 + async function loadBindApplications() { + const res = await request.fetch(api.getApplicationList, {}); + bindApplications.value = (res.data?.applications || []).map( + (item: string) => ({ + valueField: item, + labelField: item, + }) + ); + } + + // 加载引擎类型 + async function loadEngineTypes(application?: string) { + const res = await request.fetch(api.getEngineTypeList, { application }); + engineTypes.value = (res.data?.engineTypes || []).map((item: string) => ({ + valueField: item, + labelField: item, + })); + } + + // 加载引擎名 + async function loadEngineNames(application?: string) { + if (!application) { + FMessage.warn('请先选择关联应用'); + return; + } + const res = await request.fetch(api.getEngineNameList, { application }); + engineNames.value = (res.data?.engineTypes || []).map((item: string) => ({ + valueField: item, + labelField: item, + })); + } + + // 加载模板名称 + async function loadTemplateNames(engineName?: string) { + if (!engineName) { + FMessage.warn('请先选择引擎类型'); + return; + } + const param = { + workspaceId: workspaceId?.value, + engineName, + pageNow: 1, + pageSize: maxPageSize, + }; + const res = await request.fetch(api.getConfTemplateList, param); + templateNames.value = (res.data?.templateList || []).map( + (item: ObjectType) => ({ + valueField: item.templateId, + labelField: item.name, + }) + ); + } + + // 函数映射关系 + const loadMap = { + bind_application: loadBindApplications, + engine_type: loadEngineTypes, + engine_name: loadEngineNames, + template_name: loadTemplateNames, + }; + + function handleSelect(type: keyof typeof loadMap, data?: string): void { + const handler = loadMap[type]; + handler(data); + } + + // 过滤出有值的对象 + function jsonFilter(obj: ObjectType): ObjectType { + if (obj.toString().toLowerCase() !== '[object object]') { + return {}; + } + const result: ObjectType = {}; + Object.keys(obj).forEach((key: string) => { + if (obj[key]) { + result[key as keyof typeof obj] = obj[key]; + } + }); + return result; + } + + // 表格内容filter + const fillText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + + return { + bindApplications, // 关联应用 + engineTypes, // 引擎类型 + engineNames, // 引擎名 + templateNames, // 模板名称 + ruleTypes, // 规则类型 + overlayAreas, // 覆盖范围 + handleSelect, + jsonFilter, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue new file mode 100644 index 000000000..3f36fc3d5 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue @@ -0,0 +1,496 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue new file mode 100644 index 000000000..46362c56f --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue @@ -0,0 +1,140 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue new file mode 100644 index 000000000..a79640638 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue @@ -0,0 +1,157 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue new file mode 100644 index 000000000..80725620b --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue @@ -0,0 +1,137 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue new file mode 100644 index 000000000..9ad9d0f22 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue @@ -0,0 +1,582 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue new file mode 100644 index 000000000..0e162212b --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue @@ -0,0 +1,186 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts b/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts new file mode 100644 index 000000000..f1dc14c09 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts @@ -0,0 +1,158 @@ +import { ref } from 'vue'; +import { + fetchTemplateUser, + fetchRelateUsers, + fetchRelateUsersDepts, + fetchEngineTypes, + fetchworkFlowProjectNames, + fetchworkFlowNames, + fetchDepartments, +} from '../api'; +import { utils } from '../hooks/utils'; +import { useDateFormat } from '@vueuse/core'; + +export const useDataList = () => { + const templateUserList = ref([]); + const allWorkSpaceUserList = ref([]); + const allWorkSpaceUserDeptsList = ref([]); + const allDeptList = ref([]); + const engineList = ref([]); + const projectNameList = ref([]); + const workflowNameList = ref([]); + const permissionTypeList = ref([ + { + value: 0, + label: '全部用户', + }, + { + value: 1, + label: '指定用户', + }, + ]); + const { paramsObjectToString } = utils(); + + const loadworkFlowNames = async (templateId: string) => { + try { + const data = await fetchworkFlowNames(templateId); + workflowNameList.value = data?.orchestratorNames.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadworkFlowProjectNames = async (templateId: string) => { + try { + const data = await fetchworkFlowProjectNames(templateId); + projectNameList.value = data?.projectNames.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadEngineList = async () => { + try { + const data = await fetchEngineTypes(); + engineList.value = data?.engineTypes.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadAllWorkSpaceUserList = async (workspaceId: string) => { + try { + const data = await fetchRelateUsers(workspaceId); + allWorkSpaceUserList.value = data?.users.accessUsers.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadAllWorkSpaceUserDeptsList = async (workspaceId: string) => { + try { + const data = await fetchRelateUsersDepts(workspaceId); + if (data && data.users && data.users.accessUsers) { + allWorkSpaceUserDeptsList.value = data.users.accessUsers.map( + (v: any) => v + ); + } else { + allWorkSpaceUserDeptsList.value = []; + } + } catch (error) { + console.log(error); + } + }; + + const loadAllDeptList = async () => { + try { + const data = await fetchDepartments(); + allDeptList.value = data?.departments.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadTemplateUserList = async (templateId: string, name: string) => { + const params = { + pageNow: 1, + pageSize: 50, + templateId: templateId, + username: name, + }; + const strParams = paramsObjectToString(params); + try { + const res = await fetchTemplateUser(strParams); + templateUserList.value = res.data.users.map((v: { name: string }) => ({ + value: v.name, + label: v.name, + })); + } catch (error) { + console.log(error); + } + }; + // 表格内容filter + const fillText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + + return { + templateUserList, + allWorkSpaceUserList, + engineList, + permissionTypeList, + projectNameList, + workflowNameList, + allDeptList, + allWorkSpaceUserDeptsList, + loadAllWorkSpaceUserDeptsList, + loadAllDeptList, + loadTemplateUserList, + loadAllWorkSpaceUserList, + loadEngineList, + loadworkFlowProjectNames, + loadworkFlowNames, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts b/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts new file mode 100644 index 000000000..1bb22a468 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts @@ -0,0 +1,27 @@ +export const utils = () => { + // 值为数组、对象前的情况未处理 + const paramsObjectToString = (params: object) => { + const strArray: string[] = []; + for (const [key, value] of Object.entries(params)) { + // console.log(`Key: ${key}, Value: ${value}`); + if (value) { + strArray.push(key + '=' + value); + } + } + const strParams = strArray.join('&'); + return strParams; + }; + + // 时间戳转为yy-MM-dd hh:mm:ss + const convertTimeStampToYMDHMS = (time: number | string) => { + let timeStr = new Date(Number(time)).toLocaleString('zh', { + hour12: false, + }); + timeStr = timeStr.replace(/\//g, '-'); + return timeStr; + }; + return { + paramsObjectToString, + convertTimeStampToYMDHMS, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/index.vue b/next-web/packages/workspace/module/resourceTmlManage/index.vue new file mode 100644 index 000000000..e9b13a136 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/index.vue @@ -0,0 +1,467 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/layout.vue b/next-web/packages/workspace/module/resourceTmlManage/layout.vue new file mode 100644 index 000000000..94e71a535 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/layout.vue @@ -0,0 +1,4 @@ + + diff --git a/next-web/packages/workspace/module/workFlow/advanceModal.vue b/next-web/packages/workspace/module/workFlow/advanceModal.vue new file mode 100644 index 000000000..5620ffca0 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/advanceModal.vue @@ -0,0 +1,148 @@ + + + + + diff --git a/next-web/packages/workspace/module/workFlow/api.ts b/next-web/packages/workspace/module/workFlow/api.ts new file mode 100644 index 000000000..28b365e46 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/api.ts @@ -0,0 +1,6 @@ +export default { + getFlowList: 'dss/workflow/get', // 获取工作流 + getNodeTypes: 'dss/workflow/listNodeType', // 获取节点类型 + getAllWorkspaceUsers: 'dss/framework/workspace/getAllWorkspaceUsers', // 获取用户 + getConfTemplateList: 'dss/framework/workspace/engineconf/getConfTemplateList', // 工作空间参数模板 +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts b/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts new file mode 100644 index 000000000..2e2b6d18c --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts @@ -0,0 +1,65 @@ +import { ref } from 'vue'; +import { request } from '@dataspherestudio/shared'; +import api from '../api'; +type ObjectType = Record; + +export const useDataList = () => { + const nodeTypes = ref([]); + const templateList = ref([]); + const userList = ref([]); + + // 获取节点类型 + async function getNodeTypes(param: ObjectType) { + await request + .fetch(api.getNodeTypes, { labels: param.labels }) + .then((res) => { + nodeTypes.value = (res.data.nodeTypes || []) + .map((item: ObjectType) => item.children) + .flat() + .map((item: ObjectType) => ({ label: item.title, value: item.type })); + }); + } + + async function getConfTemplateList(param: ObjectType, pageNow = 1) { + await request + .fetch(api.getConfTemplateList, { + workspaceId: param.workspaceId, + pageNow, + pageSize: 10000, + }) + .then((res) => { + const list = (res.data.templateList || []).map((item: ObjectType) => ({ + engineType: item.engineType, + label: item.name, + value: item.templateId, + })); + templateList.value = [...templateList.value, ...list]; + if (pageNow * 10000 < res.data.total) { + getConfTemplateList(param, pageNow + 1); + } + }); + } + + // 获取用户 + function getUserList(param: ObjectType) { + request + .fetch(api.getAllWorkspaceUsers, { + workspaceId: param.workspaceId, + }) + .then((res) => { + userList.value = res.data.users.accessUsers.map((item: ObjectType) => ({ + value: item, + label: item, + })); + }); + } + + return { + nodeTypes, + templateList, + userList, + getNodeTypes, + getConfTemplateList, + getUserList, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts b/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts new file mode 100644 index 000000000..d498c726b --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts @@ -0,0 +1,83 @@ +import { ref } from 'vue'; +type ObjectType = Record; + +export const useDataUtils = () => { + // 按照条件过滤 + function filterate(item: ObjectType, param: ObjectType) { + const validList = []; + if (param['nodeName']) { + const valid1 = item.title === param['nodeName']; + validList.push(valid1); + } + if (param['nodeType'] && (param['nodeType'] as string[]).length > 0) { + const valid2 = (param['nodeType'] as string[]).includes( + item.jobType as string + ); + validList.push(valid2); + } + if (param['templateId']) { + const valid3 = item.ecConfTemplateId === param['templateId']; + validList.push(valid3); + } + if (param['modifyUser']) { + const valid4 = item.modifyUser === param['modifyUser']; + validList.push(valid4); + } + if (param['updateTimes'] && (param['updateTimes'] as number[]).length > 0) { + const [startTime, endTime] = param['updateTimes'] as number[]; + const valid5 = + (item.modifyTime as number) >= startTime && + (item.modifyTime as number) <= endTime; + validList.push(valid5); + } + return !validList.some((valid) => valid === false); + } + + const timer = ref(); + + function handleNodeClick( + node: ObjectType | null, + action: string, + flowId: string | number + ) { + if (timer.value) { + window.clearTimeout(timer.value); + } + timer.value = window.setTimeout(() => { + window.parent.postMessage( + JSON.stringify({ + action, + flowId, + type: 'dss-nextweb', + node, + }), + '*' + ); + }, 300); + } + + function handleNodeDblClick( + node: ObjectType, + action: string, + flowId: string | number + ) { + if (timer.value) { + clearTimeout(timer.value); + } + window.parent.postMessage( + JSON.stringify({ + action, + flowId, + type: 'dss-nextweb', + node, + }), + '*' + ); + } + + return { + filterate, + handleNodeClick, + handleNodeDblClick, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts b/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts new file mode 100644 index 000000000..91dd6ee39 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts @@ -0,0 +1,36 @@ +import { ref } from 'vue'; +type ObjectType = Record; +export const useTemplateList = (templates: { [key: string]: any }) => { + const templateData = ref([]); + + function createData(data: ObjectType[]): ObjectType[] | undefined { + if (!data) return; + return data.map((v) => ({ + label: v.label, + value: v.value, + scopeBus: v.scopeBus, + children: createData(v.children as ObjectType[]), + prefix: null, + suffix: null, + })); + } + + const loadTemplates = async (node: ObjectType) => { + if (!node) { + return Promise.resolve([]); + } + const list = templates.value + .filter((item: ObjectType) => item.engineType === node.value) + .map((item: ObjectType) => { + item.isLeaf = true; + return item; + }); + return Promise.resolve(list); + }; + + return { + templateData, + createData, + loadTemplates, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/index.vue b/next-web/packages/workspace/module/workFlow/index.vue new file mode 100644 index 000000000..29a91fd1d --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/index.vue @@ -0,0 +1,638 @@ + + + diff --git a/next-web/packages/workspace/module/workFlow/nodeDrawer.vue b/next-web/packages/workspace/module/workFlow/nodeDrawer.vue new file mode 100644 index 000000000..9e015e234 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/nodeDrawer.vue @@ -0,0 +1,283 @@ + + + diff --git a/next-web/packages/workspace/package.json b/next-web/packages/workspace/package.json new file mode 100644 index 000000000..b150f7879 --- /dev/null +++ b/next-web/packages/workspace/package.json @@ -0,0 +1,9 @@ +{ + "name": "@dataspherestudio/workspace", + "version": "0.0.1", + "dependencies": { + "@dataspherestudio/shared": "^0.0.1" + }, + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/workspace/router.js b/next-web/packages/workspace/router.js new file mode 100644 index 000000000..983e537a1 --- /dev/null +++ b/next-web/packages/workspace/router.js @@ -0,0 +1,64 @@ +const routes = [ + { + path: '/workspace', + name: 'workspace', + meta: { + title: 'Workspace', + publicPage: true, + }, + component: () => import('./view/layout.vue'), + redirect: '/workspace/resourceTmlManage', + children: [ + { + path: 'resourceTmlManage', + name: 'resourceTmlManage', + meta: { + title: 'Resource Template', + publicPage: true, + }, + component: () => import('./module/resourceTmlManage/layout.vue'), + children: [ + { + path: '', + name: 'resourceTmlManage', + meta: { + title: 'Resource Template', + publicPage: true, + }, + component: () => import('./module/resourceTmlManage/index.vue'), + }, + { + path: 'applicationRule', + name: 'applicationRule', + meta: { + title: 'Application Rule', + publicPage: true, + }, + component: () => + import('./module/resourceTmlManage/applicationRule/index.vue'), + }, + ], + }, + { + path: 'announcement', + name: 'announcement', + meta: { + title: 'Announcement', + publicPage: true, + }, + component: () => import('./module/announcement/index.vue'), + }, + { + path: 'workflow', + name: 'workflow', + meta: { + title: 'Workflow', + publicPage: true, + }, + component: () => import('./module/workFlow/index.vue'), + }, + ], + }, +]; + +export default routes; diff --git a/next-web/packages/workspace/view/layout.vue b/next-web/packages/workspace/view/layout.vue new file mode 100644 index 000000000..32ca531e9 --- /dev/null +++ b/next-web/packages/workspace/view/layout.vue @@ -0,0 +1,6 @@ + diff --git a/next-web/public/vite.svg b/next-web/public/vite.svg new file mode 100644 index 000000000..e7b8dfb1b --- /dev/null +++ b/next-web/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/next-web/src/App.vue b/next-web/src/App.vue new file mode 100644 index 000000000..52ed022a9 --- /dev/null +++ b/next-web/src/App.vue @@ -0,0 +1,20 @@ + + + + + diff --git a/next-web/src/assets/vue.svg b/next-web/src/assets/vue.svg new file mode 100644 index 000000000..770e9d333 --- /dev/null +++ b/next-web/src/assets/vue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/next-web/src/base.less b/next-web/src/base.less new file mode 100644 index 000000000..d8c5a1e9f --- /dev/null +++ b/next-web/src/base.less @@ -0,0 +1,75 @@ +* { + box-sizing: border-box; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', + 'Hiragino Sans GB', 'Microsoft YaHei', 'Helvetica Neue', Helvetica, Arial, + sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', + 'JinbiaoSong', 'JinbiaoSongExt'; +} +html { + height: 100%; +} +body { + margin: 0; + padding: 0; + height: 100%; + color: #0f1222; + font-size: 14px; +} + +h1, +h2, +h3, +h4, +h5, +h6, +p { + margin: 0; + padding: 0; +} + +a { + text-decoration: none; + color: #5384ff; +} + +ul { + padding: 0; + margin: 0; + list-style: none; +} + +#app { + height: 100%; +} + +.table-container { + .table-operation-item { + padding: 0; + width: 100%; + min-width: 100%; + border: 0; + font-family: PingFangSC-Regular; + font-size: 14px; + color: #5384ff; + letter-spacing: 0; + line-height: 22px; + font-weight: 400; + &:hover { + color: #5384ff; + } + } +} + +.a-link { + cursor: pointer; +} +.fes-ellipsis { + .a-link { + color: #5384ff; + } +} +.fes-popper { + .a-link { + color: #fff; + } +} diff --git a/next-web/src/i18n.ts b/next-web/src/i18n.ts new file mode 100644 index 000000000..d88cde150 --- /dev/null +++ b/next-web/src/i18n.ts @@ -0,0 +1,26 @@ +import { createI18n } from 'vue-i18n'; +import Apps from 'virtual-app-module'; +import { merge } from 'lodash'; + +const messages = { + en: {}, + 'zh-CN': {}, +}; + +Apps.appsI18n.forEach((appI18n: any) => { + Promise.all([appI18n['zh-CN'], appI18n['en']]).then((item) => { + const msgconf = { + 'zh-CN': item[0].default, + en: item[1].default, + }; + merge(messages, msgconf); + }); +}); + +const i18n = createI18n({ + legacy: false, + locale: 'zh-CN', + messages, +}); + +export default i18n; diff --git a/next-web/src/main.ts b/next-web/src/main.ts new file mode 100644 index 000000000..cd48633a3 --- /dev/null +++ b/next-web/src/main.ts @@ -0,0 +1,65 @@ +import { createApp } from 'vue'; +import router from './router'; +import i18n from './i18n'; +import App from './App.vue'; +import { + FSpace, + FMessage, + FInput, + FButton, + FTable, + FPagination, + FSelect, + FTableColumn, + FDropdown, + FTooltip, + FCard, + FEllipsis, + FModal, + FDrawer, + FForm, + FDatePicker, + FSelectCascader, +} from '@fesjs/fes-design'; +import { + BTablePage, + BSearch, + BTableHeaderConfig, +} from '@fesjs/traction-widget'; +import { request } from '@dataspherestudio/shared'; +import './base.less'; +// 设置请求错误提示 +request.setError({ + showErrorMsg: function (error: any) { + FMessage.error(error.message || '这是一条消息'); + }, +}); + +router().then((router) => { + const app = createApp(App); + app.use(BTablePage); + app.use(BSearch); + app.use(BTableHeaderConfig); + app.use(FDrawer); + app.use(FSpace); + // @ts-ignore + app.use(FMessage); + app.use(FInput); + app.use(FButton); + app.use(FTable); + app.use(FPagination); + app.use(FSelect); + app.use(FTableColumn); + app.use(FDropdown); + app.use(FTooltip); + app.use(FCard); + app.use(FEllipsis); + app.use(FModal); + app.use(FForm); + app.use(FSpace); + app.use(FDatePicker); + app.use(FSelectCascader); + app.use(i18n); + app.use(router); + app.mount('#app'); +}); diff --git a/next-web/src/router.ts b/next-web/src/router.ts new file mode 100644 index 000000000..2748fc167 --- /dev/null +++ b/next-web/src/router.ts @@ -0,0 +1,35 @@ +import type { RouteRecordRaw } from 'vue-router'; +import { createRouter, createWebHashHistory } from 'vue-router'; + +import Apps from 'virtual-app-module'; + +console.log(Apps.appsRoutes); +function initRoutes() { + return Promise.all(Object.values(Apps.appsRoutes)).then((route) => { + let routes: RouteRecordRaw[] = []; + //@ts-ignore + route.forEach((it) => (routes = routes.concat(it.default))); + return routes; + }); +} + +export default async function init() { + const routes: RouteRecordRaw[] = await initRoutes(); + const router = createRouter({ + history: createWebHashHistory(), + routes: routes, + }); + + // 全局前置守卫 + router.beforeEach((to, from, next) => { + next(); + }); + + router.afterEach((to) => { + const _title = to.meta.title as string; + if (_title) { + window.document.title = _title; + } + }); + return router; +} diff --git a/next-web/src/vite-env.d.ts b/next-web/src/vite-env.d.ts new file mode 100644 index 000000000..11f02fe2a --- /dev/null +++ b/next-web/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/next-web/tsconfig.json b/next-web/tsconfig.json new file mode 100644 index 000000000..e649c6cd9 --- /dev/null +++ b/next-web/tsconfig.json @@ -0,0 +1,46 @@ +{ + "extends": "@vue/tsconfig/tsconfig.dom.json", + "compilerOptions": { + "target": "ES2020", + + "typeRoots": ["./node_modules/@types", "./types"], + "paths": { + "@/*": ["./*"], + "@dataspherestudio/*": ["./packages/*"] + }, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "preserve", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": false, + "noFallthroughCasesInSwitch": true + }, + "include": [ + "src/**/*.ts", + "src/**/*.d.ts", + "src/**/*.tsx", + "src/**/*.vue", + "packages/**/*.ts", + "packages/**/*.d.ts", + "packages/**/*.tsx", + "packages/**/*.vue", + "exts/**/*.ts", + "exts/**/*.d.ts", + "exts/**/*.tsx", + "exts/**/*.vue", + "types/**/*.d.ts" + ], + "references": [ + { + "path": "./tsconfig.node.json" + } + ] +} diff --git a/next-web/tsconfig.node.json b/next-web/tsconfig.node.json new file mode 100644 index 000000000..49b0ac29c --- /dev/null +++ b/next-web/tsconfig.node.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2020", + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true + }, + + "include": ["vite.config.ts", "vite-plugin/*.ts", "types/**/*.d.ts"] +} diff --git a/next-web/types/index.d.ts b/next-web/types/index.d.ts new file mode 100644 index 000000000..91e74db7f --- /dev/null +++ b/next-web/types/index.d.ts @@ -0,0 +1,23 @@ +declare type AppModule = 'workspace'; + +declare type ConfigOfApps = { + routes: string; + module: string; + i18n: { + en: string; + 'zh-CN': string; + }; +}; + +declare type AppConf = { + apps: { + [key in AppModule]: ConfigOfApps; + }; + exts: object; + conf: object; + version: string; +}; + +declare interface Window { + __MICRO_APP_ENVIRONMENT__: any; +} diff --git a/next-web/types/modules.d.ts b/next-web/types/modules.d.ts new file mode 100644 index 000000000..6ccf7e259 --- /dev/null +++ b/next-web/types/modules.d.ts @@ -0,0 +1,16 @@ +declare module 'virtual-app-module' { + import type { RouteRecordRaw } from 'vue-router'; + type AppModuleType = 'dss' | 'workspace' | 'scriptis'; + + let exts: object; + let appsRoutes: { + [key in AppModuleType]: RouteRecordRaw[]; + }; + let appsI18n: []; + let conf: { + [key: string]: any; + }; + let gitInfo: { + [key: string]: any; + }; +} diff --git a/next-web/types/shims-vue.d.ts b/next-web/types/shims-vue.d.ts new file mode 100644 index 000000000..798e8fcfa --- /dev/null +++ b/next-web/types/shims-vue.d.ts @@ -0,0 +1,5 @@ +declare module '*.vue' { + import { defineComponent } from 'vue'; + const component: ReturnType; + export default component; +} diff --git a/next-web/vite-plugin/vite-dynamic-module.ts b/next-web/vite-plugin/vite-dynamic-module.ts new file mode 100644 index 000000000..cd7314a0e --- /dev/null +++ b/next-web/vite-plugin/vite-dynamic-module.ts @@ -0,0 +1,108 @@ +import child_process from 'child_process'; + +// 默认打包DSS +let configFile = `dss.js`; +if (process.env.npm_config_configfile) { + configFile = `.${process.env.npm_config_configfile}`; +} + +const config = await import(`./config/${configFile}`); +const { apps = {}, exts = {}, conf = {}, version } = config.default as AppConf; + +if (version) { + process.env.VUE_APP_VERSION = version; +} + +const appsRoutes: string[] = []; +const appsI18n: string[] = []; +const confs: string[] = []; + +Object.entries(apps).forEach((item) => { + // 处理路由 + const conf = item[1] as ConfigOfApps; + if (conf.routes) { + appsRoutes.push(`${item[0]}: import('@/packages/${conf.routes}')`); + } + // 处理国际化 + if (conf.i18n) { + appsI18n.push(`{ + 'zh-CN': import('@/packages/${conf.i18n['zh-CN']}'), + 'en': import('@/packages/${conf.i18n['en']}') + }`); + } +}); + +// 扩展模块 +const extsMoule: string[] = []; +Object.keys(exts).forEach((item, index) => { + extsMoule[index] = ` + '${item}': { + 'module': import('@/exts/${exts[item].module}'), + 'options': ${JSON.stringify(exts[item].options)} + } + `; + // 扩展模块国际化文件 + if (exts[item].i18n) { + appsI18n.push(`{ + 'zh-CN': import('@/exts/${exts[item].i18n['zh-CN']}'), + 'en': import('@/exts/${exts[item].i18n['en']}') + }`); + } +}); + +// config +Object.keys(conf).forEach((item) => { + if (['app_logo'].includes(item)) { + confs.push(`${item}: import('@/${conf[item]}')`); + } else if (typeof conf[item] == 'string') { + confs.push(`${item}: '${conf[item]}'`); + } else if (typeof conf[item] == 'object') { + confs.push(`${item}: ${JSON.stringify(conf[item])}`); + } else { + confs.push(`${item}: ${conf[item]}`); + } +}); + +const gitInfo: { branchName: string; gitUrl: string; commitInfo: string } = { + branchName: '', + gitUrl: '', + commitInfo: '', +}; +try { + // 当前构建分支信息 + gitInfo.branchName = child_process + .execSync('git branch --show-current') + .toString(); + gitInfo.gitUrl = child_process.execSync('git remote -v').toString(); + const commitInfo = child_process.execSync('git branch -vv').toString(); + gitInfo.commitInfo = commitInfo + .split(/[\r\n]/) + .filter((item) => item.indexOf('*') > -1) + .join(''); +} catch (e) { + /* empty */ +} + +export default function dynamicPlugin() { + const virtualModuleId = 'virtual-app-module'; + const resolvedVirtualModuleId = '\0' + virtualModuleId; + return { + name: 'virtual-module-plugin', + resolveId(id) { + if (id === virtualModuleId) { + return resolvedVirtualModuleId; + } + }, + load(id) { + if (id === resolvedVirtualModuleId) { + return `export default { + exts: {${extsMoule.join(',')}}, + appsRoutes: {${appsRoutes.join(',')}}, + appsI18n: [${appsI18n.join(',')}], + conf: {${confs.join(',')}}, + gitInfo: ${JSON.stringify(gitInfo)} + };`; + } + }, + }; +} diff --git a/next-web/vite-plugin/vite-host.ts b/next-web/vite-plugin/vite-host.ts new file mode 100644 index 000000000..15f775f52 --- /dev/null +++ b/next-web/vite-plugin/vite-host.ts @@ -0,0 +1,25 @@ +import devip from 'dev-ip'; +import { Plugin } from 'vite'; + +export default function vitePluginHost(): Plugin { + return { + name: 'vite-plugin-host', + apply: 'serve', + config: () => ({ + server: { + host: '0.0.0.0', + hmr: { + host: devip()[0], + port: 5173, + }, + }, + }), + configureServer(server) { + server.middlewares.use((req, _, next) => { + // @ts-ignore + req.url = req._parsedUrl.path; + next(); + }); + }, + }; +} diff --git a/next-web/vite.config.ts b/next-web/vite.config.ts new file mode 100644 index 000000000..2f32f9377 --- /dev/null +++ b/next-web/vite.config.ts @@ -0,0 +1,54 @@ +import { resolve } from 'path'; +import vue from '@vitejs/plugin-vue'; +import { ConfigEnv, defineConfig, loadEnv, UserConfig } from 'vite'; +import host from './vite-plugin/vite-host'; +import eslintPlugin from 'vite-plugin-eslint'; +import virtualModule from './vite-plugin/vite-dynamic-module'; + +// https://vitejs.dev/config/ +export default defineConfig(({ command, mode }: ConfigEnv): UserConfig => { + // 根据当前工作目录中的 `mode` 加载 .env 文件 + // 设置第三个参数为 '' 来加载所有环境变量，而不管是否有 `VITE_` 前缀。 + const env = loadEnv(mode, process.cwd(), ''); + console.log(env, command, mode); + return { + define: {}, + plugins: [host(), vue(), virtualModule(), eslintPlugin()], + resolve: { + alias: { + '@': resolve(__dirname, './'), + '@dataspherestudio': resolve(__dirname, './packages'), + }, + }, + css: { + preprocessorOptions: { + less: { + javascriptEnabled: true, + }, + }, + }, + server: { + https: false, + proxy: { + '/api': { + target: 'http://***REMOVED***:8088', + // target: ***REMOVED*** + // target: ***REMOVED*** + secure: false, + changeOrigin: true, + followRedirects: true, + headers: { + cookie: '', + // cookie: + // 'linkis_user_session_ticket_id_v1=FmzyJmP2DGUzEo5Hy/D0OqyinoWZ4+L52vOV55MqIl4=; workspaceId=104; workspaceName=bdapWorkspace_move', + TokenCode: 'admin-kmsnd', + TokenUser: 'stacyyan', + TokenAlive: 'true', + }, + }, + }, + }, + base: './', + build: { target: 'chrome66' }, + }; +}); diff --git a/plugins/azkaban/linkis-jobtype/pom.xml b/plugins/azkaban/linkis-jobtype/pom.xml index e2ace7bb3..6600628c1 100644 --- a/plugins/azkaban/linkis-jobtype/pom.xml +++ b/plugins/azkaban/linkis-jobtype/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml linkis-jobtype diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml index ca0cbacb8..222f1dc02 100644 --- a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml index 0a01b3df1..5b61f26e5 100644 --- a/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml index e2d8af4fd..2d9b5a552 100644 --- a/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../pom.xml diff --git a/plugins/dolphinscheduler/pom.xml b/plugins/dolphinscheduler/pom.xml index c4fc2db6c..5ba083172 100644 --- a/plugins/dolphinscheduler/pom.xml +++ b/plugins/dolphinscheduler/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/plugins/linkis/dss-gateway-support/pom.xml b/plugins/linkis/dss-gateway-support/pom.xml index 4d24332da..1b8e576fb 100644 --- a/plugins/linkis/dss-gateway-support/pom.xml +++ b/plugins/linkis/dss-gateway-support/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index b1372a31f..f7c4f040b 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ pom com.webank.wedatasphere.dss dss - 1.1.0.20-SNAPSHOT + 1.5.0-SNAPSHOT dss-commons @@ -37,15 +37,16 @@ plugins/linkis/dss-gateway-support assembly plugins/dolphinscheduler + dss-git - 1.1.0.20-SNAPSHOT - 1.1.19-wds-SNAPSHOT + 1.5.0-SNAPSHOT + 1.3.0-wds-SNAPSHOT 2.11.12 1.8 3.3.3 - 2.8.5 + 2.8.9 2.14.1 3.1.1 4.5.4 diff --git a/web/README-DEV.md b/web/README-DEV.md index c75b99752..6b3ad0dbb 100644 --- a/web/README-DEV.md +++ b/web/README-DEV.md @@ -32,7 +32,6 @@ - 子应用可以配置自己的layout需要在应用router模块导出配置subAppRoutes - 各应用需要使用iView作为UI库，并提供路由，国际化等配置写入config.json -- 各应用间不要相互直接依赖，确有依赖通过lerna管理 - 可复用组件，资源需要合理放置，packages/shared 共享组件方法，修改需要注意影响 - 各应用路由应以应用名做统一前缀 - 各应用之间需要事件通信，应当在config.json 里声明对应module文件路径 @@ -53,11 +52,7 @@ http://localhost:8080/#/demoHome ``` # 安装依赖 -lerna bootstrap -# 添加依赖通过lerna add 添加 -lerna add [@version] -# 指定组件添加依赖 -lerna add [@version] --@scope=@dataspherestudio/workspace +npm i # 开发启动DSS npm run serve # 运行部分模块子应用，支持通过module组合。如科管版本： diff --git a/web/lerna.json b/web/lerna.json deleted file mode 100644 index 7e9ca0719..000000000 --- a/web/lerna.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "packages": [ - "packages/*" - ], - "command": { - "bootstrap": { - "hoist": true - } - }, - "npmClient": "npm", - "version": "1.1.6" -} diff --git a/web/package-lock.json b/web/package-lock.json index b73e82c40..0565f7887 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,13 +1,28 @@ { "name": "dataspherestudio", - "version": "1.1.6", + "version": "1.1.20", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "dataspherestudio", - "version": "1.1.6", + "version": "1.1.20", "hasInstallScript": true, + "workspaces": [ + "packages/apiServices", + "packages/cyeditor", + "packages/dataGovernance", + "packages/dss", + "packages/editor", + "packages/editorLsp", + "packages/exts", + "packages/scheduleCenter", + "packages/dolphinScheduler", + "packages/scriptis", + "packages/shared", + "packages/workflows", + "packages/workspace" + ], "dependencies": { "axios": "0.21.1", "babel-polyfill": "6.26.0", @@ -20,6 +35,7 @@ "md5": "2.3.0", "moment": "2.29.1", "monaco-editor": "0.19.3", + "monaco-languageclient": "^0.13.0", "qs": "6.9.4", "svgo": "1.3.0", "vue": "2.6.12", @@ -47,8 +63,8 @@ "less-loader": "6.1.0", "lint-staged": "8.2.1", "monaco-editor-webpack-plugin": "1.8.2", - "node-sass": "6.0.1", "patch-package": "6.2.2", + "sass": "1.49.0", "sass-loader": "10.2.1", "speed-measure-webpack-plugin": "1.5.0", "svg-sprite-loader": "5.0.0", @@ -1336,7 +1352,6 @@ "version": "7.17.9", "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.17.9.tgz", "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "dev": true, "dependencies": { "regenerator-runtime": "^0.13.4" }, @@ -1428,6 +1443,54 @@ "node": ">=6.9.0" } }, + "node_modules/@dataspherestudio/apiServices": { + "resolved": "packages/apiServices", + "link": true + }, + "node_modules/@dataspherestudio/cyeditor": { + "resolved": "packages/cyeditor", + "link": true + }, + "node_modules/@dataspherestudio/dataGovernance": { + "resolved": "packages/dataGovernance", + "link": true + }, + "node_modules/@dataspherestudio/dolphinScheduler": { + "resolved": "packages/dolphinScheduler", + "link": true + }, + "node_modules/@dataspherestudio/dss": { + "resolved": "packages/dss", + "link": true + }, + "node_modules/@dataspherestudio/editor": { + "resolved": "packages/editor", + "link": true + }, + "node_modules/@dataspherestudio/editor-lsp": { + "resolved": "packages/editorLsp", + "link": true + }, + "node_modules/@dataspherestudio/exts": { + "resolved": "packages/exts", + "link": true + }, + "node_modules/@dataspherestudio/scriptis": { + "resolved": "packages/scriptis", + "link": true + }, + "node_modules/@dataspherestudio/shared": { + "resolved": "packages/shared", + "link": true + }, + "node_modules/@dataspherestudio/workflows": { + "resolved": "packages/workflows", + "link": true + }, + "node_modules/@dataspherestudio/workspace": { + "resolved": "packages/workspace", + "link": true + }, "node_modules/@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -6009,6 +6072,24 @@ "@octokit/openapi-types": "^11.2.0" } }, + "node_modules/@riophae/vue-treeselect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@riophae/vue-treeselect/-/vue-treeselect-0.4.0.tgz", + "integrity": "sha512-J4atYmBqXQmiPFK/0B5sXKjtnGc21mBJEiyKIDZwk0Q9XuynVFX6IJ4EpaLmUgL5Tve7HAS7wkiGGSti6Uaxcg==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "babel-helper-vue-jsx-merge-props": "^2.0.3", + "easings-css": "^1.0.0", + "fuzzysearch": "^1.0.3", + "is-promise": "^2.1.0", + "lodash": "^4.0.0", + "material-colors": "^1.2.6", + "watch-size": "^2.0.0" + }, + "peerDependencies": { + "vue": "^2.2.0" + } + }, "node_modules/@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmmirror.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -7172,6 +7253,8 @@ "resolved": "https://registry.npmmirror.com/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.4.2" } @@ -7245,7 +7328,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "optional": true, + "devOptional": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -7549,6 +7632,8 @@ "resolved": "https://registry.npmmirror.com/async-foreach/-/async-foreach-0.1.3.tgz", "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "*" } @@ -7730,6 +7815,11 @@ "eslint": ">= 4.12.1" } }, + "node_modules/babel-helper-vue-jsx-merge-props": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-2.0.3.tgz", + "integrity": "sha512-gsLiKK7Qrb7zYJNgiXKpXblxbV5ffSwR0f5whkPAaBAR4fhi6bwRZxX9wBlIc5M/v8CCkXUbXZL4N/nSE97cqg==" + }, "node_modules/babel-loader": { "version": "8.2.5", "resolved": "https://registry.npmmirror.com/babel-loader/-/babel-loader-8.2.5.tgz", @@ -7955,6 +8045,12 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/babel-plugin-transform-remove-console": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-remove-console/-/babel-plugin-transform-remove-console-6.9.4.tgz", + "integrity": "sha512-88blrUrMX3SPiGkT1GnvVY8E/7A+k6oj3MNvUtTIxJflFzXTw1bHkuJ/y039ouhFMp2prRn5cQGzokViYi1dsg==", + "dev": true + }, "node_modules/babel-polyfill": { "version": "6.26.0", "resolved": "https://registry.npmmirror.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz", @@ -8110,7 +8206,7 @@ "version": "2.2.0", "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -8230,6 +8326,14 @@ "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "node_modules/bootstrap": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", + "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==", + "engines": { + "node": ">=6" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -8815,7 +8919,7 @@ "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "optional": true, + "devOptional": true, "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -8836,7 +8940,7 @@ "version": "3.0.2", "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -8848,7 +8952,7 @@ "version": "7.0.1", "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -8860,7 +8964,7 @@ "version": "7.0.0", "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.12.0" } @@ -8869,7 +8973,7 @@ "version": "5.0.1", "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "dependencies": { "is-number": "^7.0.0" }, @@ -9159,6 +9263,16 @@ "dev": true, "optional": true }, + "node_modules/clipboard": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-2.0.8.tgz", + "integrity": "sha512-Y6WO0unAIQp5bLmk1zdThRhgJt/x3ks6f30s3oE3H1mgIEU33XyQjEf8gsf6DxC7NPX8Y1SsNWjUjL/ywLnnbQ==", + "dependencies": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "node_modules/clipboardy": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/clipboardy/-/clipboardy-2.3.0.tgz", @@ -11092,6 +11206,11 @@ "node": ">=4" } }, + "node_modules/cssfilter": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", + "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==" + }, "node_modules/cssnano": { "version": "4.1.11", "resolved": "https://registry.npmmirror.com/cssnano/-/cssnano-4.1.11.tgz", @@ -11231,6 +11350,39 @@ "resolved": "https://registry.npmmirror.com/cyclist/-/cyclist-1.0.1.tgz", "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" }, + "node_modules/cytoscape": { + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.27.0.tgz", + "integrity": "sha512-pPZJilfX9BxESwujODz5pydeGi+FBrXq1rcaB1mfhFXXFJ9GjE6CNndAk+8jPzoXGD+16LtSS4xlYEIUiW4Abg==", + "dependencies": { + "heap": "^0.2.6", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-klay": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cytoscape-klay/-/cytoscape-klay-3.1.4.tgz", + "integrity": "sha512-VwPj0VR25GPfy6qXVQRi/MYlZM/zkdvRhHlgqbM//lSvstgM6fhp3ik/uM8Wr8nlhskfqz/M1fIDmR6UckbS2A==", + "dependencies": { + "klayjs": "^0.4.1" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape/node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/d3": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", + "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==" + }, "node_modules/d3-dispatch": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-2.0.0.tgz", @@ -11301,6 +11453,11 @@ "node": "*" } }, + "node_modules/dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "node_modules/de-indent": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/de-indent/-/de-indent-1.0.2.tgz", @@ -11672,6 +11829,11 @@ "node": ">=0.4.0" } }, + "node_modules/delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz", @@ -11727,6 +11889,14 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "node_modules/dexie": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.0.4.tgz", + "integrity": "sha512-rwS3k8BBstjGFJAS/yjYrsRZucqitnrP3NBhqghl9ihWcWABlT3I5f2dZV9T3pmXGAP7G9p1q00g/axcsuOT8A==", + "engines": { + "node": ">=6.0" + } + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/dezalgo/-/dezalgo-1.0.4.tgz", @@ -11903,6 +12073,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "node_modules/dt-sql-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/dt-sql-parser/-/dt-sql-parser-1.2.1.tgz", + "integrity": "sha512-bp6Rtm8N3m2Z3tavtKGo8J5ffaPQdPodYV2xGi9f6IsWbHzOkpNBxHlORcWtnSLtA/JvZcvImQxgbkdaGiv5cw==" + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -11920,6 +12095,11 @@ "stream-shift": "^1.0.0" } }, + "node_modules/easings-css": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/easings-css/-/easings-css-1.0.0.tgz", + "integrity": "sha512-7Uq7NdazNfVtr0RNmPAys8it0zKCuaqxJStYKEl72D3j4gbvXhhaM7iWNbqhA4C94ygCye6VuyhzBRQC4szeBg==" + }, "node_modules/easy-stack": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/easy-stack/-/easy-stack-1.0.1.tgz", @@ -14225,6 +14405,11 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "node_modules/fuzzysearch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fuzzysearch/-/fuzzysearch-1.0.3.tgz", + "integrity": "sha512-s+kNWQuI3mo9OALw0HJ6YGmMbLqEufCh2nX/zzV5CrICQ/y4AwPxM+6TIiF9ItFCHXFCyM/BfCCmN57NTIJuPg==" + }, "node_modules/g-status": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/g-status/-/g-status-2.0.2.tgz", @@ -14316,6 +14501,8 @@ "resolved": "https://registry.npmmirror.com/gaze/-/gaze-1.1.3.tgz", "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "globule": "^1.0.0" }, @@ -14660,8 +14847,7 @@ "node_modules/glob-to-regexp": { "version": "0.3.0", "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", - "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==", - "dev": true + "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==" }, "node_modules/glob2base": { "version": "0.0.12", @@ -14717,6 +14903,8 @@ "resolved": "https://registry.npmmirror.com/globule/-/globule-1.3.3.tgz", "integrity": "sha512-mb1aYtDbIjTu4ShMB85m3UzjX9BVKe9WCzsnfMSZk+K5GpIbBOexgg4PPCt5eHDEG5/ZQAUX2Kct02zfiPLsKg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "~7.1.1", "lodash": "~4.17.10", @@ -14731,6 +14919,8 @@ "resolved": "https://registry.npmmirror.com/glob/-/glob-7.1.7.tgz", "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -14748,6 +14938,8 @@ "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.0.8.tgz", "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -14755,6 +14947,14 @@ "node": "*" } }, + "node_modules/good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha512-goW1b+d9q/HIwbVYZzZ6SsTr4IgE+WA44A0GmPIQstuOrgsFcT7VEJ48nmr9GaRtNu0XTKacFLGnBPAM6Afouw==", + "dependencies": { + "delegate": "^3.1.2" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -15018,12 +15218,26 @@ "he": "bin/he" } }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "node_modules/hex-color-regex": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz", "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==", "dev": true }, + "node_modules/highlight.js": { + "version": "9.18.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.3.tgz", + "integrity": "sha512-zBZAmhSupHIl5sITeMqIJnYCDfAEc3Gdkqj65wC1lpI468MMQeeQkhcIAvk+RylAkxrCcI9xy9piHiXeQ1BdzQ==", + "deprecated": "Version no longer supported. Upgrade to @latest", + "engines": { + "node": "*" + } + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -15553,6 +15767,12 @@ "node": ">=0.10.0" } }, + "node_modules/immutable": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", + "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==", + "dev": true + }, "node_modules/import-cwd": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/import-cwd/-/import-cwd-2.1.0.tgz", @@ -15962,7 +16182,7 @@ "version": "2.1.0", "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "optional": true, + "devOptional": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -16306,8 +16526,7 @@ "node_modules/is-promise": { "version": "2.2.2", "resolved": "https://registry.npmmirror.com/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", - "dev": true + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" }, "node_modules/is-regex": { "version": "1.1.4", @@ -16489,8 +16708,7 @@ "node_modules/jquery": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/jquery/-/jquery-3.6.0.tgz", - "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==", - "peer": true + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, "node_modules/js-base64": { "version": "2.6.4", @@ -16691,6 +16909,11 @@ "graceful-fs": "^4.1.11" } }, + "node_modules/klayjs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/klayjs/-/klayjs-0.4.1.tgz", + "integrity": "sha512-WUNxuO7O79TEkxCj6OIaK5TJBkaWaR/IKNTakgV9PwDn+mrr63MLHed34AcE2yTaDntgO6l0zGFIzhcoTeroTA==" + }, "node_modules/klona": { "version": "2.0.5", "resolved": "https://registry.npmmirror.com/klona/-/klona-2.0.5.tgz", @@ -18013,12 +18236,25 @@ "node": ">=4" } }, + "node_modules/material-colors": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", + "integrity": "sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==" + }, "node_modules/math-random": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/math-random/-/math-random-1.0.4.tgz", "integrity": "sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==", "dev": true }, + "node_modules/mavon-editor": { + "version": "2.10.4", + "resolved": "https://registry.npmjs.org/mavon-editor/-/mavon-editor-2.10.4.tgz", + "integrity": "sha512-CFsBLkgt/KZBDg+SJYe2fyYv4zClY149PiwpH0rDAiiP4ae1XNs0GC8nBsoTeipsHcebDLN1QMkt3bUsnMDjQw==", + "dependencies": { + "xss": "^1.0.6" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz", @@ -18829,6 +19065,25 @@ "node": "*" } }, + "node_modules/moment-timezone": { + "version": "0.5.45", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", + "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", + "dependencies": { + "moment": "^2.29.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone/node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "engines": { + "node": "*" + } + }, "node_modules/monaco-editor": { "version": "0.19.3", "resolved": "https://registry.npmmirror.com/monaco-editor/-/monaco-editor-0.19.3.tgz", @@ -18847,6 +19102,20 @@ "webpack": "^4.5.0" } }, + "node_modules/monaco-languageclient": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-0.13.0.tgz", + "integrity": "sha512-aCwd33dTitwV5QwY56rpYHwzEGXei8TZ+yvZcvP3gEMd6Mizr8m3pOuoknDi2SUfLuNAHS6+ulvLgZlNQB5awg==", + "dependencies": { + "glob-to-regexp": "^0.3.0", + "vscode-jsonrpc": "^5.0.0", + "vscode-languageclient": "^6.0.0", + "vscode-uri": "^2.1.1" + }, + "engines": { + "vscode": "^1.41.0" + } + }, "node_modules/move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -18972,7 +19241,7 @@ "version": "2.15.0", "resolved": "https://registry.npmmirror.com/nan/-/nan-2.15.0.tgz", "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "devOptional": true + "optional": true }, "node_modules/nanomatch": { "version": "1.2.13", @@ -19183,6 +19452,8 @@ "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==", "dev": true, "hasInstallScript": true, + "optional": true, + "peer": true, "dependencies": { "async-foreach": "^0.1.3", "chalk": "^1.1.1", @@ -19212,6 +19483,8 @@ "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19221,6 +19494,8 @@ "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19230,6 +19505,8 @@ "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -19246,6 +19523,8 @@ "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -19255,6 +19534,8 @@ "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -19269,6 +19550,8 @@ "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -19282,6 +19565,8 @@ "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "minipass": "^3.0.0" }, @@ -19294,6 +19579,8 @@ "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19303,6 +19590,8 @@ "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -19315,6 +19604,8 @@ "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -19327,6 +19618,8 @@ "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -19339,6 +19632,8 @@ "resolved": "https://registry.npmmirror.com/meow/-/meow-9.0.0.tgz", "integrity": "sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/minimist": "^1.2.0", "camelcase-keys": "^6.2.2", @@ -19362,6 +19657,8 @@ "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.4.tgz", "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -19374,6 +19671,8 @@ "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -19387,6 +19686,8 @@ "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", "dev": true, + "optional": true, + "peer": true, "bin": { "mkdirp": "bin/cmd.js" }, @@ -19399,6 +19700,8 @@ "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-7.1.2.tgz", "integrity": "sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "env-paths": "^2.2.0", "glob": "^7.1.4", @@ -19423,6 +19726,8 @@ "resolved": "https://registry.npmmirror.com/nopt/-/nopt-5.0.0.tgz", "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abbrev": "1" }, @@ -19438,6 +19743,8 @@ "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -19453,6 +19760,8 @@ "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -19465,6 +19774,8 @@ "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19474,6 +19785,8 @@ "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19483,6 +19796,8 @@ "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz", "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "find-up": "^4.1.0", "read-pkg": "^5.2.0", @@ -19497,6 +19812,8 @@ "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19506,6 +19823,8 @@ "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -19518,6 +19837,8 @@ "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -19533,6 +19854,8 @@ "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -19545,6 +19868,8 @@ "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19554,6 +19879,8 @@ "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -19566,6 +19893,8 @@ "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.0" } @@ -19575,6 +19904,8 @@ "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.12.tgz", "integrity": "sha512-jU4TdemS31uABHd+Lt5WEYJuzn+TJTCBLljvIAHZOz6M9Os5pJ4dD+vRFLxPa/n3T0iEFzpi+0x1UfuDZYbRMw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -19592,6 +19923,8 @@ "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.18.1.tgz", "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -19601,6 +19934,8 @@ "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -19615,7 +19950,9 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/nopt": { "version": "4.0.3", @@ -22553,7 +22890,7 @@ "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "optional": true, + "devOptional": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -22561,6 +22898,11 @@ "node": ">=8.10.0" } }, + "node_modules/reconnecting-websocket": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/reconnecting-websocket/-/reconnecting-websocket-4.4.0.tgz", + "integrity": "sha512-D2E33ceRPga0NvTDhJmphEgJ7FUYF0v4lr1ki0csq06OdlxKfugGzN0dSkxM/NfqCxYELK4KcaTOUOjTV6Dcng==" + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/redent/-/redent-3.0.0.tgz", @@ -22607,8 +22949,7 @@ "node_modules/regenerator-runtime": { "version": "0.13.9", "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", - "dev": true + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" }, "node_modules/regenerator-transform": { "version": "0.15.0", @@ -23170,11 +23511,30 @@ "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sass": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.49.0.tgz", + "integrity": "sha512-TVwVdNDj6p6b4QymJtNtRS2YtLJ/CqZriGg0eIAbAKMlN8Xy6kbv33FsEZSF7FufFFM705SQviHjjThfaQ4VNw==", + "dev": true, + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/sass-graph": { "version": "2.2.5", "resolved": "https://registry.npmmirror.com/sass-graph/-/sass-graph-2.2.5.tgz", "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.0.0", "lodash": "^4.0.0", @@ -23189,13 +23549,17 @@ "version": "7.0.3", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/sass-graph/node_modules/string-width": { "version": "3.1.0", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", @@ -23210,6 +23574,8 @@ "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "cliui": "^5.0.0", "find-up": "^3.0.0", @@ -23228,6 +23594,8 @@ "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -23362,6 +23730,8 @@ "resolved": "https://registry.npmmirror.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", "integrity": "sha512-dYE8LhncfBUar6POCxMTm0Ln+erjeczqEvCJib5/7XNkdw1FkUGgwMPY360FY0FgPWQxHWCx29Jl3oejyGLM9Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "js-base64": "^2.1.8", "source-map": "^0.4.2" @@ -23372,6 +23742,8 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.4.4.tgz", "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "amdefine": ">=0.0.4" }, @@ -23379,6 +23751,11 @@ "node": ">=0.8.0" } }, + "node_modules/select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" + }, "node_modules/select-hose": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", @@ -23968,6 +24345,11 @@ "node": ">=0.10.0" } }, + "node_modules/sortablejs": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.10.2.tgz", + "integrity": "sha512-YkPGufevysvfwn5rfdlGyrGjt7/CRHwvRPogD/lC+TnvcN29jDpCifKP+rBqf+LRldfXSTh+0CGLcSg0VIxq3A==" + }, "node_modules/source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/source-list-map/-/source-list-map-2.0.1.tgz", @@ -23981,6 +24363,15 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-resolve": { "version": "0.5.3", "resolved": "https://registry.npmmirror.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz", @@ -24369,6 +24760,8 @@ "resolved": "https://registry.npmmirror.com/stdout-stream/-/stdout-stream-1.4.1.tgz", "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.1" } @@ -25332,6 +25725,11 @@ "integrity": "sha512-qsdtZH+vMoCARQtyod4imc2nIJwg9Cc7lPRrw9CzF8ZKR0khdr8+2nX80PBhET3tcyTtJDxAffGh2rXH4tyU8A==", "dev": true }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "node_modules/tinycolor2": { "version": "1.4.2", "resolved": "https://registry.npmmirror.com/tinycolor2/-/tinycolor2-1.4.2.tgz", @@ -25474,6 +25872,8 @@ "resolved": "https://registry.npmmirror.com/true-case-path/-/true-case-path-1.0.3.tgz", "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.2" } @@ -26018,6 +26418,61 @@ "resolved": "https://registry.npmmirror.com/vm-browserify/-/vm-browserify-1.1.2.tgz", "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" }, + "node_modules/vscode-jsonrpc": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz", + "integrity": "sha512-JvONPptw3GAQGXlVV2utDcHx0BiY34FupW/kI6mZ5x06ER5DdPG/tXWMVHjTNULF5uKPOUUD0SaXg5QaubJL0A==", + "engines": { + "node": ">=8.0.0 || >=10.0.0" + } + }, + "node_modules/vscode-languageclient": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-6.1.4.tgz", + "integrity": "sha512-EUOU+bJu6axmt0RFNo3nrglQLPXMfanbYViJee3Fbn2VuQoX0ZOI4uTYhSRvYLP2vfwTP/juV62P/mksCdTZMA==", + "dependencies": { + "semver": "^6.3.0", + "vscode-languageserver-protocol": "3.15.3" + }, + "engines": { + "vscode": "^1.41.0" + } + }, + "node_modules/vscode-languageclient/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.15.3.tgz", + "integrity": "sha512-zrMuwHOAQRhjDSnflWdJG+O2ztMWss8GqUUB8dXLR/FPenwkiBNkMIJJYfSN6sgskvsF0rHAoBowNQfbyZnnvw==", + "dependencies": { + "vscode-jsonrpc": "^5.0.1", + "vscode-languageserver-types": "3.15.1" + } + }, + "node_modules/vscode-languageserver-types": { + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", + "integrity": "sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ==" + }, + "node_modules/vscode-uri": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz", + "integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==" + }, + "node_modules/vscode-ws-jsonrpc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/vscode-ws-jsonrpc/-/vscode-ws-jsonrpc-0.2.0.tgz", + "integrity": "sha512-NE9HNRgPjCaPyTJvIudcpyIWPImxwRDtuTX16yks7SAiZgSXigxAiZOvSvVBGmD1G/OMfrFo6BblOtjVR9DdVA==", + "dependencies": { + "vscode-jsonrpc": "^5.0.0" + } + }, "node_modules/vue": { "version": "2.6.12", "resolved": "https://registry.npmmirror.com/vue/-/vue-2.6.12.tgz", @@ -26152,6 +26607,14 @@ "integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==", "dev": true }, + "node_modules/vuedraggable": { + "version": "2.24.3", + "resolved": "https://registry.npmjs.org/vuedraggable/-/vuedraggable-2.24.3.tgz", + "integrity": "sha512-6/HDXi92GzB+Hcs9fC6PAAozK1RLt1ewPTLjK0anTYguXLAeySDmcnqE8IC0xa7shvSzRjQXq3/+dsZ7ETGF3g==", + "dependencies": { + "sortablejs": "1.10.2" + } + }, "node_modules/vuescroll": { "version": "4.16.1", "resolved": "https://registry.npmmirror.com/vuescroll/-/vuescroll-4.16.1.tgz", @@ -26160,6 +26623,11 @@ "vue": "^2.0.0" } }, + "node_modules/watch-size": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watch-size/-/watch-size-2.0.0.tgz", + "integrity": "sha512-M92R89dNoTPWyCD+HuUEDdhaDnh9jxPGOwlDc0u51jAgmjUvzqaEMynXSr3BaWs+QdHYk4KzibPy1TFtjLmOZQ==" + }, "node_modules/watchpack": { "version": "1.7.5", "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-1.7.5.tgz", @@ -27225,6 +27693,21 @@ "async-limiter": "~1.0.0" } }, + "node_modules/xss": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/xss/-/xss-1.0.15.tgz", + "integrity": "sha512-FVdlVVC67WOIPvfOwhoMETV72f6GbW7aOabBC3WxN/oUdoEMDyLz4OgRv5/gck2ZeNqEQu+Tb0kloovXOfpYVg==", + "dependencies": { + "commander": "^2.20.3", + "cssfilter": "0.0.10" + }, + "bin": { + "xss": "bin/xss" + }, + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", @@ -27474,6 +27957,115 @@ "version": "4.3.2", "resolved": "https://registry.npmmirror.com/zrender/-/zrender-4.3.2.tgz", "integrity": "sha512-bIusJLS8c4DkIcdiK+s13HiQ/zjQQVgpNohtd8d94Y2DnJqgM1yjh/jpDb8DoL6hd7r8Awagw8e3qK/oLaWr3g==" + }, + "packages/apiServices": { + "name": "@dataspherestudio/apiServices", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/cyeditor": { + "name": "@dataspherestudio/cyeditor", + "version": "0.0.1", + "dependencies": { + "cytoscape": "3.27.0", + "cytoscape-klay": "^3.1.4" + }, + "devDependencies": {} + }, + "packages/dataGovernance": { + "name": "@dataspherestudio/dataGovernance", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "clipboard": "2.0.8", + "jquery": "3.6.0" + } + }, + "packages/dolphinScheduler": { + "name": "@dataspherestudio/dolphinScheduler", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "bootstrap": "^3.4.1", + "d3": "3.5.17", + "dayjs": "1.10.7", + "moment-timezone": "^0.5.34" + } + }, + "packages/dss": { + "name": "@dataspherestudio/dss", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + }, + "devDependencies": { + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0", + "babel-plugin-transform-remove-console": "6.9.4" + } + }, + "packages/editor": { + "name": "@dataspherestudio/editor", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "dt-sql-parser": "1.2.1" + } + }, + "packages/editorLsp": { + "name": "@dataspherestudio/editor-lsp", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "reconnecting-websocket": "4.4.0", + "vscode-ws-jsonrpc": "0.2.0" + } + }, + "packages/exts": { + "name": "@dataspherestudio/exts", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/scriptis": { + "name": "@dataspherestudio/scriptis", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "vuedraggable": "2.24.3" + } + }, + "packages/shared": { + "name": "@dataspherestudio/shared", + "version": "1.1.19", + "dependencies": { + "dexie": "3.0.4" + } + }, + "packages/workflows": { + "name": "@dataspherestudio/workflows", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/cyeditor": "^0.0.1", + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/workspace": { + "name": "@dataspherestudio/workspace", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "@riophae/vue-treeselect": "0.4.0", + "highlight.js": "9.18.3", + "mavon-editor": "2.10.4" + } } }, "dependencies": { @@ -28379,7 +28971,6 @@ "version": "7.17.9", "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.17.9.tgz", "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "dev": true, "requires": { "regenerator-runtime": "^0.13.4" } @@ -28450,6 +29041,100 @@ "to-fast-properties": "^2.0.0" } }, + "@dataspherestudio/apiServices": { + "version": "file:packages/apiServices", + "requires": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/cyeditor": { + "version": "file:packages/cyeditor", + "requires": { + "cytoscape": "3.27.0", + "cytoscape-klay": "^3.1.4" + } + }, + "@dataspherestudio/dataGovernance": { + "version": "file:packages/dataGovernance", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "clipboard": "2.0.8", + "jquery": "3.6.0" + } + }, + "@dataspherestudio/dolphinScheduler": { + "version": "file:packages/dolphinScheduler", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "bootstrap": "^3.4.1", + "d3": "3.5.17", + "dayjs": "1.10.7", + "moment-timezone": "^0.5.34" + } + }, + "@dataspherestudio/dss": { + "version": "file:packages/dss", + "requires": { + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19", + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0", + "babel-plugin-transform-remove-console": "6.9.4" + } + }, + "@dataspherestudio/editor": { + "version": "file:packages/editor", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "dt-sql-parser": "1.2.1" + } + }, + "@dataspherestudio/editor-lsp": { + "version": "file:packages/editorLsp", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "reconnecting-websocket": "4.4.0", + "vscode-ws-jsonrpc": "0.2.0" + } + }, + "@dataspherestudio/exts": { + "version": "file:packages/exts", + "requires": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/scriptis": { + "version": "file:packages/scriptis", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "vuedraggable": "2.24.3" + } + }, + "@dataspherestudio/shared": { + "version": "file:packages/shared", + "requires": { + "dexie": "3.0.4" + } + }, + "@dataspherestudio/workflows": { + "version": "file:packages/workflows", + "requires": { + "@dataspherestudio/cyeditor": "^0.0.1", + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/workspace": { + "version": "file:packages/workspace", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "@riophae/vue-treeselect": "0.4.0", + "highlight.js": "9.18.3", + "mavon-editor": "2.10.4" + } + }, "@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -32074,6 +32759,21 @@ "@octokit/openapi-types": "^11.2.0" } }, + "@riophae/vue-treeselect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@riophae/vue-treeselect/-/vue-treeselect-0.4.0.tgz", + "integrity": "sha512-J4atYmBqXQmiPFK/0B5sXKjtnGc21mBJEiyKIDZwk0Q9XuynVFX6IJ4EpaLmUgL5Tve7HAS7wkiGGSti6Uaxcg==", + "requires": { + "@babel/runtime": "^7.3.1", + "babel-helper-vue-jsx-merge-props": "^2.0.3", + "easings-css": "^1.0.0", + "fuzzysearch": "^1.0.3", + "is-promise": "^2.1.0", + "lodash": "^4.0.0", + "material-colors": "^1.2.6", + "watch-size": "^2.0.0" + } + }, "@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmmirror.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -33071,7 +33771,9 @@ "version": "1.0.1", "resolved": "https://registry.npmmirror.com/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "ansi-colors": { "version": "3.2.4", @@ -33121,7 +33823,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "optional": true, + "devOptional": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -33379,7 +34081,9 @@ "version": "0.1.3", "resolved": "https://registry.npmmirror.com/async-foreach/-/async-foreach-0.1.3.tgz", "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "async-limiter": { "version": "1.0.1", @@ -33525,6 +34229,11 @@ "resolve": "^1.12.0" } }, + "babel-helper-vue-jsx-merge-props": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-2.0.3.tgz", + "integrity": "sha512-gsLiKK7Qrb7zYJNgiXKpXblxbV5ffSwR0f5whkPAaBAR4fhi6bwRZxX9wBlIc5M/v8CCkXUbXZL4N/nSE97cqg==" + }, "babel-loader": { "version": "8.2.5", "resolved": "https://registry.npmmirror.com/babel-loader/-/babel-loader-8.2.5.tgz", @@ -33696,6 +34405,12 @@ "@babel/helper-define-polyfill-provider": "^0.3.1" } }, + "babel-plugin-transform-remove-console": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-remove-console/-/babel-plugin-transform-remove-console-6.9.4.tgz", + "integrity": "sha512-88blrUrMX3SPiGkT1GnvVY8E/7A+k6oj3MNvUtTIxJflFzXTw1bHkuJ/y039ouhFMp2prRn5cQGzokViYi1dsg==", + "dev": true + }, "babel-polyfill": { "version": "6.26.0", "resolved": "https://registry.npmmirror.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz", @@ -33836,7 +34551,7 @@ "version": "2.2.0", "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "optional": true + "devOptional": true }, "bindings": { "version": "1.5.0", @@ -33949,6 +34664,11 @@ "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "bootstrap": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", + "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -34468,7 +35188,7 @@ "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "optional": true, + "devOptional": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -34484,7 +35204,7 @@ "version": "3.0.2", "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "requires": { "fill-range": "^7.0.1" } @@ -34493,7 +35213,7 @@ "version": "7.0.1", "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "requires": { "to-regex-range": "^5.0.1" } @@ -34502,13 +35222,13 @@ "version": "7.0.0", "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true + "devOptional": true }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "requires": { "is-number": "^7.0.0" } @@ -34737,6 +35457,16 @@ "dev": true, "optional": true }, + "clipboard": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-2.0.8.tgz", + "integrity": "sha512-Y6WO0unAIQp5bLmk1zdThRhgJt/x3ks6f30s3oE3H1mgIEU33XyQjEf8gsf6DxC7NPX8Y1SsNWjUjL/ywLnnbQ==", + "requires": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "clipboardy": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/clipboardy/-/clipboardy-2.3.0.tgz", @@ -36346,6 +37076,11 @@ "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", "dev": true }, + "cssfilter": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", + "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==" + }, "cssnano": { "version": "4.1.11", "resolved": "https://registry.npmmirror.com/cssnano/-/cssnano-4.1.11.tgz", @@ -36463,6 +37198,35 @@ "resolved": "https://registry.npmmirror.com/cyclist/-/cyclist-1.0.1.tgz", "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" }, + "cytoscape": { + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.27.0.tgz", + "integrity": "sha512-pPZJilfX9BxESwujODz5pydeGi+FBrXq1rcaB1mfhFXXFJ9GjE6CNndAk+8jPzoXGD+16LtSS4xlYEIUiW4Abg==", + "requires": { + "heap": "^0.2.6", + "lodash": "^4.17.21" + }, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + } + } + }, + "cytoscape-klay": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cytoscape-klay/-/cytoscape-klay-3.1.4.tgz", + "integrity": "sha512-VwPj0VR25GPfy6qXVQRi/MYlZM/zkdvRhHlgqbM//lSvstgM6fhp3ik/uM8Wr8nlhskfqz/M1fIDmR6UckbS2A==", + "requires": { + "klayjs": "^0.4.1" + } + }, + "d3": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", + "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==" + }, "d3-dispatch": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-2.0.0.tgz", @@ -36524,6 +37288,11 @@ "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", "dev": true }, + "dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "de-indent": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/de-indent/-/de-indent-1.0.2.tgz", @@ -36818,6 +37587,11 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz", @@ -36863,6 +37637,11 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "dexie": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.0.4.tgz", + "integrity": "sha512-rwS3k8BBstjGFJAS/yjYrsRZucqitnrP3NBhqghl9ihWcWABlT3I5f2dZV9T3pmXGAP7G9p1q00g/axcsuOT8A==" + }, "dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/dezalgo/-/dezalgo-1.0.4.tgz", @@ -37026,6 +37805,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "dt-sql-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/dt-sql-parser/-/dt-sql-parser-1.2.1.tgz", + "integrity": "sha512-bp6Rtm8N3m2Z3tavtKGo8J5ffaPQdPodYV2xGi9f6IsWbHzOkpNBxHlORcWtnSLtA/JvZcvImQxgbkdaGiv5cw==" + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -37043,6 +37827,11 @@ "stream-shift": "^1.0.0" } }, + "easings-css": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/easings-css/-/easings-css-1.0.0.tgz", + "integrity": "sha512-7Uq7NdazNfVtr0RNmPAys8it0zKCuaqxJStYKEl72D3j4gbvXhhaM7iWNbqhA4C94ygCye6VuyhzBRQC4szeBg==" + }, "easy-stack": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/easy-stack/-/easy-stack-1.0.1.tgz", @@ -38896,6 +39685,11 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "fuzzysearch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fuzzysearch/-/fuzzysearch-1.0.3.tgz", + "integrity": "sha512-s+kNWQuI3mo9OALw0HJ6YGmMbLqEufCh2nX/zzV5CrICQ/y4AwPxM+6TIiF9ItFCHXFCyM/BfCCmN57NTIJuPg==" + }, "g-status": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/g-status/-/g-status-2.0.2.tgz", @@ -38973,6 +39767,8 @@ "resolved": "https://registry.npmmirror.com/gaze/-/gaze-1.1.3.tgz", "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", "dev": true, + "optional": true, + "peer": true, "requires": { "globule": "^1.0.0" } @@ -39252,8 +40048,7 @@ "glob-to-regexp": { "version": "0.3.0", "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", - "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==", - "dev": true + "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==" }, "glob2base": { "version": "0.0.12", @@ -39299,6 +40094,8 @@ "resolved": "https://registry.npmmirror.com/globule/-/globule-1.3.3.tgz", "integrity": "sha512-mb1aYtDbIjTu4ShMB85m3UzjX9BVKe9WCzsnfMSZk+K5GpIbBOexgg4PPCt5eHDEG5/ZQAUX2Kct02zfiPLsKg==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "~7.1.1", "lodash": "~4.17.10", @@ -39310,6 +40107,8 @@ "resolved": "https://registry.npmmirror.com/glob/-/glob-7.1.7.tgz", "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -39324,12 +40123,22 @@ "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.0.8.tgz", "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, + "optional": true, + "peer": true, "requires": { "brace-expansion": "^1.1.7" } } } }, + "good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha512-goW1b+d9q/HIwbVYZzZ6SsTr4IgE+WA44A0GmPIQstuOrgsFcT7VEJ48nmr9GaRtNu0XTKacFLGnBPAM6Afouw==", + "requires": { + "delegate": "^3.1.2" + } + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -39541,12 +40350,22 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, + "heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "hex-color-regex": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz", "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==", "dev": true }, + "highlight.js": { + "version": "9.18.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.3.tgz", + "integrity": "sha512-zBZAmhSupHIl5sITeMqIJnYCDfAEc3Gdkqj65wC1lpI468MMQeeQkhcIAvk+RylAkxrCcI9xy9piHiXeQ1BdzQ==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -39993,6 +40812,12 @@ "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", "dev": true }, + "immutable": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", + "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==", + "dev": true + }, "import-cwd": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/import-cwd/-/import-cwd-2.1.0.tgz", @@ -40330,7 +41155,7 @@ "version": "2.1.0", "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "optional": true, + "devOptional": true, "requires": { "binary-extensions": "^2.0.0" } @@ -40587,8 +41412,7 @@ "is-promise": { "version": "2.2.2", "resolved": "https://registry.npmmirror.com/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", - "dev": true + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" }, "is-regex": { "version": "1.1.4", @@ -40735,8 +41559,7 @@ "jquery": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/jquery/-/jquery-3.6.0.tgz", - "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==", - "peer": true + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, "js-base64": { "version": "2.6.4", @@ -40901,6 +41724,11 @@ "graceful-fs": "^4.1.11" } }, + "klayjs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/klayjs/-/klayjs-0.4.1.tgz", + "integrity": "sha512-WUNxuO7O79TEkxCj6OIaK5TJBkaWaR/IKNTakgV9PwDn+mrr63MLHed34AcE2yTaDntgO6l0zGFIzhcoTeroTA==" + }, "klona": { "version": "2.0.5", "resolved": "https://registry.npmmirror.com/klona/-/klona-2.0.5.tgz", @@ -41966,12 +42794,25 @@ "escape-string-regexp": "^1.0.4" } }, + "material-colors": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", + "integrity": "sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==" + }, "math-random": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/math-random/-/math-random-1.0.4.tgz", "integrity": "sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==", "dev": true }, + "mavon-editor": { + "version": "2.10.4", + "resolved": "https://registry.npmjs.org/mavon-editor/-/mavon-editor-2.10.4.tgz", + "integrity": "sha512-CFsBLkgt/KZBDg+SJYe2fyYv4zClY149PiwpH0rDAiiP4ae1XNs0GC8nBsoTeipsHcebDLN1QMkt3bUsnMDjQw==", + "requires": { + "xss": "^1.0.6" + } + }, "md5": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz", @@ -42643,6 +43484,21 @@ "resolved": "https://registry.npmmirror.com/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "moment-timezone": { + "version": "0.5.45", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", + "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", + "requires": { + "moment": "^2.29.4" + }, + "dependencies": { + "moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" + } + } + }, "monaco-editor": { "version": "0.19.3", "resolved": "https://registry.npmmirror.com/monaco-editor/-/monaco-editor-0.19.3.tgz", @@ -42657,6 +43513,17 @@ "loader-utils": "^1.2.3" } }, + "monaco-languageclient": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-0.13.0.tgz", + "integrity": "sha512-aCwd33dTitwV5QwY56rpYHwzEGXei8TZ+yvZcvP3gEMd6Mizr8m3pOuoknDi2SUfLuNAHS6+ulvLgZlNQB5awg==", + "requires": { + "glob-to-regexp": "^0.3.0", + "vscode-jsonrpc": "^5.0.0", + "vscode-languageclient": "^6.0.0", + "vscode-uri": "^2.1.1" + } + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -42768,7 +43635,7 @@ "version": "2.15.0", "resolved": "https://registry.npmmirror.com/nan/-/nan-2.15.0.tgz", "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "devOptional": true + "optional": true }, "nanomatch": { "version": "1.2.13", @@ -42950,6 +43817,8 @@ "resolved": "https://registry.npmmirror.com/node-sass/-/node-sass-6.0.1.tgz", "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "async-foreach": "^0.1.3", "chalk": "^1.1.1", @@ -42972,19 +43841,25 @@ "version": "2.1.1", "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", "dev": true, + "optional": true, + "peer": true, "requires": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -42997,13 +43872,17 @@ "version": "2.0.0", "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, + "optional": true, + "peer": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -43015,6 +43894,8 @@ "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, + "optional": true, + "peer": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -43025,6 +43906,8 @@ "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "dev": true, + "optional": true, + "peer": true, "requires": { "minipass": "^3.0.0" } @@ -43033,13 +43916,17 @@ "version": "4.0.1", "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "hosted-git-info": { "version": "4.1.0", "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, + "optional": true, + "peer": true, "requires": { "lru-cache": "^6.0.0" } @@ -43049,6 +43936,8 @@ "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, + "optional": true, + "peer": true, "requires": { "p-locate": "^4.1.0" } @@ -43058,6 +43947,8 @@ "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, + "optional": true, + "peer": true, "requires": { "yallist": "^4.0.0" } @@ -43067,6 +43958,8 @@ "resolved": "https://registry.npmmirror.com/meow/-/meow-9.0.0.tgz", "integrity": "sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "@types/minimist": "^1.2.0", "camelcase-keys": "^6.2.2", @@ -43087,6 +43980,8 @@ "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.4.tgz", "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", "dev": true, + "optional": true, + "peer": true, "requires": { "yallist": "^4.0.0" } @@ -43096,6 +43991,8 @@ "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", "dev": true, + "optional": true, + "peer": true, "requires": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -43105,13 +44002,17 @@ "version": "1.0.4", "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node-gyp": { "version": "7.1.2", "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-7.1.2.tgz", "integrity": "sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "env-paths": "^2.2.0", "glob": "^7.1.4", @@ -43130,6 +44031,8 @@ "resolved": "https://registry.npmmirror.com/nopt/-/nopt-5.0.0.tgz", "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "abbrev": "1" } @@ -43139,6 +44042,8 @@ "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, + "optional": true, + "peer": true, "requires": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -43151,6 +44056,8 @@ "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, + "optional": true, + "peer": true, "requires": { "p-limit": "^2.2.0" } @@ -43159,19 +44066,25 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "read-pkg-up": { "version": "7.0.1", "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz", "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", "dev": true, + "optional": true, + "peer": true, "requires": { "find-up": "^4.1.0", "read-pkg": "^5.2.0", @@ -43182,7 +44095,9 @@ "version": "0.8.1", "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", - "dev": true + "dev": true, + "optional": true, + "peer": true } } }, @@ -43191,6 +44106,8 @@ "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.1.3" } @@ -43200,6 +44117,8 @@ "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, + "peer": true, "requires": { "lru-cache": "^6.0.0" } @@ -43209,6 +44128,8 @@ "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "optional": true, + "peer": true, "requires": { "shebang-regex": "^3.0.0" } @@ -43217,13 +44138,17 @@ "version": "3.0.0", "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", "dev": true, + "optional": true, + "peer": true, "requires": { "ansi-regex": "^2.0.0" } @@ -43232,13 +44157,17 @@ "version": "2.0.0", "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "tar": { "version": "6.1.12", "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.12.tgz", "integrity": "sha512-jU4TdemS31uABHd+Lt5WEYJuzn+TJTCBLljvIAHZOz6M9Os5pJ4dD+vRFLxPa/n3T0iEFzpi+0x1UfuDZYbRMw==", "dev": true, + "optional": true, + "peer": true, "requires": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -43252,13 +44181,17 @@ "version": "0.18.1", "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.18.1.tgz", "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "which": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "optional": true, + "peer": true, "requires": { "isexe": "^2.0.0" } @@ -43267,7 +44200,9 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "optional": true, + "peer": true } } }, @@ -45722,11 +46657,16 @@ "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "optional": true, + "devOptional": true, "requires": { "picomatch": "^2.2.1" } }, + "reconnecting-websocket": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/reconnecting-websocket/-/reconnecting-websocket-4.4.0.tgz", + "integrity": "sha512-D2E33ceRPga0NvTDhJmphEgJ7FUYF0v4lr1ki0csq06OdlxKfugGzN0dSkxM/NfqCxYELK4KcaTOUOjTV6Dcng==" + }, "redent": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/redent/-/redent-3.0.0.tgz", @@ -45766,8 +46706,7 @@ "regenerator-runtime": { "version": "0.13.9", "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", - "dev": true + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" }, "regenerator-transform": { "version": "0.15.0", @@ -46233,11 +47172,24 @@ "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sass": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.49.0.tgz", + "integrity": "sha512-TVwVdNDj6p6b4QymJtNtRS2YtLJ/CqZriGg0eIAbAKMlN8Xy6kbv33FsEZSF7FufFFM705SQviHjjThfaQ4VNw==", + "dev": true, + "requires": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + } + }, "sass-graph": { "version": "2.2.5", "resolved": "https://registry.npmmirror.com/sass-graph/-/sass-graph-2.2.5.tgz", "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.0.0", "lodash": "^4.0.0", @@ -46249,13 +47201,17 @@ "version": "7.0.3", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, + "optional": true, + "peer": true, "requires": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", @@ -46267,6 +47223,8 @@ "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", "dev": true, + "optional": true, + "peer": true, "requires": { "cliui": "^5.0.0", "find-up": "^3.0.0", @@ -46285,6 +47243,8 @@ "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "dev": true, + "optional": true, + "peer": true, "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -46379,6 +47339,8 @@ "resolved": "https://registry.npmmirror.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", "integrity": "sha512-dYE8LhncfBUar6POCxMTm0Ln+erjeczqEvCJib5/7XNkdw1FkUGgwMPY360FY0FgPWQxHWCx29Jl3oejyGLM9Q==", "dev": true, + "optional": true, + "peer": true, "requires": { "js-base64": "^2.1.8", "source-map": "^0.4.2" @@ -46389,12 +47351,19 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.4.4.tgz", "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", "dev": true, + "optional": true, + "peer": true, "requires": { "amdefine": ">=0.0.4" } } } }, + "select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" + }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", @@ -46886,6 +47855,11 @@ "is-plain-obj": "^1.0.0" } }, + "sortablejs": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.10.2.tgz", + "integrity": "sha512-YkPGufevysvfwn5rfdlGyrGjt7/CRHwvRPogD/lC+TnvcN29jDpCifKP+rBqf+LRldfXSTh+0CGLcSg0VIxq3A==" + }, "source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/source-list-map/-/source-list-map-2.0.1.tgz", @@ -46896,6 +47870,12 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.5.7.tgz", "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" }, + "source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "dev": true + }, "source-map-resolve": { "version": "0.5.3", "resolved": "https://registry.npmmirror.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz", @@ -47215,6 +48195,8 @@ "resolved": "https://registry.npmmirror.com/stdout-stream/-/stdout-stream-1.4.1.tgz", "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", "dev": true, + "optional": true, + "peer": true, "requires": { "readable-stream": "^2.0.1" } @@ -48006,6 +48988,11 @@ "integrity": "sha512-qsdtZH+vMoCARQtyod4imc2nIJwg9Cc7lPRrw9CzF8ZKR0khdr8+2nX80PBhET3tcyTtJDxAffGh2rXH4tyU8A==", "dev": true }, + "tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "tinycolor2": { "version": "1.4.2", "resolved": "https://registry.npmmirror.com/tinycolor2/-/tinycolor2-1.4.2.tgz", @@ -48117,6 +49104,8 @@ "resolved": "https://registry.npmmirror.com/true-case-path/-/true-case-path-1.0.3.tgz", "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.1.2" } @@ -48582,6 +49571,54 @@ "resolved": "https://registry.npmmirror.com/vm-browserify/-/vm-browserify-1.1.2.tgz", "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" }, + "vscode-jsonrpc": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz", + "integrity": "sha512-JvONPptw3GAQGXlVV2utDcHx0BiY34FupW/kI6mZ5x06ER5DdPG/tXWMVHjTNULF5uKPOUUD0SaXg5QaubJL0A==" + }, + "vscode-languageclient": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-6.1.4.tgz", + "integrity": "sha512-EUOU+bJu6axmt0RFNo3nrglQLPXMfanbYViJee3Fbn2VuQoX0ZOI4uTYhSRvYLP2vfwTP/juV62P/mksCdTZMA==", + "requires": { + "semver": "^6.3.0", + "vscode-languageserver-protocol": "3.15.3" + }, + "dependencies": { + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==" + } + } + }, + "vscode-languageserver-protocol": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.15.3.tgz", + "integrity": "sha512-zrMuwHOAQRhjDSnflWdJG+O2ztMWss8GqUUB8dXLR/FPenwkiBNkMIJJYfSN6sgskvsF0rHAoBowNQfbyZnnvw==", + "requires": { + "vscode-jsonrpc": "^5.0.1", + "vscode-languageserver-types": "3.15.1" + } + }, + "vscode-languageserver-types": { + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", + "integrity": "sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ==" + }, + "vscode-uri": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz", + "integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==" + }, + "vscode-ws-jsonrpc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/vscode-ws-jsonrpc/-/vscode-ws-jsonrpc-0.2.0.tgz", + "integrity": "sha512-NE9HNRgPjCaPyTJvIudcpyIWPImxwRDtuTX16yks7SAiZgSXigxAiZOvSvVBGmD1G/OMfrFo6BblOtjVR9DdVA==", + "requires": { + "vscode-jsonrpc": "^5.0.0" + } + }, "vue": { "version": "2.6.12", "resolved": "https://registry.npmmirror.com/vue/-/vue-2.6.12.tgz", @@ -48697,12 +49734,25 @@ "integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==", "dev": true }, + "vuedraggable": { + "version": "2.24.3", + "resolved": "https://registry.npmjs.org/vuedraggable/-/vuedraggable-2.24.3.tgz", + "integrity": "sha512-6/HDXi92GzB+Hcs9fC6PAAozK1RLt1ewPTLjK0anTYguXLAeySDmcnqE8IC0xa7shvSzRjQXq3/+dsZ7ETGF3g==", + "requires": { + "sortablejs": "1.10.2" + } + }, "vuescroll": { "version": "4.16.1", "resolved": "https://registry.npmmirror.com/vuescroll/-/vuescroll-4.16.1.tgz", "integrity": "sha512-7fRsG2Yw5Z07LUz/IIu9barpmYiN9q+ZTC+CrVamvCbmsxyhz8mU1OuYFbfORysaUskioNMxTGDo+HOzeDfSyQ==", "requires": {} }, + "watch-size": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watch-size/-/watch-size-2.0.0.tgz", + "integrity": "sha512-M92R89dNoTPWyCD+HuUEDdhaDnh9jxPGOwlDc0u51jAgmjUvzqaEMynXSr3BaWs+QdHYk4KzibPy1TFtjLmOZQ==" + }, "watchpack": { "version": "1.7.5", "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-1.7.5.tgz", @@ -49573,6 +50623,15 @@ "async-limiter": "~1.0.0" } }, + "xss": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/xss/-/xss-1.0.15.tgz", + "integrity": "sha512-FVdlVVC67WOIPvfOwhoMETV72f6GbW7aOabBC3WxN/oUdoEMDyLz4OgRv5/gck2ZeNqEQu+Tb0kloovXOfpYVg==", + "requires": { + "commander": "^2.20.3", + "cssfilter": "0.0.10" + } + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", diff --git a/web/package.json b/web/package.json index ed9ef9533..be84dc4c5 100644 --- a/web/package.json +++ b/web/package.json @@ -38,6 +38,7 @@ "md5": "2.3.0", "moment": "2.29.4", "monaco-editor": "0.19.3", + "monaco-languageclient": "^0.13.0", "qs": "6.9.4", "svgo": "1.3.0", "vue": "2.6.12", @@ -65,27 +66,28 @@ "less-loader": "6.1.0", "lint-staged": "8.2.1", "monaco-editor-webpack-plugin": "1.8.2", - "node-sass": "6.0.1", + "patch-package": "6.2.2", + "sass": "1.49.0", "sass-loader": "10.2.1", "speed-measure-webpack-plugin": "1.5.0", "svg-sprite-loader": "5.0.0", - "patch-package": "6.2.2", "vue-cli-plugin-mockjs": "0.1.3", "vue-template-compiler": "2.6.12", "webpack-virtual-modules": "0.3.2" }, "workspaces": [ - "packages/apiServices", - "packages/cyeditor", - "packages/dataGovernance", - "packages/dss", - "packages/editor", - "packages/editorLsp", - "packages/exts", - "packages/scheduleCenter", - "packages/scriptis", - "packages/shared", - "packages/workflows", - "packages/workspace" + "packages/apiServices", + "packages/cyeditor", + "packages/dataGovernance", + "packages/dss", + "packages/editor", + "packages/editorLsp", + "packages/exts", + "packages/scheduleCenter", + "packages/dolphinScheduler", + "packages/scriptis", + "packages/shared", + "packages/workflows", + "packages/workspace" ] } diff --git a/web/packages/apiServices/module/apiServicesExecute/result.vue b/web/packages/apiServices/module/apiServicesExecute/result.vue index 024b9d1c1..3e817d9a1 100644 --- a/web/packages/apiServices/module/apiServicesExecute/result.vue +++ b/web/packages/apiServices/module/apiServicesExecute/result.vue @@ -155,15 +155,14 @@ export default { }, // 验证发布和更新的默认值是否满足条件 verificationValue (row) { + const method = this.workInfo.method || ''; let flag; - if(row.defaultValue.length > 1024 && Number(row.type) !== 4) { - this.$Message.error({ content: this.$t('message.apiServices.more1024') }); - flag = true; - } else if(row.type == 4 && row.defaultValue.split('\n').length > 5000) { - this.$Message.error({ content: this.$t('message.apiServices.moreline') }); + if (method.toUpperCase() === 'GET' && row.defaultValue.length > 500) { + this.$Message.error({ content: '不能超过500字符' }); flag = true; } else { - flag = false + flag = false; + } console.warn(row) this.$set(this.tip, row.name, flag) @@ -211,6 +210,15 @@ export default { if (this.showConditionList.length > 0) { this.$refs.searchFrom.validate(valid => { if (valid) { + const method = this.workInfo.method || ''; + if (method.toUpperCase() === 'GET') { + const conditionStr = this.showConditionList.map(item => item.defaultValue).filter(item => item).join(''); + if (conditionStr.length > 3000) { + this.$Message.error({ content: '输入内容总长度不能超过3000字符' }); + return; + } + } + this.executAction(); } else { console.log(this.showConditionList) @@ -337,6 +345,8 @@ export default { margin: 0 25px; .content-top { padding: 25px; + max-height: 600px; + overflow-y: auto; .alert-bar { margin-left: 15px; } @@ -347,5 +357,4 @@ export default { } } - - \ No newline at end of file + diff --git a/web/packages/cyeditor/cyeditor-context-menu/index.js b/web/packages/cyeditor/cyeditor-context-menu/index.js index 11b543351..05c7d6e00 100644 --- a/web/packages/cyeditor/cyeditor-context-menu/index.js +++ b/web/packages/cyeditor/cyeditor-context-menu/index.js @@ -61,10 +61,13 @@ class ContextMenu { this._listeners.eventCyTap = (event) => { this.renderedPos = event.renderedPosition || event.cyRenderedPosition let left = this.renderedPos.x + if (this._options.editorOptions.dragAddNodes) { + left += 200 + } let top = this.renderedPos.y utils.css(this.ctxmenu, { top: top + 'px', - left: (left + 200) + 'px' + left: left + 'px' }) this.show(event) } diff --git a/web/packages/cyeditor/cyeditor-snap-grid/index.js b/web/packages/cyeditor/cyeditor-snap-grid/index.js index 45f622421..f79d2087b 100644 --- a/web/packages/cyeditor/cyeditor-snap-grid/index.js +++ b/web/packages/cyeditor/cyeditor-snap-grid/index.js @@ -7,7 +7,7 @@ class SnapToGrid { constructor (cy, params) { this.cy = cy let defaults = { - stackOrder: -1, + stackOrder: 0, gridSpacing: 35, strokeStyle: '#eeeeee', lineWidth: 1.0, diff --git a/web/packages/cyeditor/example.js b/web/packages/cyeditor/example.js index 0734b0ec1..0b9544764 100644 --- a/web/packages/cyeditor/example.js +++ b/web/packages/cyeditor/example.js @@ -278,6 +278,9 @@ editor.on('addnode', ({ target }) => { console.log('node-add', { ...target, }) + + const data = editor.json() + console.log(data) }) editor.on('addlink', ({ target }) => { @@ -299,7 +302,101 @@ editor.on('dblclick', (e) => { // 方法API editor.json({ boxSelectionEnabled: true, - elements: [], + elements: { + "nodes": [ + { + "data": { + "type": "rect", + "name": "矩形1", + "bg": "#fff", + "width": 150, + "height": 40, + "image": "/img/vite.8e3a10e1.svg", + "id": "9158b849-d468-48ef-b59c-440ccd0c1009" + }, + "position": { + "x": 262.5, + "y": 122.5 + }, + "group": "nodes", + "removed": false, + "selected": false, + "selectable": true, + "locked": false, + "grabbable": true, + "pannable": false, + "classes": "eh-preview-active" + }, + { + "data": { + "type": "rect", + "name": "矩形1", + "bg": "#fff", + "width": 150, + "height": 40, + "image": "/img/vite.8e3a10e1.svg", + "id": "18be7b5d-4925-408f-b99b-6829038489ae" + }, + "position": { + "x": 192.5, + "y": 262.5 + }, + "group": "nodes", + "removed": false, + "selected": false, + "selectable": true, + "locked": false, + "grabbable": true, + "pannable": false, + "classes": "eh-preview-active" + }, + { + "data": { + "type": "rect", + "name": "矩形1", + "bg": "#fff", + "width": 150, + "height": 40, + "image": "/img/vite.8e3a10e1.svg", + "id": "86b161a5-2cac-4047-a1bd-f02514a13032" + }, + "position": { + "x": 157.5, + "y": 332.5 + }, + "group": "nodes", + "removed": false, + "selected": false, + "selectable": true, + "locked": false, + "grabbable": true, + "pannable": false, + "classes": "" + } + ], + "edges": [ + { + "data": { + "source": "9158b849-d468-48ef-b59c-440ccd0c1009", + "target": "18be7b5d-4925-408f-b99b-6829038489ae", + "lineType": "bezier", + "id": "85b93f50-0469-4c79-bb9d-e5e3c99c25cf" + }, + "position": { + "x": 17.5, + "y": 17.5 + }, + "group": "edges", + "removed": false, + "selected": false, + "selectable": true, + "locked": false, + "grabbable": true, + "pannable": true, + "classes": "" + } + ] + }, pan: { x: 0, y: 0 }, panningEnabled: true, userPanningEnabled: true, diff --git a/web/packages/cyeditor/index.js b/web/packages/cyeditor/index.js index 951d52166..42ad7633e 100644 --- a/web/packages/cyeditor/index.js +++ b/web/packages/cyeditor/index.js @@ -218,6 +218,8 @@ class CyEditor extends EventBus { container: this.editorContianer.querySelector('.cy-editor-container .left'), nodeTypes: this.editorOptions.nodeTypes, }) + } else { + this.editorContianer.className += ' hide_shapes_panel' } // snap-grid @@ -235,7 +237,7 @@ class CyEditor extends EventBus { // context-menu if (contextMenu) { - this._plugins.contextMenu = this.cy.contextMenu(contextMenu) + this._plugins.contextMenu = this.cy.contextMenu({editorOptions: this.editorOptions, ...contextMenu}) } } diff --git a/web/packages/cyeditor/package.json b/web/packages/cyeditor/package.json index e9f7722bb..d312b095d 100644 --- a/web/packages/cyeditor/package.json +++ b/web/packages/cyeditor/package.json @@ -7,6 +7,7 @@ }, "devDependencies": {}, "scripts": { - "serve": "vue-cli-service serve example.js" + "serve": "vue-cli-service serve example.js", + "build": "vue-cli-service build --target lib --name cyeditor index.js" } } diff --git a/web/packages/dataGovernance/module/common/tabCard/index.vue b/web/packages/dataGovernance/module/common/tabCard/index.vue index 64220e6c5..467202e2a 100644 --- a/web/packages/dataGovernance/module/common/tabCard/index.vue +++ b/web/packages/dataGovernance/module/common/tabCard/index.vue @@ -210,7 +210,7 @@ export default { } } .content-html { - /deep/ span { + ::v-deep span { color: #3495F7; } } diff --git a/web/packages/dataGovernance/view/fieldInfo/index.vue b/web/packages/dataGovernance/view/fieldInfo/index.vue index 01fb2dbc2..5ea0f1695 100644 --- a/web/packages/dataGovernance/view/fieldInfo/index.vue +++ b/web/packages/dataGovernance/view/fieldInfo/index.vue @@ -30,7 +30,7 @@ @@ -48,7 +48,7 @@ id="copy-button" data-clipboard-action="copy" data-clipboard-target="#copy-button" - @click="e => copy(e, DDLsql)" + @click="(e) => copy(e, DDLsql)" >复制 @@ -76,193 +76,193 @@ - diff --git a/web/packages/dolphinScheduler/assets/styles/workflow.scss b/web/packages/dolphinScheduler/assets/styles/workflow.scss index a44e9eed7..ad468bd2b 100644 --- a/web/packages/dolphinScheduler/assets/styles/workflow.scss +++ b/web/packages/dolphinScheduler/assets/styles/workflow.scss @@ -14,7 +14,7 @@ * limitations under the License. * */ -@import "@dataspherestudio/shared/common/style/variables.scss"; +@import '@dataspherestudio/shared/common/style/variables.scss'; .workflow-wrap { width: 100%; height: 100%; @@ -89,7 +89,7 @@ font-weight: 400; cursor: pointer; &:hover { - background-color: #2E92F7; + background-color: #2e92f7; color: #fff; } } @@ -162,7 +162,7 @@ position: relative; flex: 1; .tabs-content { - /deep/ .ivu-tabs-tab { + ::v-deep .ivu-tabs-tab { font-size: $font-size-large; } } @@ -220,47 +220,51 @@ @include border-color($border-color-base, $dark-border-color-base); padding: 0 5px; .workflow-tabs-item { - margin: 0; - height: 31px; - padding: 5px 16px 4px; - border-bottom: $border-width-base $border-style-base $border-color-base; - @include border-color($border-color-base, $dark-border-color-base); - border-radius: 4px 4px 0 0; - // background: #f8f8f9; - @include bg-color($workflow-body-bg-color, $dark-workflow-body-bg-color); + margin: 0; + height: 31px; + padding: 5px 16px 4px; + border-bottom: $border-width-base $border-style-base + $border-color-base; + @include border-color($border-color-base, $dark-border-color-base); + border-radius: 4px 4px 0 0; + // background: #f8f8f9; + @include bg-color( + $workflow-body-bg-color, + $dark-workflow-body-bg-color + ); + display: inline-block; + cursor: pointer; + position: relative; + &.active { + height: 32px; + padding-bottom: 5px; + background: #fff; + transform: translateZ(0); + border: 1px solid #dcdee2; + border-bottom: 1px solid #fff; + // color: #2d8cf0; + @include font-color($primary-color, $dark-primary-color); + } + .workflow-tabs-name { display: inline-block; - cursor: pointer; + } + .workflow-tabs-close { + width: 22px; + margin-right: -6px; + height: 22px; + font-size: 22px; + color: #999; + text-align: right; + vertical-align: middle; + overflow: hidden; position: relative; - &.active { - height: 32px; - padding-bottom: 5px; - background: #fff; - transform: translateZ(0); - border: 1px solid #dcdee2; - border-bottom:1px solid #fff; - // color: #2d8cf0; - @include font-color($primary-color, $dark-primary-color); - } - .workflow-tabs-name { - display: inline-block; - } - .workflow-tabs-close { - width: 22px; - margin-right: -6px; - height: 22px; - font-size: 22px; - color: #999; - text-align: right; - vertical-align: middle; - overflow: hidden; - position: relative; - top: -1px; - transform-origin: 100% 50%; - transition: all .3s ease-in-out; - cursor: pointer; - } + top: -1px; + transform-origin: 100% 50%; + transition: all 0.3s ease-in-out; + cursor: pointer; + } } + } } } - } } diff --git a/web/packages/dolphinScheduler/components/tabList/index.scss b/web/packages/dolphinScheduler/components/tabList/index.scss index 390a5676e..adede936b 100644 --- a/web/packages/dolphinScheduler/components/tabList/index.scss +++ b/web/packages/dolphinScheduler/components/tabList/index.scss @@ -1,4 +1,4 @@ -@import "@dataspherestudio/shared/common/style/variables.scss"; +@import '@dataspherestudio/shared/common/style/variables.scss'; .workflowTabContainer { margin-left: 250px; transition: margin-left 0.3s; @@ -36,7 +36,7 @@ min-width: 320px; .ivu-breadcrumb { font-size: 21px; - /deep/.ivu-breadcrumb-item-separator { + ::v-deep.ivu-breadcrumb-item-separator { // color: rgba(0,0,0,0.65); @include font-color($light-text-color, $dark-text-color); } @@ -48,43 +48,53 @@ font-size: 14px; } .bottomTapList { - padding: 0px $padding-25; - border-bottom: $border-width-base $border-style-base $border-color-base; - @include border-color($background-color-base, $dark-workspace-body-bg-color); - @include font-color($workspace-title-color, $dark-workspace-title-color); - margin-top: 12px; + padding: 0px $padding-25; + border-bottom: $border-width-base $border-style-base + $border-color-base; + @include border-color( + $background-color-base, + $dark-workspace-body-bg-color + ); + @include font-color( + $workspace-title-color, + $dark-workspace-title-color + ); + margin-top: 12px; + flex: none; + display: flex; + align-items: center; + font-size: $font-size-large; + .bottomLeftText { + cursor: pointer; flex: none; - display: flex; - align-items: center; font-size: $font-size-large; - .bottomLeftText { - cursor: pointer; - flex: none; - font-size: $font-size-large; - padding: 0 15px; - margin-bottom: -1px; - line-height: 40px; - position: relative; - &::after { - content: ""; - border-left: 1px solid #DEE4EC; - @include border-color($border-color-base, $dark-border-color-base); - width: 0; - position: absolute; - right: -15px; - top: 12px; - height: 16px; - margin: 0 15px; - } - } - .active { - border-bottom: 2px solid $primary-color; - @include border-color($primary-color, $dark-primary-color); - } - .bottomRightContainer { - flex: 1; - height: 40px; + padding: 0 15px; + margin-bottom: -1px; + line-height: 40px; + position: relative; + &::after { + content: ''; + border-left: 1px solid #dee4ec; + @include border-color( + $border-color-base, + $dark-border-color-base + ); + width: 0; + position: absolute; + right: -15px; + top: 12px; + height: 16px; + margin: 0 15px; } + } + .active { + border-bottom: 2px solid $primary-color; + @include border-color($primary-color, $dark-primary-color); + } + .bottomRightContainer { + flex: 1; + height: 40px; + } } } } @@ -109,7 +119,7 @@ line-height: 40px; position: relative; &::after { - content: ""; + content: ''; border-left: 1px solid #dee4ec; @include border-color($border-color-base, $dark-border-color-base); width: 0; @@ -149,13 +159,13 @@ padding: 0 10px; overflow: hidden; margin-right: 8px; - @include bg-color( #E1E5EA, $dark-workspace-body-bg-color); + @include bg-color(#e1e5ea, $dark-workspace-body-bg-color); border-radius: 12px; &.active { height: 24px; @include font-color($primary-color, $dark-primary-color); line-height: 24px; - @include bg-color(#E8EEF4, $dark-workspace-body-bg-color); + @include bg-color(#e8eef4, $dark-workspace-body-bg-color); border-radius: 12px; } &:hover { @@ -163,7 +173,7 @@ @include font-color($primary-color, $dark-primary-color); line-height: 24px; border-radius: 12px; - @include bg-color(#D1D7DD, $dark-workspace-body-bg-color); + @include bg-color(#d1d7dd, $dark-workspace-body-bg-color); } } diff --git "a/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" "b/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" deleted file mode 100644 index 098bf4437..000000000 --- "a/web/packages/dss/public/streamis/Streamis\346\265\201\345\274\217\345\272\224\347\224\250ZIP\345\214\205\344\270\212\344\274\240\346\240\274\345\274\217.txt" +++ /dev/null @@ -1,64 +0,0 @@ -流式应用物料包是指的按照Streamis打包规范，将元数据信息(流式应用描述信息),流式应用代码，流式应用使用到的物料等内容打包成zip包。其具体格式如下： - -xxx.zip - ├── meta.json - ├── test.sql - ├── test.jar - ├── file3 - -其中，meta.json格式为： -{ - "projectName": "", # 项目名 - "jobName": "", # 作业名 - "jobType": "flink.sql", # 目前只支持flink.sql、flink.jar - "tags": "", # 应用标签 - "description": "" # 作业描述, - "jobContent": { - # 不同的jobType，其内容各不相同，具体请往下看 - } -} - -如果jobType为"flink.sql"，则jobContent为： -{ - "type": "" # file, bml or sql - "sql": "select 1", - "file": "test.sql", - "resourceId": "", - "version": "" -} -其中，如果type为"file"，则只识别file字段；如果type为"sql"，则只识别sql字段；如果type为"bml"，则只识别resourceId和version字段。 - - -如果jobType为"flink.jar"，则jobContent为： - -{ - "main.class.jar": "", # string。main class的jar，如：test.jar - "main.class": "", # main class，如 com.webank.Test - "args": "", # main class 的入参，即main函数的args，请以空格为分隔符 - "hdfs.jars"； [], # 依赖的HDFS jars，如：hdfs:///user/enjoyyin/test1.jar - "dependency.jars": [], # 依赖的jars，如：test2.jar - "resources": [] # 依赖的资源文件，如：test.properties -} - - -也支持直接requestBody为JSON的方式创建新Job，或者是更新Job版本，requestBody格式跟meta.json一模一样，只是相关的文件是工程级别的文件。 - - -上传文件： -/api/rest_j/v1/streamis/project/files/upload -{ - "fileName": "", - "version": "", - "projectName": "", - "updateWhenExists": false -} -mutildata带上文件字节流 - -/api/rest_j/v1/streamis/project/files/list -/api/rest_j/v1/streamis/project/files/delete - -/api/rest_j/v1/streamis/job/upload -带上zip - -/api/rest_j/v1/streamis/job/createOrUpdate -requestBody为zip的meta.json \ No newline at end of file diff --git a/web/packages/dss/public/streamis/flinkJar.zip b/web/packages/dss/public/streamis/flinkJar.zip deleted file mode 100644 index 80b64c836..000000000 Binary files a/web/packages/dss/public/streamis/flinkJar.zip and /dev/null differ diff --git a/web/packages/exts/open-source/scriptis/apiPublish/index.vue b/web/packages/exts/open-source/scriptis/apiPublish/index.vue index b4bafcc95..6eac89868 100644 --- a/web/packages/exts/open-source/scriptis/apiPublish/index.vue +++ b/web/packages/exts/open-source/scriptis/apiPublish/index.vue @@ -600,14 +600,12 @@ export default { // 验证发布和更新的默认值是否满足条件 verificationValue (row) { let flag; - if(row.defaultValue.length > 1024) { - this.$Message.error({ content: this.$t('message.ext.opensource.longer1024') }); - flag = true; - } else if(row.paramType === 4 && row.defaultValue.split('\n').length > 1000) { - this.$Message.error({ content: this.$t('message.ext.opensource.rowlimit') }); + const method = this.updateApiData.id ? this.updateApiData.requestType : this.addApiData.requestType; + if (method.toUpperCase() === 'GET' && row.defaultValue.length > 500) { + this.$Message.error({ content: '不能超过500字符' }); flag = true; } else { - flag = false + flag = false; } this.$set(this.tip, row.paramName, flag) }, @@ -825,6 +823,7 @@ export default { }, saveApiOk() { if (this.saveLoading) return; + if(Object.values(this.tip).some(i => i)) return this.$Message.error({ content: this.$t('message.apiServices.outlimit') }); this.$emit('on-save'); this.$refs['addApi'].validate((valid) => { if (valid) { @@ -881,6 +880,7 @@ export default { }, updateApiOk() { if (this.saveLoading) return; + if(Object.values(this.tip).some(i => i)) return this.$Message.error({ content: this.$t('message.apiServices.outlimit') }); this.$emit('on-save'); this.$refs['updateApi'].validate((valid) => { diff --git a/web/packages/scriptis/components/importToHive/index.vue b/web/packages/scriptis/components/importToHive/index.vue index 126645b0e..520eaa2f6 100644 --- a/web/packages/scriptis/components/importToHive/index.vue +++ b/web/packages/scriptis/components/importToHive/index.vue @@ -431,6 +431,7 @@ export default { { label: this.$t('message.scripts.importToHive.FH'), value: ';' }, { label: this.$t('message.scripts.importToHive.ZBF'), value: '\\t' }, { label: this.$t('message.scripts.importToHive.KG'), value: '%20' }, + { label: this.$t('message.scripts.hiveTableExport.SX'), value: '|' }, ], chartset: [ { label: 'utf-8', value: 'utf-8' }, diff --git a/web/packages/scriptis/i18n/common/en.json b/web/packages/scriptis/i18n/common/en.json index bb0687ff4..babf8b9c8 100644 --- a/web/packages/scriptis/i18n/common/en.json +++ b/web/packages/scriptis/i18n/common/en.json @@ -14,6 +14,9 @@ "saveErr": "Save failed!", "createdTitle": "Table creation guide", "saveToWidget": "Save widget to solution", + "query": { + "more": "More" + }, "contextMenu": { "openToTab": "Open as a tab", "copyPath": "Copy path", @@ -193,7 +196,11 @@ "YES": "Yes", "GSHJX": "Formatted to", "ZJXYGE": "The format required by the component", - "FZBZDXX": "Copy table field information" + "FZBZDXX": "Copy table field information", + "QHPTST": "Switch common view", + "QHGLYST": "Switch administrator view", + "QSRBSZ": "Please enter the table owner", + "MCJZCZYYWYH": "Only one user's table can be transferred at a time" }, "logView": { "taskId": "Task ID：", diff --git a/web/packages/scriptis/i18n/common/zh.json b/web/packages/scriptis/i18n/common/zh.json index 8cbce1401..33825a3b3 100644 --- a/web/packages/scriptis/i18n/common/zh.json +++ b/web/packages/scriptis/i18n/common/zh.json @@ -14,6 +14,9 @@ "saveErr": "保存失败！", "createdTitle": "建表向导", "saveToWidget": "保存widget到解决方案", + "query": { + "more": "更多" + }, "contextMenu": { "openToTab": "打开到侧边", "copyPath": "复制路径", @@ -193,7 +196,11 @@ "YES": "对", "GSHJX": "进行格式化成", "ZJXYGE": "组件需要的格式", - "FZBZDXX": "复制表字段信息" + "FZBZDXX": "复制表字段信息", + "QHPTST": "切换普通视图", + "QHGLYST": "切换管理员视图", + "QSRBSZ": "请输入表属主", + "MCJZCZYYWYH": "每次仅支持转移一位用户的表" }, "logView": { "taskId": "任务ID：", diff --git a/web/packages/scriptis/module/hdfsSidebar/hdfsSidebar.vue b/web/packages/scriptis/module/hdfsSidebar/hdfsSidebar.vue index 352fefce1..f90fa3d98 100644 --- a/web/packages/scriptis/module/hdfsSidebar/hdfsSidebar.vue +++ b/web/packages/scriptis/module/hdfsSidebar/hdfsSidebar.vue @@ -709,6 +709,8 @@ export default { separator = '%5Ct'; } else if (option.separator === '%20') { separator = ' '; + } else if (option.separator === '|') { + separator = '%5C%7C'; } const encoding = type ? '' : option.chartset; const fieldDelimiter = type ? '' : separator; @@ -759,7 +761,12 @@ export default { if (secondStep.partition && secondStep.partitionValue) { isPartition = true; } - const separator = firstStep.separator === '%20' ? ' ' : firstStep.separator; + let separator = firstStep.separator; + if (firstStep.separator === '%20') { + separator = ' '; + } else if (firstStep.separator === '|') { + separator = '\\|'; + } if (firstStep.quote) { escapeQuotes = true; quote = firstStep.quote; diff --git a/web/packages/scriptis/module/hiveSidebar/hiveTableExport/index.vue b/web/packages/scriptis/module/hiveSidebar/hiveTableExport/index.vue index e852d99e5..0f7c4c162 100644 --- a/web/packages/scriptis/module/hiveSidebar/hiveTableExport/index.vue +++ b/web/packages/scriptis/module/hiveSidebar/hiveTableExport/index.vue @@ -332,6 +332,7 @@ export default { { label: this.$t('message.scripts.hiveTableExport.FH'), value: ';' }, { label: this.$t('message.scripts.hiveTableExport.ZBF'), value: '\\t' }, { label: this.$t('message.scripts.hiveTableExport.KG'), value: '%20' }, + { label: this.$t('message.scripts.hiveTableExport.SX'), value: '|' }, ], exportTypes: [ { label: 'xlsx', value: 'xlsx' }, diff --git a/web/packages/scriptis/module/workSidebar/workSidebar.vue b/web/packages/scriptis/module/workSidebar/workSidebar.vue index c7b7088d8..a8a4158fb 100644 --- a/web/packages/scriptis/module/workSidebar/workSidebar.vue +++ b/web/packages/scriptis/module/workSidebar/workSidebar.vue @@ -1077,6 +1077,8 @@ export default { separator = '%5Ct'; } else if (option.separator === '%20') { separator = ' '; + } else if (option.separator === '|') { + separator = '%5C%7C'; } const encoding = type ? '' : option.chartset; const fieldDelimiter = type ? '' : separator; @@ -1127,7 +1129,12 @@ export default { if (secondStep.partition && secondStep.partitionValue) { isPartition = true; } - const separator = firstStep.separator === '%20' ? ' ' : firstStep.separator; + let separator = firstStep.separator; + if (firstStep.separator === '%20') { + separator = ' '; + } else if (firstStep.separator === '|') { + separator = '\\|'; + } if (firstStep.quote) { escapeQuotes = true; quote = firstStep.quote; diff --git a/web/packages/scriptis/module/workbench/createTable/components/importTable.vue b/web/packages/scriptis/module/workbench/createTable/components/importTable.vue index ab15f34a0..be9f6b24c 100644 --- a/web/packages/scriptis/module/workbench/createTable/components/importTable.vue +++ b/web/packages/scriptis/module/workbench/createTable/components/importTable.vue @@ -116,6 +116,8 @@ export default { separator = '%5Ct'; } else if (option.separator === '%20') { separator = ' '; + } else if (option.separator === '|') { + separator = '%5C%7C'; } const encoding = this.source.importType === 'xlsx' ? '' : option.chartset; const fieldDelimiter = this.source.importType === 'xlsx' ? '' : separator; diff --git a/web/packages/scriptis/module/workbench/createTable/components/importTable_csv.vue b/web/packages/scriptis/module/workbench/createTable/components/importTable_csv.vue index dbf24c7e9..2dee6128c 100644 --- a/web/packages/scriptis/module/workbench/createTable/components/importTable_csv.vue +++ b/web/packages/scriptis/module/workbench/createTable/components/importTable_csv.vue @@ -148,6 +148,7 @@ export default { { label: `${this.$t('message.scripts.createTable.FH')}(;)`, value: ';' }, { label: `${this.$t('message.scripts.createTable.ZBF')}(\\t)`, value: '\\t' }, { label: this.$t('message.scripts.createTable.KG'), value: '%20' }, + { label: this.$t('message.scripts.hiveTableExport.SX'), value: '|' }, ], chartset: [ { label: 'utf-8', value: 'utf-8' }, diff --git a/web/packages/scriptis/module/workbench/createTable/index.vue b/web/packages/scriptis/module/workbench/createTable/index.vue index 887a3af62..ec1d2ff4a 100644 --- a/web/packages/scriptis/module/workbench/createTable/index.vue +++ b/web/packages/scriptis/module/workbench/createTable/index.vue @@ -342,16 +342,23 @@ export default { if (!isXls) { delete source.table.sheet } + let fieldDelimiter = source.table.separator; + if (source.table.separator === '%20') { + fieldDelimiter = ' '; + } else if (source.table.separator === '|') { + fieldDelimiter = '\\|'; + } const sourceP = { path, pathType: source.table.type, hasHeader: source.table.isHasHeader, encoding: isXls ? '' : source.table.chartset, - fieldDelimiter: isXls ? '' : source.table.separator, + fieldDelimiter: isXls ? '' : fieldDelimiter, sheet: source.table.sheet && source.table.sheet.toString(), quote, escapeQuotes, }; + const importInfo = { importType: ['hive', 'csv', 'xlsx'].indexOf(source.importType), args: isHive ? { diff --git a/web/packages/scriptis/module/workbench/dbDetails/components/tableList.vue b/web/packages/scriptis/module/workbench/dbDetails/components/tableList.vue index 94d9a1ad5..7b62eaa4c 100644 --- a/web/packages/scriptis/module/workbench/dbDetails/components/tableList.vue +++ b/web/packages/scriptis/module/workbench/dbDetails/components/tableList.vue @@ -1,6 +1,6 @@