SUG-2795 integrate T2/T6 LB IPVS cloud changes

conf/db/upgrade/V5.5.28__schema.sql

@@ -98,3 +98,15 @@ CREATE TABLE IF NOT EXISTS `zstack`.`ScimEventVO` (
     UNIQUE KEY `ukScimEventVOClientEvent` (`clientId`, `eventId`),
     KEY `idxScimEventVOResourceVersion` (`clientId`, `resourceType`, `resourceId`, `resourceVersion`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+-- SUG-2795: listener-level TCP IPVS data plane and forward mode.
+CALL ADD_COLUMN('LoadBalancerListenerVO', 'data_plane', 'VARCHAR(32)', 1, NULL);
+CALL ADD_COLUMN('LoadBalancerListenerVO', 'forward_mode', 'VARCHAR(32)', 1, NULL);
+
+UPDATE `zstack`.`LoadBalancerListenerVO`
+SET data_plane = 'ipvs'
+WHERE protocol = 'udp' AND data_plane IS NULL;
+
+UPDATE `zstack`.`LoadBalancerListenerVO`
+SET data_plane = 'haproxy'
+WHERE data_plane IS NULL;

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java

@@ -44,6 +44,9 @@ public class APIChangeLoadBalancerListenerMsg extends APIMessage implements Load
     @APIParam(numberRange = {LoadBalancerConstants.HEALTH_CHECK_INTERVAL_MIN, LoadBalancerConstants.HEALTH_CHECK_INTERVAL_MAX}, required = false)
     private Integer healthCheckInterval;
 
+    @APIParam(numberRange = {LoadBalancerConstants.HEALTH_CHECK_TIMEOUT_MIN, LoadBalancerConstants.HEALTH_CHECK_TIMEOUT_MAX}, required = false)
+    private Integer healthCheckTimeout;
+
     @APIParam(validValues = {LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE}, required = false)
     private String healthCheckProtocol;
     @APIParam(validValues = {"GET", "HEAD"}, required = false)
@@ -167,6 +170,14 @@ public void setHealthCheckInterval(Integer healthCheckInterval) {
         this.healthCheckInterval = healthCheckInterval;
     }
 
+    public Integer getHealthCheckTimeout() {
+        return healthCheckTimeout;
+    }
+
+    public void setHealthCheckTimeout(Integer healthCheckTimeout) {
+        this.healthCheckTimeout = healthCheckTimeout;
+    }
+
     public String getLoadBalancerUuid() {
         return loadBalancerUuid;
     }

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy

@@ -94,6 +94,15 @@ doc {
 					optional true
 					since "3.4"
 				}
+				column {
+					name "healthCheckTimeout"
+					enclosedIn "changeLoadBalancerListener"
+					desc "健康检查超时时间"
+					location "body"
+					type "Integer"
+					optional true
+					since "5.5.28"
+				}
 				column {
 					name "healthCheckProtocol"
 					enclosedIn "changeLoadBalancerListener"

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsg.java

@@ -53,6 +53,10 @@ public class APICreateLoadBalancerListenerMsg extends APICreateMessage implement
     private List<String> aclUuids;
     @APIParam(validValues = {"white","black"}, required = false)
     private String aclType = LoadBalancerAclType.black.toString();
+    @APIParam(validValues = {LoadBalancerConstants.DATA_PLANE_HAPROXY, LoadBalancerConstants.DATA_PLANE_IPVS}, required = false)
+    private String dataPlane;
+    @APIParam(validValues = {LoadBalancerConstants.FORWARD_MODE_FULL_NAT, LoadBalancerConstants.FORWARD_MODE_NAT, LoadBalancerConstants.FORWARD_MODE_DR}, required = false)
+    private String forwardMode;
     @APIParam(validValues = {LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_DEFAULT, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_0, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_1,
             LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2_STRICT,
             LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2_STRICT_WITH_1_3}, required = false)
@@ -180,6 +184,22 @@ public void setAclType(String aclType) {
         this.aclType = aclType;
     }
 
+    public String getDataPlane() {
+        return dataPlane;
+    }
+
+    public void setDataPlane(String dataPlane) {
+        this.dataPlane = dataPlane;
+    }
+
+    public String getForwardMode() {
+        return forwardMode;
+    }
+
+    public void setForwardMode(String forwardMode) {
+        this.forwardMode = forwardMode;
+    }
+
     public String getSecurityPolicyType() {
         return securityPolicyType;
     }

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy

@@ -225,6 +225,26 @@ doc {
 					optional true
 					since "5.0.0"
 				}
+				column {
+					name "dataPlane"
+					enclosedIn "params"
+					desc "监听器使用的数据平面。haproxy 表示使用 HAProxy，ipvs 表示使用 IPVS。"
+					location "body"
+					type "String"
+					optional true
+					since "5.5.28"
+					values ("haproxy","ipvs")
+				}
+				column {
+					name "forwardMode"
+					enclosedIn "params"
+					desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat。"
+					location "body"
+					type "String"
+					optional true
+					since "5.5.28"
+					values ("full_nat","nat","dr")
+				}
 			}
         }
 

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java

@@ -728,18 +728,48 @@ private Boolean verifyHttpCode(String httpCode) {
         });
     }
 
-    private boolean isHealthCheckProtocolNotSupportedByListenerProtocol(String listenerProtocol, String healthCheckProtocol) {
+    private boolean isHealthCheckProtocolNotSupportedByListenerProtocol(String listenerProtocol, String dataPlane, String healthCheckProtocol) {
         boolean isUdpListener = LoadBalancerConstants.LB_PROTOCOL_UDP.equals(listenerProtocol);
+        boolean isTcpIpvsListener = isTcpIpvsListener(listenerProtocol, dataPlane);
         boolean isUdpHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP.equals(healthCheckProtocol);
+        boolean isTcpHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP.equals(healthCheckProtocol);
         boolean isNoneHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE.equals(healthCheckProtocol);
 
         if (isUdpListener) {
             return !(isUdpHealthCheck || isNoneHealthCheck);
         }
 
+        if (isTcpIpvsListener) {
+            return !(isTcpHealthCheck || isNoneHealthCheck);
+        }
+
         return isUdpHealthCheck || isNoneHealthCheck;
     }
 
+    private String getListenerDataPlane(String listenerUuid) {
+        LoadBalancerListenerVO listener = Q.New(LoadBalancerListenerVO.class)
+                .eq(LoadBalancerListenerVO_.uuid, listenerUuid)
+                .find();
+        return listener == null || listener.getDataPlane() == null ? DATA_PLANE_HAPROXY : listener.getDataPlane();
+    }
+
+    private boolean isTcpIpvsListener(String listenerProtocol, String dataPlane) {
+        return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(listenerProtocol) &&
+                DATA_PLANE_IPVS.equals(dataPlane);
+    }
+
+    private boolean hasHttpHealthCheckParameters(APICreateLoadBalancerListenerMsg msg) {
+        return msg.getHealthCheckMethod() != null ||
+                msg.getHealthCheckURI() != null ||
+                msg.getHealthCheckHttpCode() != null;
+    }
+
+    private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg msg) {
+        return msg.getHealthCheckMethod() != null ||
+                msg.getHealthCheckURI() != null ||
+                msg.getHealthCheckHttpCode() != null;
+    }
+
     private String getHealthCheckProtocolFromTarget(String healthCheckTarget) {
         if (healthCheckTarget == null) {
             return null;
@@ -769,6 +799,36 @@ private boolean isNoneHealthCheckTargetWithSpecificPort(String healthCheckTarget
                 !"default".equals(ts[1]);
     }
 
+    private boolean isValidHealthCheckProtocolInTarget(String protocol) {
+        return LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP.equals(protocol) ||
+                LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP.equals(protocol) ||
+                LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP.equals(protocol) ||
+                LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE.equals(protocol);
+    }
+
+    private void normalizeChangeHealthCheckTarget(APIChangeLoadBalancerListenerMsg msg) {
+        String target = msg.getHealthCheckTarget();
+        if (target == null || !target.contains(":")) {
+            return;
+        }
+
+        String[] ts = target.split(":");
+        if (ts.length != 2 || !isValidHealthCheckProtocolInTarget(ts[0])) {
+            throw new ApiMessageInterceptionException(
+                    argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10100, "healthCheck target [%s] error, it must be 'default' or number between[1~65535] ",
+                            target));
+        }
+
+        if (msg.getHealthCheckProtocol() != null && !msg.getHealthCheckProtocol().equals(ts[0])) {
+            throw new ApiMessageInterceptionException(
+                    operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "the health check protocol [%s] conflicts with health check target [%s]",
+                            msg.getHealthCheckProtocol(), target));
+        }
+
+        msg.setHealthCheckProtocol(ts[0]);
+        msg.setHealthCheckTarget(ts[1]);
+    }
+
     private void validate(APICreateLoadBalancerListenerMsg msg) {
         LoadBalancerVO lbVO = dbf.findByUuid(msg.getLoadBalancerUuid(), LoadBalancerVO.class);
 
@@ -787,6 +847,39 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
             }
         }
 
+        if (msg.getDataPlane() == null) {
+            msg.setDataPlane(LoadBalancerConstants.DATA_PLANE_HAPROXY);
+        }
+
+        if (LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane())) {
+            if (!LB_PROTOCOL_TCP.equals(msg.getProtocol())) {
+                throw new ApiMessageInterceptionException(
+                        operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "data plane [%s] only supports tcp listener", msg.getDataPlane()));
+            }
+
+            if (msg.getForwardMode() == null) {
+                msg.setForwardMode(LoadBalancerConstants.FORWARD_MODE_FULL_NAT);
+            }
+
+            if (!LoadBalancerConstants.FORWARD_MODE_FULL_NAT.equals(msg.getForwardMode())) {
+                throw new ApiMessageInterceptionException(
+                        operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "TCP IPVS only supports forwardMode[%s] in current version, but got [%s]",
+                                LoadBalancerConstants.FORWARD_MODE_FULL_NAT, msg.getForwardMode()));
+            }
+
+        } else {
+            if (!LoadBalancerConstants.DATA_PLANE_HAPROXY.equals(msg.getDataPlane())) {
+                throw new ApiMessageInterceptionException(
+                        operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane()));
+            }
+
+            if (msg.getForwardMode() != null) {
+                throw new ApiMessageInterceptionException(
+                        operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "forwardMode is only supported when dataPlane is ipvs"));
+            }
+        }
+        String dataPlane = msg.getDataPlane();
+
         List<String> healthCheckTargets = getSystemTagTokens(msg, LoadBalancerSystemTags.HEALTH_TARGET,
                 LoadBalancerSystemTags.HEALTH_TARGET_TOKEN);
         if (healthCheckTargets.size() > 1) {
@@ -818,7 +911,13 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
             }
         }
 
-        if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), msg.getHealthCheckProtocol())) {
+        if (isTcpIpvsListener(msg.getProtocol(), dataPlane) &&
+                (hasHttpHealthCheckParameters(msg) || hasTag(msg, LoadBalancerSystemTags.HEALTH_PARAMETER))) {
+            throw new ApiMessageInterceptionException(
+                    operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters"));
+        }
+
+        if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) {
             throw new ApiMessageInterceptionException(
                     operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10062, "the listener with protocol [%s] doesn't support this health check:[%s]",
                             msg.getProtocol(), msg.getHealthCheckProtocol()));
@@ -1306,6 +1405,7 @@ private void validate(APIRemoveCertificateFromLoadBalancerListenerMsg msg) {
     }
 
     private void validate(APIChangeLoadBalancerListenerMsg msg) {
+        normalizeChangeHealthCheckTarget(msg);
         String target = msg.getHealthCheckTarget();
         if (target != null) {
             if (!LoadBalancerConstants.HEALTH_CHECK_TARGET_DEFAULT.equals(target)) {
@@ -1457,6 +1557,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) {
 
         LoadBalancerListenerVO listenerVO = Q.New(LoadBalancerListenerVO.class).
                                            eq(LoadBalancerListenerVO_.uuid,msg.getLoadBalancerListenerUuid()).find();
+        String dataPlane = getListenerDataPlane(msg.getLoadBalancerListenerUuid());
 
         if (msg.getSecurityPolicyType() != null) {
             if (!listenerVO.getProtocol().equals(LB_PROTOCOL_HTTPS)) {
@@ -1483,8 +1584,13 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) {
                             effectiveHealthCheckTarget));
         }
 
+        if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && hasHttpHealthCheckParameters(msg)) {
+            throw new ApiMessageInterceptionException(
+                    operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters"));
+        }
+
         if (msg.getHealthCheckProtocol() != null) {
-            if (isHealthCheckProtocolNotSupportedByListenerProtocol(listenerVO.getProtocol(), msg.getHealthCheckProtocol())) {
+            if (isHealthCheckProtocolNotSupportedByListenerProtocol(listenerVO.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) {
                 throw new ApiMessageInterceptionException(
                         operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10120, "the listener with protocol [%s] doesn't support this health check:[%s]",
                                 listenerVO.getProtocol(), msg.getHealthCheckProtocol()));

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java

@@ -1836,6 +1836,8 @@ private void createListener(final APICreateLoadBalancerListenerMsg msg, final No
         vo.setInstancePort(msg.getInstancePort());
         vo.setLoadBalancerPort(msg.getLoadBalancerPort());
         vo.setProtocol(msg.getProtocol());
+        vo.setDataPlane(msg.getDataPlane());
+        vo.setForwardMode(LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane()) ? msg.getForwardMode() : null);
         vo.setAccountUuid(msg.getSession().getAccountUuid());
         vo.setSecurityPolicyType(msg.getSecurityPolicyType());
         vo = dbf.persistAndRefresh(vo);
@@ -2334,6 +2336,9 @@ public void run(SyncTaskChain chain) {
                     updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, msg.getUuid(), LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, msg.getConnectionIdleTimeout());
                 }
 
+                final String oldHealthCheckTimeout = LoadBalancerSystemTags.HEALTH_TIMEOUT.getTokenByResourceUuid(
+                        msg.getUuid(), LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN);
+
                 if (msg.getHealthCheckInterval() != null) {
                     updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.HEALTH_INTERVAL, msg.getUuid(), LoadBalancerSystemTags.HEALTH_INTERVAL_TOKEN, msg.getHealthCheckInterval());
                 }
@@ -2359,6 +2364,10 @@ public void run(SyncTaskChain chain) {
                     updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.UNHEALTHY_THRESHOLD, msg.getUuid(), LoadBalancerSystemTags.UNHEALTHY_THRESHOLD_TOKEN, msg.getUnhealthyThreshold());
                 }
 
+                if (msg.getHealthCheckTimeout() != null) {
+                    updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.HEALTH_TIMEOUT, msg.getUuid(), LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, msg.getHealthCheckTimeout());
+                }
+
                 if (msg.getMaxConnection() != null) {
                     updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.MAX_CONNECTION, msg.getUuid(), LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, msg.getMaxConnection());
                 }
@@ -2488,6 +2497,17 @@ public void run(MessageReply reply) {
                                         LoadBalancerSystemTags.BALANCER_ACL.delete(msg.getUuid());
                                     }
                                 }
+                                if (msg.getHealthCheckTimeout() != null) {
+                                    logger.warn(String.format( "rollback health check timeout for listener [uuid:%s]", msg.getUuid()));
+                                    if (oldHealthCheckTimeout != null) {
+                                        LoadBalancerSystemTags.HEALTH_TIMEOUT.update(msg.getUuid(),
+                                                LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag(map(
+                                                        e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, oldHealthCheckTimeout)
+                                                )));
+                                    } else {
+                                        LoadBalancerSystemTags.HEALTH_TIMEOUT.delete(msg.getUuid());
+                                    }
+                                }
                             } else {
                                 evt.setInventory(LoadBalancerListenerInventory.valueOf(lblVo));
                             }

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java

@@ -111,6 +111,13 @@ public String toString() {
     public static final int PROTOCOL_HTTP_DEFAULT_PORT = 80;
     public static final int PROTOCOL_HTTPS_DEFAULT_PORT = 443;
 
+    public static final String DATA_PLANE_HAPROXY = "haproxy";
+    public static final String DATA_PLANE_IPVS = "ipvs";
+
+    public static final String FORWARD_MODE_FULL_NAT = "full_nat";
+    public static final String FORWARD_MODE_NAT = "nat";
+    public static final String FORWARD_MODE_DR = "dr";
+
     public static final int DNS_PORT = 53;
     public static final int SSH_PORT = 22;
     public static final int ZVR_PORT = 7272;
@@ -159,6 +166,9 @@ public static enum Param {
     public static final int HEALTH_CHECK_INTERVAL_MIN = 1;
     public static final int HEALTH_CHECK_INTERVAL_MAX = Integer.MAX_VALUE;
 
+    public static final int HEALTH_CHECK_TIMEOUT_MIN = 1;
+    public static final int HEALTH_CHECK_TIMEOUT_MAX = Integer.MAX_VALUE;
+
     public static final int NUMBER_OF_PROCESS_MIN = 1;
     public static final int NUMBER_OF_PROCESS_MAX = 64;