diff --git a/conf/db/upgrade/V5.5.28__schema.sql b/conf/db/upgrade/V5.5.28__schema.sql index 7c1ab9fe15f..8252f8d2233 100644 --- a/conf/db/upgrade/V5.5.28__schema.sql +++ b/conf/db/upgrade/V5.5.28__schema.sql @@ -98,3 +98,15 @@ CREATE TABLE IF NOT EXISTS `zstack`.`ScimEventVO` ( UNIQUE KEY `ukScimEventVOClientEvent` (`clientId`, `eventId`), KEY `idxScimEventVOResourceVersion` (`clientId`, `resourceType`, `resourceId`, `resourceVersion`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- SUG-2795: listener-level TCP IPVS data plane and forward mode. +CALL ADD_COLUMN('LoadBalancerListenerVO', 'data_plane', 'VARCHAR(32)', 1, NULL); +CALL ADD_COLUMN('LoadBalancerListenerVO', 'forward_mode', 'VARCHAR(32)', 1, NULL); + +UPDATE `zstack`.`LoadBalancerListenerVO` +SET data_plane = 'ipvs' +WHERE protocol = 'udp' AND data_plane IS NULL; + +UPDATE `zstack`.`LoadBalancerListenerVO` +SET data_plane = 'haproxy' +WHERE data_plane IS NULL; diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java index 3d77b9d1335..6e47b12bebe 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsg.java @@ -44,6 +44,9 @@ public class APIChangeLoadBalancerListenerMsg extends APIMessage implements Load @APIParam(numberRange = {LoadBalancerConstants.HEALTH_CHECK_INTERVAL_MIN, LoadBalancerConstants.HEALTH_CHECK_INTERVAL_MAX}, required = false) private Integer healthCheckInterval; + @APIParam(numberRange = {LoadBalancerConstants.HEALTH_CHECK_TIMEOUT_MIN, LoadBalancerConstants.HEALTH_CHECK_TIMEOUT_MAX}, required = false) + private Integer healthCheckTimeout; + @APIParam(validValues = {LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP, LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE}, required = false) private String healthCheckProtocol; @APIParam(validValues = {"GET", "HEAD"}, required = false) @@ -167,6 +170,14 @@ public void setHealthCheckInterval(Integer healthCheckInterval) { this.healthCheckInterval = healthCheckInterval; } + public Integer getHealthCheckTimeout() { + return healthCheckTimeout; + } + + public void setHealthCheckTimeout(Integer healthCheckTimeout) { + this.healthCheckTimeout = healthCheckTimeout; + } + public String getLoadBalancerUuid() { return loadBalancerUuid; } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy index c92f14186a4..bf2c1f80054 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APIChangeLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -94,6 +94,15 @@ doc { optional true since "3.4" } + column { + name "healthCheckTimeout" + enclosedIn "changeLoadBalancerListener" + desc "健康检查超时时间" + location "body" + type "Integer" + optional true + since "5.5.28" + } column { name "healthCheckProtocol" enclosedIn "changeLoadBalancerListener" diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsg.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsg.java index 5b495571cc9..e8e4b4694dc 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsg.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsg.java @@ -53,6 +53,10 @@ public class APICreateLoadBalancerListenerMsg extends APICreateMessage implement private List aclUuids; @APIParam(validValues = {"white","black"}, required = false) private String aclType = LoadBalancerAclType.black.toString(); + @APIParam(validValues = {LoadBalancerConstants.DATA_PLANE_HAPROXY, LoadBalancerConstants.DATA_PLANE_IPVS}, required = false) + private String dataPlane; + @APIParam(validValues = {LoadBalancerConstants.FORWARD_MODE_FULL_NAT, LoadBalancerConstants.FORWARD_MODE_NAT, LoadBalancerConstants.FORWARD_MODE_DR}, required = false) + private String forwardMode; @APIParam(validValues = {LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_DEFAULT, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_0, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_1, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2_STRICT, LoadBalanceSecurityPolicyConstant.TLS_CIPHER_POLICY_1_2_STRICT_WITH_1_3}, required = false) @@ -180,6 +184,22 @@ public void setAclType(String aclType) { this.aclType = aclType; } + public String getDataPlane() { + return dataPlane; + } + + public void setDataPlane(String dataPlane) { + this.dataPlane = dataPlane; + } + + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public String getSecurityPolicyType() { return securityPolicyType; } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy index 9496c068530..099aa923130 100644 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/APICreateLoadBalancerListenerMsgDoc_zh_cn.groovy @@ -225,6 +225,26 @@ doc { optional true since "5.0.0" } + column { + name "dataPlane" + enclosedIn "params" + desc "监听器使用的数据平面。haproxy 表示使用 HAProxy，ipvs 表示使用 IPVS。" + location "body" + type "String" + optional true + since "5.5.28" + values ("haproxy","ipvs") + } + column { + name "forwardMode" + enclosedIn "params" + desc "监听器使用 IPVS 数据平面时的转发模式。当前 TCP IPVS 监听器支持 full_nat。" + location "body" + type "String" + optional true + since "5.5.28" + values ("full_nat","nat","dr") + } } } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index b846b54185e..707538f2d5a 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -728,18 +728,48 @@ private Boolean verifyHttpCode(String httpCode) { }); } - private boolean isHealthCheckProtocolNotSupportedByListenerProtocol(String listenerProtocol, String healthCheckProtocol) { + private boolean isHealthCheckProtocolNotSupportedByListenerProtocol(String listenerProtocol, String dataPlane, String healthCheckProtocol) { boolean isUdpListener = LoadBalancerConstants.LB_PROTOCOL_UDP.equals(listenerProtocol); + boolean isTcpIpvsListener = isTcpIpvsListener(listenerProtocol, dataPlane); boolean isUdpHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP.equals(healthCheckProtocol); + boolean isTcpHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP.equals(healthCheckProtocol); boolean isNoneHealthCheck = LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE.equals(healthCheckProtocol); if (isUdpListener) { return !(isUdpHealthCheck || isNoneHealthCheck); } + if (isTcpIpvsListener) { + return !(isTcpHealthCheck || isNoneHealthCheck); + } + return isUdpHealthCheck || isNoneHealthCheck; } + private String getListenerDataPlane(String listenerUuid) { + LoadBalancerListenerVO listener = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listenerUuid) + .find(); + return listener == null || listener.getDataPlane() == null ? DATA_PLANE_HAPROXY : listener.getDataPlane(); + } + + private boolean isTcpIpvsListener(String listenerProtocol, String dataPlane) { + return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(listenerProtocol) && + DATA_PLANE_IPVS.equals(dataPlane); + } + + private boolean hasHttpHealthCheckParameters(APICreateLoadBalancerListenerMsg msg) { + return msg.getHealthCheckMethod() != null || + msg.getHealthCheckURI() != null || + msg.getHealthCheckHttpCode() != null; + } + + private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg msg) { + return msg.getHealthCheckMethod() != null || + msg.getHealthCheckURI() != null || + msg.getHealthCheckHttpCode() != null; + } + private String getHealthCheckProtocolFromTarget(String healthCheckTarget) { if (healthCheckTarget == null) { return null; @@ -769,6 +799,36 @@ private boolean isNoneHealthCheckTargetWithSpecificPort(String healthCheckTarget !"default".equals(ts[1]); } + private boolean isValidHealthCheckProtocolInTarget(String protocol) { + return LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_TCP.equals(protocol) || + LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_UDP.equals(protocol) || + LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_HTTP.equals(protocol) || + LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE.equals(protocol); + } + + private void normalizeChangeHealthCheckTarget(APIChangeLoadBalancerListenerMsg msg) { + String target = msg.getHealthCheckTarget(); + if (target == null || !target.contains(":")) { + return; + } + + String[] ts = target.split(":"); + if (ts.length != 2 || !isValidHealthCheckProtocolInTarget(ts[0])) { + throw new ApiMessageInterceptionException( + argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10100, "healthCheck target [%s] error, it must be 'default' or number between[1~65535] ", + target)); + } + + if (msg.getHealthCheckProtocol() != null && !msg.getHealthCheckProtocol().equals(ts[0])) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "the health check protocol [%s] conflicts with health check target [%s]", + msg.getHealthCheckProtocol(), target)); + } + + msg.setHealthCheckProtocol(ts[0]); + msg.setHealthCheckTarget(ts[1]); + } + private void validate(APICreateLoadBalancerListenerMsg msg) { LoadBalancerVO lbVO = dbf.findByUuid(msg.getLoadBalancerUuid(), LoadBalancerVO.class); @@ -787,6 +847,39 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { } } + if (msg.getDataPlane() == null) { + msg.setDataPlane(LoadBalancerConstants.DATA_PLANE_HAPROXY); + } + + if (LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane())) { + if (!LB_PROTOCOL_TCP.equals(msg.getProtocol())) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "data plane [%s] only supports tcp listener", msg.getDataPlane())); + } + + if (msg.getForwardMode() == null) { + msg.setForwardMode(LoadBalancerConstants.FORWARD_MODE_FULL_NAT); + } + + if (!LoadBalancerConstants.FORWARD_MODE_FULL_NAT.equals(msg.getForwardMode())) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "TCP IPVS only supports forwardMode[%s] in current version, but got [%s]", + LoadBalancerConstants.FORWARD_MODE_FULL_NAT, msg.getForwardMode())); + } + + } else { + if (!LoadBalancerConstants.DATA_PLANE_HAPROXY.equals(msg.getDataPlane())) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "invalid dataPlane[%s], valid dataPlanes are [haproxy, ipvs]", msg.getDataPlane())); + } + + if (msg.getForwardMode() != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "forwardMode is only supported when dataPlane is ipvs")); + } + } + String dataPlane = msg.getDataPlane(); + List healthCheckTargets = getSystemTagTokens(msg, LoadBalancerSystemTags.HEALTH_TARGET, LoadBalancerSystemTags.HEALTH_TARGET_TOKEN); if (healthCheckTargets.size() > 1) { @@ -818,7 +911,13 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { } } - if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), msg.getHealthCheckProtocol())) { + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && + (hasHttpHealthCheckParameters(msg) || hasTag(msg, LoadBalancerSystemTags.HEALTH_PARAMETER))) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + } + + if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { throw new ApiMessageInterceptionException( operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10062, "the listener with protocol [%s] doesn't support this health check:[%s]", msg.getProtocol(), msg.getHealthCheckProtocol())); @@ -1306,6 +1405,7 @@ private void validate(APIRemoveCertificateFromLoadBalancerListenerMsg msg) { } private void validate(APIChangeLoadBalancerListenerMsg msg) { + normalizeChangeHealthCheckTarget(msg); String target = msg.getHealthCheckTarget(); if (target != null) { if (!LoadBalancerConstants.HEALTH_CHECK_TARGET_DEFAULT.equals(target)) { @@ -1457,6 +1557,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { LoadBalancerListenerVO listenerVO = Q.New(LoadBalancerListenerVO.class). eq(LoadBalancerListenerVO_.uuid,msg.getLoadBalancerListenerUuid()).find(); + String dataPlane = getListenerDataPlane(msg.getLoadBalancerListenerUuid()); if (msg.getSecurityPolicyType() != null) { if (!listenerVO.getProtocol().equals(LB_PROTOCOL_HTTPS)) { @@ -1483,8 +1584,13 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { effectiveHealthCheckTarget)); } + if (isTcpIpvsListener(listenerVO.getProtocol(), dataPlane) && hasHttpHealthCheckParameters(msg)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10179, "tcp ipvs listener doesn't support http health check parameters")); + } + if (msg.getHealthCheckProtocol() != null) { - if (isHealthCheckProtocolNotSupportedByListenerProtocol(listenerVO.getProtocol(), msg.getHealthCheckProtocol())) { + if (isHealthCheckProtocolNotSupportedByListenerProtocol(listenerVO.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { throw new ApiMessageInterceptionException( operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10120, "the listener with protocol [%s] doesn't support this health check:[%s]", listenerVO.getProtocol(), msg.getHealthCheckProtocol())); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index e219f24eb8b..75a1e7c5ffc 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1836,6 +1836,8 @@ private void createListener(final APICreateLoadBalancerListenerMsg msg, final No vo.setInstancePort(msg.getInstancePort()); vo.setLoadBalancerPort(msg.getLoadBalancerPort()); vo.setProtocol(msg.getProtocol()); + vo.setDataPlane(msg.getDataPlane()); + vo.setForwardMode(LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane()) ? msg.getForwardMode() : null); vo.setAccountUuid(msg.getSession().getAccountUuid()); vo.setSecurityPolicyType(msg.getSecurityPolicyType()); vo = dbf.persistAndRefresh(vo); @@ -2334,6 +2336,9 @@ public void run(SyncTaskChain chain) { updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, msg.getUuid(), LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, msg.getConnectionIdleTimeout()); } + final String oldHealthCheckTimeout = LoadBalancerSystemTags.HEALTH_TIMEOUT.getTokenByResourceUuid( + msg.getUuid(), LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN); + if (msg.getHealthCheckInterval() != null) { updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.HEALTH_INTERVAL, msg.getUuid(), LoadBalancerSystemTags.HEALTH_INTERVAL_TOKEN, msg.getHealthCheckInterval()); } @@ -2359,6 +2364,10 @@ public void run(SyncTaskChain chain) { updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.UNHEALTHY_THRESHOLD, msg.getUuid(), LoadBalancerSystemTags.UNHEALTHY_THRESHOLD_TOKEN, msg.getUnhealthyThreshold()); } + if (msg.getHealthCheckTimeout() != null) { + updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.HEALTH_TIMEOUT, msg.getUuid(), LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, msg.getHealthCheckTimeout()); + } + if (msg.getMaxConnection() != null) { updateLoadBalancerListenerSystemTag(LoadBalancerSystemTags.MAX_CONNECTION, msg.getUuid(), LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, msg.getMaxConnection()); } @@ -2488,6 +2497,17 @@ public void run(MessageReply reply) { LoadBalancerSystemTags.BALANCER_ACL.delete(msg.getUuid()); } } + if (msg.getHealthCheckTimeout() != null) { + logger.warn(String.format( "rollback health check timeout for listener [uuid:%s]", msg.getUuid())); + if (oldHealthCheckTimeout != null) { + LoadBalancerSystemTags.HEALTH_TIMEOUT.update(msg.getUuid(), + LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag(map( + e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, oldHealthCheckTimeout) + ))); + } else { + LoadBalancerSystemTags.HEALTH_TIMEOUT.delete(msg.getUuid()); + } + } } else { evt.setInventory(LoadBalancerListenerInventory.valueOf(lblVo)); } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java index f48b53c930b..d0edf9139e2 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java @@ -111,6 +111,13 @@ public String toString() { public static final int PROTOCOL_HTTP_DEFAULT_PORT = 80; public static final int PROTOCOL_HTTPS_DEFAULT_PORT = 443; + public static final String DATA_PLANE_HAPROXY = "haproxy"; + public static final String DATA_PLANE_IPVS = "ipvs"; + + public static final String FORWARD_MODE_FULL_NAT = "full_nat"; + public static final String FORWARD_MODE_NAT = "nat"; + public static final String FORWARD_MODE_DR = "dr"; + public static final int DNS_PORT = 53; public static final int SSH_PORT = 22; public static final int ZVR_PORT = 7272; @@ -159,6 +166,9 @@ public static enum Param { public static final int HEALTH_CHECK_INTERVAL_MIN = 1; public static final int HEALTH_CHECK_INTERVAL_MAX = Integer.MAX_VALUE; + public static final int HEALTH_CHECK_TIMEOUT_MIN = 1; + public static final int HEALTH_CHECK_TIMEOUT_MAX = Integer.MAX_VALUE; + public static final int NUMBER_OF_PROCESS_MIN = 1; public static final int NUMBER_OF_PROCESS_MAX = 64; diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java index 3650a9fd9b2..2111af073ea 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerInventory.java @@ -38,6 +38,8 @@ public class LoadBalancerListenerInventory implements Serializable { private Integer loadBalancerPort; private String securityPolicyType; private String protocol; + private String dataPlane; + private String forwardMode; private String serverGroupUuid; private Timestamp createDate; private Timestamp lastOpDate; @@ -55,6 +57,11 @@ public static LoadBalancerListenerInventory valueOf(LoadBalancerListenerVO vo) { inv.setInstancePort(vo.getInstancePort()); inv.setLoadBalancerPort(vo.getLoadBalancerPort()); inv.setProtocol(vo.getProtocol()); + if (LoadBalancerConstants.LB_PROTOCOL_TCP.equals(vo.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(vo.getDataPlane())) { + inv.setDataPlane(vo.getDataPlane()); + inv.setForwardMode(vo.getForwardMode()); + } inv.setSecurityPolicyType(vo.getSecurityPolicyType()); inv.setName(vo.getName()); inv.setDescription(vo.getDescription()); @@ -164,6 +171,22 @@ public void setProtocol(String protocol) { this.protocol = protocol; } + public String getDataPlane() { + return dataPlane; + } + + public void setDataPlane(String dataPlane) { + this.dataPlane = dataPlane; + } + + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public String getSecurityPolicyType() { return securityPolicyType; } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO.java index eb153fd6577..d77c0a8c61c 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO.java @@ -48,6 +48,12 @@ public class LoadBalancerListenerVO extends ResourceVO implements OwnedByAccount @Column private String protocol; + @Column(name = "data_plane") + private String dataPlane; + + @Column(name = "forward_mode") + private String forwardMode; + @Column private String securityPolicyType; @@ -156,6 +162,22 @@ public void setProtocol(String protocol) { this.protocol = protocol; } + public String getDataPlane() { + return dataPlane; + } + + public void setDataPlane(String dataPlane) { + this.dataPlane = dataPlane; + } + + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public String getSecurityPolicyType() { return securityPolicyType; } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO_.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO_.java index 4051b03137b..d3922257359 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO_.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerListenerVO_.java @@ -18,6 +18,8 @@ public class LoadBalancerListenerVO_ extends ResourceVO_ { public static volatile SingularAttribute instancePort; public static volatile SingularAttribute loadBalancerPort; public static volatile SingularAttribute protocol; + public static volatile SingularAttribute dataPlane; + public static volatile SingularAttribute forwardMode; public static volatile SingularAttribute serverGroupUuid; public static volatile SingularAttribute createDate; public static volatile SingularAttribute lastOpDate; diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java index 6b4da585104..fe6c99a0826 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java @@ -740,9 +740,12 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy .eq(LoadBalancerListenerVO_.uuid, resourceUuid) .find(); if (listener != null) { + boolean isTcpIpvsListener = LoadBalancerConstants.LB_PROTOCOL_TCP.equals(listener.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(listener.getDataPlane()); if (LoadBalancerConstants.HEALTH_CHECK_TARGET_PROTOCL_NONE.equals(protocol) && - !LoadBalancerConstants.LB_PROTOCOL_UDP.equals(listener.getProtocol())) { - throw new OperationFailureException(argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10014, "invalid health target[%s], health check protocol none is only supported by udp listener", systemTag)); + !LoadBalancerConstants.LB_PROTOCOL_UDP.equals(listener.getProtocol()) && + !isTcpIpvsListener) { + throw new OperationFailureException(argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10014, "invalid health target[%s], health check protocol none is only supported by udp listener and tcp ipvs listener", systemTag)); } } diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java index 1b4266d6e05..72faff543a0 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java @@ -250,6 +250,7 @@ private VirtualRouterVmInventory findVirtualRouterVm(String lbUuid, List vmNicL3NetworkUuids, vr.get().getUuid(), lbUuid)); throw new CloudRuntimeException("not support separate vr with multiple networks vpc!"); } + return vrInventory; } DebugUtils.Assert(vrs.size() <= 1, String.format("multiple virtual routers[uuids:%s] found", @@ -267,6 +268,8 @@ public static class LbTO { int instancePort; int loadBalancerPort; String mode; + String dataPlane; + String forwardMode; List parameters; String certificateUuid; String securityPolicyType; @@ -462,6 +465,22 @@ public void setMode(String mode) { this.mode = mode; } + public String getDataPlane() { + return dataPlane; + } + + public void setDataPlane(String dataPlane) { + this.dataPlane = dataPlane; + } + + public String getForwardMode() { + return forwardMode; + } + + public void setForwardMode(String forwardMode) { + this.forwardMode = forwardMode; + } + public String getCertificateUuid() { return certificateUuid; } @@ -898,6 +917,8 @@ public LbTO call(LoadBalancerListenerInventory l) { to.setLbUuid(l.getLoadBalancerUuid()); to.setListenerUuid(l.getUuid()); to.setMode(l.getProtocol()); + to.setDataPlane(l.getDataPlane()); + to.setForwardMode(l.getForwardMode()); if (vip != null) { to.setVip(vip.getIp()); } diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java index b5314b49922..6ab8413099c 100644 --- a/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java +++ b/sdk/src/main/java/org/zstack/sdk/ChangeLoadBalancerListenerAction.java @@ -49,6 +49,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, numberRange = {1L,2147483647L}, noTrim = false) public java.lang.Integer healthCheckInterval; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, numberRange = {1L,2147483647L}, noTrim = false) + public java.lang.Integer healthCheckTimeout; + @Param(required = false, validValues = {"tcp","udp","http","none"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String healthCheckProtocol; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateLoadBalancerListenerAction.java b/sdk/src/main/java/org/zstack/sdk/CreateLoadBalancerListenerAction.java index dd4399e0b0f..f0693d3c872 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateLoadBalancerListenerAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateLoadBalancerListenerAction.java @@ -67,6 +67,12 @@ public Result throwExceptionIfError() { @Param(required = false, validValues = {"white","black"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String aclType = "black"; + @Param(required = false, validValues = {"haproxy","ipvs"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String dataPlane; + + @Param(required = false, validValues = {"full_nat","nat","dr"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String forwardMode; + @Param(required = false, validValues = {"tls_cipher_policy_default","tls_cipher_policy_1_0","tls_cipher_policy_1_1","tls_cipher_policy_1_2","tls_cipher_policy_1_2_strict","tls_cipher_policy_1_2_strict_with_1_3"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String securityPolicyType; diff --git a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java index fe361c584e5..5dc0eeecaec 100644 --- a/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/LoadBalancerListenerInventory.java @@ -68,6 +68,22 @@ public java.lang.String getProtocol() { return this.protocol; } + public java.lang.String dataPlane; + public void setDataPlane(java.lang.String dataPlane) { + this.dataPlane = dataPlane; + } + public java.lang.String getDataPlane() { + return this.dataPlane; + } + + public java.lang.String forwardMode; + public void setForwardMode(java.lang.String forwardMode) { + this.forwardMode = forwardMode; + } + public java.lang.String getForwardMode() { + return this.forwardMode; + } + public java.lang.String serverGroupUuid; public void setServerGroupUuid(java.lang.String serverGroupUuid) { this.serverGroupUuid = serverGroupUuid; diff --git a/test/src/test/groovy/org/zstack/test/integration/db/schema/CheckSchemaUpgradeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/db/schema/CheckSchemaUpgradeCase.groovy index 91d985ce50e..773206c7303 100644 --- a/test/src/test/groovy/org/zstack/test/integration/db/schema/CheckSchemaUpgradeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/db/schema/CheckSchemaUpgradeCase.groovy @@ -32,5 +32,19 @@ class CheckSchemaUpgradeCase extends SubCase { @Override void test() { + testLoadBalancerListenerDataPlaneUpgradeSchema() + } + + void testLoadBalancerListenerDataPlaneUpgradeSchema() { + String upgradeSchemaDir = Paths.get("../conf/db/upgrade").toAbsolutePath().normalize().toString() + File schema = new File(upgradeSchemaDir + "/V5.5.28__schema.sql") + assert schema.exists() + + String sql = schema.text + assert sql.contains("CALL ADD_COLUMN('LoadBalancerListenerVO', 'data_plane', 'VARCHAR(32)', 1, NULL)") + assert sql.contains("CALL ADD_COLUMN('LoadBalancerListenerVO', 'forward_mode', 'VARCHAR(32)', 1, NULL)") + assert sql.contains("WHERE protocol = 'udp' AND data_plane IS NULL") + assert sql.contains("SET data_plane = 'haproxy'") + assert sql.contains("WHERE data_plane IS NULL") } } diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy new file mode 100644 index 00000000000..3fe612e0641 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -0,0 +1,499 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.loadbalancer + +import org.zstack.core.db.Q +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.network.service.eip.EipConstant +import org.zstack.network.service.lb.LoadBalancerConstants +import org.zstack.network.service.lb.LoadBalancerListenerVO +import org.zstack.network.service.lb.LoadBalancerListenerVO_ +import org.zstack.network.service.lb.LoadBalancerVO +import org.zstack.network.service.lb.LoadBalancerVO_ +import org.zstack.network.service.portforwarding.PortForwardingConstant +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerBackend +import org.zstack.network.service.virtualrouter.vyos.VyosConstants +import org.zstack.kvm.KVMConstant +import org.zstack.kvm.KVMAgentCommands +import org.zstack.sdk.ChangeLoadBalancerListenerAction +import org.zstack.sdk.ApiResult +import org.zstack.sdk.CreateLoadBalancerListenerAction +import org.zstack.sdk.CreateLoadBalancerListenerResult +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.LoadBalancerInventory +import org.zstack.sdk.LoadBalancerListenerInventory +import org.zstack.sdk.LoadBalancerServerGroupInventory +import org.zstack.sdk.VipInventory +import org.zstack.sdk.VmInstanceInventory +import org.zstack.sdk.ZSClient +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil +import org.springframework.http.HttpEntity + +class TcpIpvsLoadBalancerListenerApiCase extends SubCase { + EnvSpec env + LoadBalancerInventory lb + List refreshCmds = [] + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(1) + cpu = 1 + } + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "vr" + url = "http://zstack.org/download/vr.qcow2" + } + } + zone { + name = "zone" + + cluster { + name = "cluster" + hypervisorType = "KVM" + + kvm { + name = "kvm" + managementIp = "localhost" + username = "root" + password = "password" + totalCpu = 8 + totalMem = SizeUnit.GIGABYTE.toByte(12) + } + + attachPrimaryStorage("local") + attachL2Network("l2") + } + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "publicL3" + ip { + startIp = "172.20.58.160" + endIp = "172.20.58.200" + gateway = "172.20.0.1" + netmask = "255.255.0.0" + } + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString(), + PortForwardingConstant.PORTFORWARDING_NETWORK_SERVICE_TYPE, + LoadBalancerConstants.LB_NETWORK_SERVICE_TYPE_STRING, + EipConstant.EIP_NETWORK_SERVICE_TYPE] + } + } + + l3Network { + name = "backendL3" + ip { + startIp = "10.58.0.160" + endIp = "10.58.0.200" + gateway = "10.58.0.1" + netmask = "255.255.255.0" + } + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString(), + LoadBalancerConstants.LB_NETWORK_SERVICE_TYPE_STRING] + } + } + + l3Network { + name = "managementL3" + ip { + startIp = "172.21.58.160" + endIp = "172.21.58.200" + gateway = "172.21.0.1" + netmask = "255.255.0.0" + } + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString()] + } + } + } + attachBackupStorage("sftp") + + virtualRouterOffering { + name = "vro" + memory = SizeUnit.MEGABYTE.toByte(512) + cpu = 2 + useManagementL3Network("managementL3") + usePublicL3Network("publicL3") + useImage("vr") + isDefault = true + } + } + + vm { + name = "backend-vm-1" + useImage("image") + useL3Networks("backendL3") + useInstanceOffering("instanceOffering") + } + + vm { + name = "backend-vm-2" + useImage("image") + useL3Networks("backendL3") + useInstanceOffering("instanceOffering") + } + + vm { + name = "backend-vm-3" + useImage("image") + useL3Networks("backendL3") + useInstanceOffering("instanceOffering") + } + + vm { + name = "backend-vm-4" + useImage("image") + useL3Networks("backendL3") + useInstanceOffering("instanceOffering") + } + } + } + + @Override + void test() { + env.create { + prepareDedicatedLoadBalancer() + testTcpHaproxyDefaultDataPlane() + testTcpIpvsFullNatListener() + testTcpIpvsDefaultForwardMode() + testTcpIpvsCreateValidation() + testTcpIpvsDedicatedListenerBackendRefreshPayload() + } + } + + void prepareDedicatedLoadBalancer() { + L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory + VipInventory vip = createVip { + delegate.name = "tcp-ipvs-api-vip" + delegate.l3NetworkUuid = publicL3.uuid + } + + lb = createLoadBalancer { + delegate.name = "tcp-ipvs-api-lb" + delegate.vipUuid = vip.uuid + delegate.systemTags = ["separateVirtualRouterVm"] + } + } + + void testTcpHaproxyDefaultDataPlane() { + CreateLoadBalancerListenerAction action = new CreateLoadBalancerListenerAction() + action.name = "tcp-haproxy-default" + action.loadBalancerUuid = lb.uuid + action.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + action.loadBalancerPort = 11080 + action.instancePort = 8080 + action.sessionId = adminSession() + + ApiResult result = ZSClient.call(action) + assert result.error == null + + String rawResult = getApiResultString(result) + assert !rawResult.contains("\"dataPlane\"") + assert !rawResult.contains("\"forwardMode\"") + + LoadBalancerListenerInventory listener = result.getResult(CreateLoadBalancerListenerResult.class).inventory + + assert listener.dataPlane == null + assert listener.forwardMode == null + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.dataPlane == LoadBalancerConstants.DATA_PLANE_HAPROXY + assert vo.forwardMode == null + } + + void testTcpIpvsFullNatListener() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-full-nat" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11081 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + + LoadBalancerListenerVO vo = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listener.uuid) + .find() + assert vo.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert vo.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + assert vo.instancePort == 8080 + } + + void testTcpIpvsDefaultForwardMode() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-ipvs-default-forward-mode" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11082 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + } + + assert listener.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert listener.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + } + + void testTcpIpvsCreateValidation() { + assertCreateListenerError(11083, LoadBalancerConstants.LB_PROTOCOL_HTTP, + LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) + assertCreateListenerError(11084, LoadBalancerConstants.LB_PROTOCOL_UDP, + LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) + assertCreateListenerError(11085, LoadBalancerConstants.LB_PROTOCOL_TCP, + LoadBalancerConstants.DATA_PLANE_HAPROXY, LoadBalancerConstants.FORWARD_MODE_FULL_NAT) + CreateLoadBalancerListenerAction.Result natResult = assertCreateListenerError(11086, LoadBalancerConstants.LB_PROTOCOL_TCP, + LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_NAT) + assert natResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") + + CreateLoadBalancerListenerAction.Result drResult = assertCreateListenerError(11087, LoadBalancerConstants.LB_PROTOCOL_TCP, + LoadBalancerConstants.DATA_PLANE_IPVS, LoadBalancerConstants.FORWARD_MODE_DR) + assert drResult.error.details.contains("TCP IPVS only supports forwardMode[full_nat]") + } + + void testTcpIpvsDedicatedListenerBackendRefreshPayload() { + env.afterSimulator(VirtualRouterLoadBalancerBackend.REFRESH_LB_PATH) { rsp, HttpEntity e -> + refreshCmds.add(JSONObjectUtil.toObject(e.body, VirtualRouterLoadBalancerBackend.RefreshLbCmd.class)) + return rsp + } + List destroyVmCmds = [] + env.afterSimulator(KVMConstant.KVM_DESTROY_VM_PATH) { rsp, HttpEntity e -> + destroyVmCmds.add(JSONObjectUtil.toObject(e.body, KVMAgentCommands.DestroyVmCmd.class)) + return rsp + } + + LoadBalancerListenerInventory listener = createTcpIpvsListener("tcp-ipvs-t2", 11090, LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN) + + String backendIp1 = backendIp("backend-vm-1") + String backendIp2 = backendIp("backend-vm-2") + String backendIp3 = backendIp("backend-vm-3") + String backendIp4 = backendIp("backend-vm-4") + + LoadBalancerServerGroupInventory group1 = createServerGroupWithVmNics( + "tcp-ipvs-t2-group-1", + [backendVmNic("backend-vm-1", "100")]) + addServerGroupToLoadBalancerListener { + listenerUuid = listener.uuid + serverGroupUuid = group1.uuid + } + + VirtualRouterLoadBalancerBackend.LbTO to = lastLbTO(listener.uuid) + assertTcpIpvsTO(to, listener.uuid, 11090, LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN) + assertServerGroups(to, [ + (group1.uuid): [(backendIp1): 100L] + ]) + + addBackendServerToServerGroup { + serverGroupUuid = group1.uuid + vmNics = [backendVmNic("backend-vm-2", "50"), backendVmNic("backend-vm-4", "80")] + } + to = lastLbTO(listener.uuid) + assertServerGroups(to, [ + (group1.uuid): [(backendIp1): 100L, (backendIp2): 50L, (backendIp4): 80L] + ]) + + changeLoadBalancerBackendServer { + serverGroupUuid = group1.uuid + vmNics = [backendVmNic("backend-vm-2", "0")] + } + to = lastLbTO(listener.uuid) + assertServerGroups(to, [ + (group1.uuid): [(backendIp1): 100L, (backendIp2): 0L, (backendIp4): 80L] + ]) + + LoadBalancerServerGroupInventory group2 = createServerGroupWithVmNics( + "tcp-ipvs-t2-group-2", + [backendVmNic("backend-vm-3", "30")]) + addServerGroupToLoadBalancerListener { + listenerUuid = listener.uuid + serverGroupUuid = group2.uuid + } + to = lastLbTO(listener.uuid) + assertServerGroups(to, [ + (group1.uuid): [(backendIp1): 100L, (backendIp2): 0L, (backendIp4): 80L], + (group2.uuid): [(backendIp3): 30L] + ]) + + [ + LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN, + LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN, + LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN, + LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_SOURCE + ].each { String algorithm -> + int offset = refreshCmds.size() + changeBalancerAlgorithm(listener.uuid, algorithm) + to = lastLbTO(listener.uuid, offset) + assertTcpIpvsTO(to, listener.uuid, 11090, algorithm) + } + + assert Q.New(LoadBalancerVO.class) + .select(LoadBalancerVO_.providerType) + .eq(LoadBalancerVO_.uuid, lb.uuid) + .findValue() == VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + + int destroyOffset = destroyVmCmds.size() + deleteLoadBalancer { + uuid = lb.uuid + } + assert destroyVmCmds.size() > destroyOffset + assert destroyVmCmds.drop(destroyOffset).any { it.uuid != null } + } + + LoadBalancerListenerInventory createTcpIpvsListener(String name, int port, String algorithm) { + return createLoadBalancerListener { + delegate.name = name + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = port + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + delegate.systemTags = ["balancerAlgorithm::${algorithm}".toString()] + } + } + + Map backendVmNic(String vmName, String weight) { + return ["uuid": backendVmNicUuid(vmName), "weight": weight] + } + + String backendVmNicUuid(String vmName) { + return (env.inventoryByName(vmName) as VmInstanceInventory).vmNics[0].uuid + } + + String backendIp(String vmName) { + return (env.inventoryByName(vmName) as VmInstanceInventory).vmNics[0].ip + } + + LoadBalancerServerGroupInventory createServerGroupWithVmNics(String groupName, List> backendNics) { + LoadBalancerServerGroupInventory group = createLoadBalancerServerGroup { + loadBalancerUuid = lb.uuid + name = groupName + } + addBackendServerToServerGroup { + serverGroupUuid = group.uuid + vmNics = backendNics + } + return group + } + + VirtualRouterLoadBalancerBackend.LbTO lastLbTO(String listenerUuid, int offset = 0) { + long deadline = System.currentTimeMillis() + 5000 + VirtualRouterLoadBalancerBackend.LbTO to = null + while (System.currentTimeMillis() < deadline) { + if (refreshCmds.size() > offset) { + to = refreshCmds.drop(offset).reverse() + .collectMany { it.lbs } + .find { it.listenerUuid == listenerUuid } + if (to != null) { + break + } + } + Thread.sleep(100) + } + assert refreshCmds.size() > offset + assert to != null + return to + } + + void assertTcpIpvsTO(VirtualRouterLoadBalancerBackend.LbTO to, String listenerUuid, int port, String algorithm) { + assert to.listenerUuid == listenerUuid + assert to.mode == LoadBalancerConstants.LB_PROTOCOL_TCP + assert to.dataPlane == LoadBalancerConstants.DATA_PLANE_IPVS + assert to.forwardMode == LoadBalancerConstants.FORWARD_MODE_FULL_NAT + assert to.loadBalancerPort == port + assert to.instancePort == 8080 + assert to.parameters.contains("balancerAlgorithm::${algorithm}".toString()) + } + + void assertServerGroups(VirtualRouterLoadBalancerBackend.LbTO to, Map> expected) { + assert to.serverGroups.size() == expected.size() + expected.each { String groupUuid, Map servers -> + def group = to.serverGroups.find { it.serverGroupUuid == groupUuid } + assert group != null + assert group.backendServers.size() == servers.size() + servers.each { String ip, Long weight -> + def server = group.backendServers.find { it.ip == ip } + assert server != null + assert server.weight == weight + assert to.nicIps.contains(ip) + assert to.parameters.contains("balancerWeight::${ip}::${weight}".toString()) + } + } + } + + void changeBalancerAlgorithm(String listenerUuid, String algorithm) { + ChangeLoadBalancerListenerAction action = new ChangeLoadBalancerListenerAction() + action.uuid = listenerUuid + action.balancerAlgorithm = algorithm + action.sessionId = adminSession() + ChangeLoadBalancerListenerAction.Result result = action.call() + assert result.error == null + } + + CreateLoadBalancerListenerAction.Result assertCreateListenerError(int port, String protocol, String dataPlane, String forwardMode) { + CreateLoadBalancerListenerAction action = new CreateLoadBalancerListenerAction() + action.name = "tcp-ipvs-invalid-${port}" + action.loadBalancerUuid = lb.uuid + action.protocol = protocol + action.loadBalancerPort = port + action.instancePort = 8080 + action.dataPlane = dataPlane + action.forwardMode = forwardMode + action.sessionId = adminSession() + + CreateLoadBalancerListenerAction.Result result = action.call() + assert result.error != null + return result + } + + String getApiResultString(ApiResult result) { + def field = ApiResult.class.getDeclaredField("resultString") + field.setAccessible(true) + return field.get(result) as String + } + + @Override + void clean() { + env.delete() + } +} diff --git a/test/src/test/java/org/zstack/test/lb/TestVirtualRouterLb13.java b/test/src/test/java/org/zstack/test/lb/TestVirtualRouterLb13.java index 477c43f3d6a..d110a447e11 100755 --- a/test/src/test/java/org/zstack/test/lb/TestVirtualRouterLb13.java +++ b/test/src/test/java/org/zstack/test/lb/TestVirtualRouterLb13.java @@ -14,6 +14,7 @@ import org.zstack.simulator.appliancevm.ApplianceVmSimulatorConfig; import org.zstack.simulator.virtualrouter.VirtualRouterSimulatorConfig; import org.zstack.test.Api; +import org.zstack.test.ApiSender; import org.zstack.test.ApiSenderException; import org.zstack.test.DBUtil; import org.zstack.test.WebBeanConstructor; @@ -322,5 +323,37 @@ public String call(String arg) { } }); Assert.assertNotNull(tval); + + final Integer changedHealthTimeout = 12; + final String changedHealthTimeoutTag = LoadBalancerSystemTags.HEALTH_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN, changedHealthTimeout)) + ); + APIChangeLoadBalancerListenerMsg cmsg = new APIChangeLoadBalancerListenerMsg(); + cmsg.setUuid(l2.getUuid()); + cmsg.setHealthCheckTimeout(changedHealthTimeout); + cmsg.setSession(session); + ApiSender sender = new ApiSender(); + vconfig.refreshLbCmds.clear(); + sender.send(cmsg, APIChangeLoadBalancerListenerEvent.class); + + String changedTimeout = LoadBalancerSystemTags.HEALTH_TIMEOUT.getTokenByResourceUuid(l2.getUuid(), + LoadBalancerSystemTags.HEALTH_TIMEOUT_TOKEN); + Assert.assertEquals(changedHealthTimeout, Integer.valueOf(changedTimeout)); + + cmd = vconfig.refreshLbCmds.get(vconfig.refreshLbCmds.size() - 1); + to = CollectionUtils.find(cmd.getLbs(), new Function() { + @Override + public LbTO call(LbTO arg) { + return arg.getListenerUuid().equals(l2.getUuid()) ? arg : null; + } + }); + Assert.assertNotNull(to); + tval = CollectionUtils.find(to.getParameters(), new Function() { + @Override + public String call(String arg) { + return arg.equals(changedHealthTimeoutTag) ? arg : null; + } + }); + Assert.assertNotNull(JSONObjectUtil.toJsonString(to), tval); } }