From 349d994e68f7813e1c9890980da0d9a50d8a98c8 Mon Sep 17 00:00:00 2001 From: Milanka00 Date: Sat, 15 Aug 2026 08:21:49 +0530 Subject: [PATCH] Adding performance test scripts --- gateway/perf/README.md | 33 + gateway/perf/api-gateway-eks-perf/README.md | 16 + gateway/perf/api-gateway-eks-perf/cleanup.sh | 67 ++ .../create-jmeter-ec2s.sh | 134 ++++ .../eks-cluster-perf.yaml.template | 41 ++ .../publish-results-pr.sh | 236 +++++++ .../run-api-gateway-eks-tests.sh | 626 ++++++++++++++++++ .../perf/performance-test-scripts/README.md | 178 +++++ .../api-gateway/config.perf-overlay.toml | 108 +++ .../deploy/create-rest-perf-api.sh | 182 +++++ .../api-gateway/eks/backend-mock-eks.yaml | 67 ++ .../eks/deploy-apis-eks-minimal.sh | 104 +++ .../api-gateway/eks/eks-common.sh | 563 ++++++++++++++++ .../api-gateway/eks/env.eks.example | 111 ++++ .../api-gateway/eks/install-gateway.sh | 83 +++ .../api-gateway/eks/values.perf.yaml | 13 + .../common.env.example | 8 + .../jmeter/api-api-test-gateway-jwt-plain.jmx | 153 +++++ .../jmeter/api-api-test-gateway-plain.jmx | 137 ++++ .../jmeter/env.example | 22 + .../jmeter/env.jmeter.api.example | 31 + .../jmeter/gateway-scenarios.sh | 92 +++ .../jmeter/generate-jwt-tokens.sh | 134 ++++ .../jmeter/generate-summary.sh | 131 ++++ .../jmeter/jmeter-server-start.sh | 7 + .../jmeter/lib/gateway-profile.sh | 17 + .../jmeter/run-scenario.sh | 361 ++++++++++ .../jmeter/setup-jmeter.sh | 56 ++ .../performance-test-scripts/lib/common.sh | 151 +++++ 29 files changed, 3862 insertions(+) create mode 100644 gateway/perf/README.md create mode 100644 gateway/perf/api-gateway-eks-perf/README.md create mode 100755 gateway/perf/api-gateway-eks-perf/cleanup.sh create mode 100755 gateway/perf/api-gateway-eks-perf/create-jmeter-ec2s.sh create mode 100644 gateway/perf/api-gateway-eks-perf/eks-cluster-perf.yaml.template create mode 100755 gateway/perf/api-gateway-eks-perf/publish-results-pr.sh create mode 100755 gateway/perf/api-gateway-eks-perf/run-api-gateway-eks-tests.sh create mode 100644 gateway/perf/performance-test-scripts/README.md create mode 100644 gateway/perf/performance-test-scripts/api-gateway/config.perf-overlay.toml create mode 100755 gateway/perf/performance-test-scripts/api-gateway/deploy/create-rest-perf-api.sh create mode 100644 gateway/perf/performance-test-scripts/api-gateway/eks/backend-mock-eks.yaml create mode 100755 gateway/perf/performance-test-scripts/api-gateway/eks/deploy-apis-eks-minimal.sh create mode 100755 gateway/perf/performance-test-scripts/api-gateway/eks/eks-common.sh create mode 100644 gateway/perf/performance-test-scripts/api-gateway/eks/env.eks.example create mode 100755 gateway/perf/performance-test-scripts/api-gateway/eks/install-gateway.sh create mode 100644 gateway/perf/performance-test-scripts/api-gateway/eks/values.perf.yaml create mode 100644 gateway/perf/performance-test-scripts/common.env.example create mode 100644 gateway/perf/performance-test-scripts/jmeter/api-api-test-gateway-jwt-plain.jmx create mode 100644 gateway/perf/performance-test-scripts/jmeter/api-api-test-gateway-plain.jmx create mode 100644 gateway/perf/performance-test-scripts/jmeter/env.example create mode 100644 gateway/perf/performance-test-scripts/jmeter/env.jmeter.api.example create mode 100755 gateway/perf/performance-test-scripts/jmeter/gateway-scenarios.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/generate-jwt-tokens.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/generate-summary.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/jmeter-server-start.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/lib/gateway-profile.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/run-scenario.sh create mode 100755 gateway/perf/performance-test-scripts/jmeter/setup-jmeter.sh create mode 100755 gateway/perf/performance-test-scripts/lib/common.sh diff --git a/gateway/perf/README.md b/gateway/perf/README.md new file mode 100644 index 000000000..fd6d28a9d --- /dev/null +++ b/gateway/perf/README.md @@ -0,0 +1,33 @@ +# API Platform Gateway Performance Test Results + + + +**4 Cpu Gateway Runtime** + +| Scenario | Users | Throughput | Avg Response Time(ms) | Err % | p90 (ms) | p99 (ms) | Samples | +| --- | --- | --- | --- | --- | --- | --- | --- | +| API Gateway JWT GET | 100 | 7205.05 | 13.69 | 0.00 | 20.00 | 27.00 | 5181719 | +| API Gateway JWT GET | 500 | 7629.20 | 65.18 | 0.00 | 85.00 | 110.00 | 5487063 | +| API Gateway JWT GET | 1000 | 7445.64 | 132.50 | 0.00 | 177.00 | 240.00 | 5357118 | +| API Gateway Plain GET | 100 | 9521.54 | 10.33 | 0.00 | 15.00 | 21.00 | 6849061 | +| API Gateway Plain GET | 500 | 9339.11 | 53.15 | 0.00 | 67.00 | 84.00 | 6716685 | +| API Gateway Plain GET | 1000 | 9202.58 | 108.32 | 0.00 | 133.00 | 163.00 | 6621784 | +| API Gateway Header Policy GET | 100 | 8879.21 | 11.09 | 0.00 | 16.00 | 23.00 | 6386256 | +| API Gateway Header Policy GET | 500 | 8706.58 | 57.19 | 0.00 | 74.00 | 93.00 | 6262678 | +| API Gateway Header Policy GET | 1000 | 8667.24 | 114.66 | 0.00 | 143.00 | 177.00 | 6236016 | + +**2 Cpu Gateway Runtime** + +| Scenario | Users | Throughput | Avg Response Time(ms) | Err % | p90 (ms) | p99 (ms) | Samples | +| --- | --- | --- | --- | --- | --- | --- | --- | +| API Gateway JWT GET | 100 | 5278.32 | 18.74 | 0.00 | 40.00 | 51.00 | 3796087 | +| API Gateway JWT GET | 500 | 4939.69 | 100.40 | 0.00 | 117.00 | 153.00 | 3553591 | +| API Gateway JWT GET | 1000 | 4972.15 | 200.46 | 0.00 | 229.00 | 265.00 | 3577539 | +| API Gateway Plain GET | 100 | 6149.57 | 16.07 | 0.00 | 35.00 | 45.00 | 4423998 | +| API Gateway Plain GET | 500 | 5707.35 | 87.12 | 0.00 | 105.00 | 137.00 | 4105722 | +| API Gateway Plain GET | 1000 | 5598.03 | 178.28 | 0.00 | 203.00 | 245.00 | 4027863 | +| API Gateway Header Policy GET | 100 | 5751.46 | 17.19 | 0.00 | 37.00 | 47.00 | 4136573 | +| API Gateway Header Policy GET | 500 | 5345.05 | 93.27 | 0.00 | 108.00 | 143.00 | 3845566 | +| API Gateway Header Policy GET | 1000 | 5312.74 | 186.91 | 0.00 | 213.00 | 257.00 | 3822285 | + + diff --git a/gateway/perf/api-gateway-eks-perf/README.md b/gateway/perf/api-gateway-eks-perf/README.md new file mode 100644 index 000000000..35b2e53e1 --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/README.md @@ -0,0 +1,16 @@ +# `api-gateway-eks-perf` — Jenkins slave orchestrator + +This directory lives on the **Jenkins slave** perf workspace. It is the job entrypoint; it does **not** contain JMeter scenarios or RestApi deploy logic. + +Those live in **`../performance-test-scripts/`** (in git: `gateway/perf/performance-test-scripts`), which this orchestrator sparse-clones via `PERF_SCRIPTS` on every run. + +| File | Role | +|------|------| +| `run-api-gateway-eks-tests.sh` | Full pipeline (EKS → gateway → JMeter → results → cleanup) | +| `create-jmeter-ec2s.sh` | 1 client + 2 server EC2s in the EKS VPC | +| `eks-cluster-perf.yaml.template` | eksctl cluster template | +| `cleanup.sh` | EXIT trap: terminate EC2s, delete cluster | + +**To extend scenarios / RestApis / JMX:** edit `performance-test-scripts` and point `PERF_SCRIPTS` at your branch. See that folder’s [README](../performance-test-scripts/README.md). + +**Change this folder only when:** job infra changes (EC2 counts, VPC wiring, clone paths, `env.eks` generation defaults, cleanup). diff --git a/gateway/perf/api-gateway-eks-perf/cleanup.sh b/gateway/perf/api-gateway-eks-perf/cleanup.sh new file mode 100755 index 000000000..b9179c271 --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/cleanup.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Clean up all resources created by run-api-gateway-eks-tests.sh. +# Called from the EXIT trap; must not fail hard (|| true guards). +# +# Required env vars: +# EKS_CLUSTER_NAME, AWS_REGION +# STATE_FILE path to the jmeter state file written by create-jmeter-ec2s.sh + +set -uo pipefail + +AWS="aws --region ${AWS_REGION:-us-east-1}" + +echo "==> Cleanup: EKS cluster ${EKS_CLUSTER_NAME:-}, region ${AWS_REGION:-us-east-1}" + +# Load JMeter EC2 state if it exists. +if [[ -f "${STATE_FILE:-}" ]]; then + # shellcheck source=/dev/null + source "${STATE_FILE}" +fi + +# Terminate JMeter EC2s first. +INSTANCE_IDS=() +for var in CLIENT_ID SERVER1_ID SERVER2_ID; do + id="${!var:-}" + [[ -n "$id" && "$id" != "None" ]] && INSTANCE_IDS+=("$id") +done + +if [[ ${#INSTANCE_IDS[@]} -gt 0 ]]; then + echo "==> Terminating JMeter EC2s: ${INSTANCE_IDS[*]}" + ${AWS} ec2 terminate-instances --instance-ids "${INSTANCE_IDS[@]}" >/dev/null 2>&1 || true + echo " Waiting for termination..." + ${AWS} ec2 wait instance-terminated --instance-ids "${INSTANCE_IDS[@]}" 2>/dev/null || true + echo " JMeter EC2s terminated." +fi + +# Remove JMeter SG rule from EKS node SG. +if [[ -n "${NODE_SG:-}" && -n "${JMETER_SG_ID:-}" ]]; then + echo "==> Removing JMeter SG rule from EKS node SG ${NODE_SG}" + ${AWS} ec2 revoke-security-group-ingress \ + --group-id "${NODE_SG}" \ + --ip-permissions \ + "IpProtocol=tcp,FromPort=0,ToPort=65535,UserIdGroupPairs=[{GroupId=${JMETER_SG_ID}}]" \ + 2>/dev/null || true +fi + +# Delete JMeter security group (must wait for EC2s to terminate first). +if [[ -n "${JMETER_SG_ID:-}" ]]; then + echo "==> Deleting JMeter security group ${JMETER_SG_ID}" + ${AWS} ec2 delete-security-group --group-id "${JMETER_SG_ID}" 2>/dev/null || \ + echo " (SG delete failed — may have dependent resources still detaching; manual cleanup needed)" +fi + +# Delete EKS cluster. +if [[ -n "${EKS_CLUSTER_NAME:-}" ]]; then + echo "==> Deleting EKS cluster ${EKS_CLUSTER_NAME} (this takes 10-20 min, runs async)..." + eksctl delete cluster --name "${EKS_CLUSTER_NAME}" --region "${AWS_REGION:-us-east-1}" \ + --wait 2>/dev/null || true + echo " EKS cluster deletion initiated." +fi + +# Remove per-run state file. +if [[ -f "${STATE_FILE:-}" ]]; then + rm -f "${STATE_FILE}" + echo " Removed state file ${STATE_FILE}" +fi + +echo "==> Cleanup complete." diff --git a/gateway/perf/api-gateway-eks-perf/create-jmeter-ec2s.sh b/gateway/perf/api-gateway-eks-perf/create-jmeter-ec2s.sh new file mode 100755 index 000000000..7cd1f2dcf --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/create-jmeter-ec2s.sh @@ -0,0 +1,134 @@ +#!/bin/bash -e +# Create 3 JMeter EC2s (1 client + 2 servers) in the EKS VPC and write state to a file. +# +# Required env vars (set by run-api-gateway-eks-tests.sh): +# EKS_CLUSTER_NAME, AWS_REGION, JMETER_SG_TAG, JMETER_KEY_NAME, +# JMETER_CLIENT_EC2_INSTANCE_TYPE, JMETER_SERVER_EC2_INSTANCE_TYPE, +# STATE_FILE (path to write instance IDs and IPs) + +set -euo pipefail + +: "${EKS_CLUSTER_NAME:?}" +: "${AWS_REGION:?}" +: "${JMETER_KEY_NAME:?}" +: "${JMETER_CLIENT_EC2_INSTANCE_TYPE:?}" +: "${JMETER_SERVER_EC2_INSTANCE_TYPE:?}" +: "${STATE_FILE:?}" + +AWS="aws --region ${AWS_REGION}" + +echo "==> Getting EKS VPC info for cluster ${EKS_CLUSTER_NAME}" +VPC_ID=$(${AWS} eks describe-cluster --name "${EKS_CLUSTER_NAME}" \ + --query "cluster.resourcesVpcConfig.vpcId" --output text) +echo " VPC: ${VPC_ID}" + +# Get a public subnet in the EKS VPC (tagged by eksctl for public load balancer). +PUBLIC_SUBNET_ID=$(${AWS} ec2 describe-subnets \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + "Name=tag:kubernetes.io/role/elb,Values=1" \ + "Name=state,Values=available" \ + --query "Subnets[0].SubnetId" --output text) + +if [[ -z "${PUBLIC_SUBNET_ID}" || "${PUBLIC_SUBNET_ID}" == "None" ]]; then + echo "ERROR: No public subnet (tag kubernetes.io/role/elb=1) found in VPC ${VPC_ID}." >&2 + echo " eksctl usually creates these; check VPC subnet tags." >&2 + exit 1 +fi +echo " Subnet: ${PUBLIC_SUBNET_ID}" + +# Create security group for JMeter EC2s. +JMETER_SG_NAME="${JMETER_SG_TAG:-jmeter-perf}" +echo "==> Creating JMeter security group: ${JMETER_SG_NAME}" +JMETER_SG_ID=$(${AWS} ec2 create-security-group \ + --group-name "${JMETER_SG_NAME}" \ + --description "JMeter perf test instances for ${EKS_CLUSTER_NAME}" \ + --vpc-id "${VPC_ID}" \ + --query "GroupId" --output text) +echo " JMeter SG: ${JMETER_SG_ID}" + +# Allow SSH from anywhere (Jenkins slave can SSH to JMeter EC2s). +${AWS} ec2 authorize-security-group-ingress \ + --group-id "${JMETER_SG_ID}" \ + --protocol tcp --port 22 --cidr 0.0.0.0/0 + +# Allow all TCP between JMeter instances themselves (RMI + results callbacks). +${AWS} ec2 authorize-security-group-ingress \ + --group-id "${JMETER_SG_ID}" \ + --ip-permissions \ + "IpProtocol=tcp,FromPort=0,ToPort=65535,UserIdGroupPairs=[{GroupId=${JMETER_SG_ID}}]" + +# Get EKS cluster security group and allow JMeter access on NodePort range. +NODE_SG=$(${AWS} eks describe-cluster --name "${EKS_CLUSTER_NAME}" \ + --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) +echo " EKS cluster SG: ${NODE_SG}" + +${AWS} ec2 authorize-security-group-ingress \ + --group-id "${NODE_SG}" \ + --ip-permissions \ + "IpProtocol=tcp,FromPort=0,ToPort=65535,UserIdGroupPairs=[{GroupId=${JMETER_SG_ID}}]" + +# Get latest Amazon Linux 2023 AMI. +AMI_ID=$(${AWS} ssm get-parameter \ + --name /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 \ + --query "Parameter.Value" --output text) +echo "==> Using AMI: ${AMI_ID}" + +launch_ec2() { + local role="$1" instance_type="$2" + ${AWS} ec2 run-instances \ + --image-id "${AMI_ID}" \ + --instance-type "${instance_type}" \ + --key-name "${JMETER_KEY_NAME}" \ + --security-group-ids "${JMETER_SG_ID}" \ + --subnet-id "${PUBLIC_SUBNET_ID}" \ + --associate-public-ip-address \ + --block-device-mappings "DeviceName=/dev/xvda,Ebs={VolumeSize=50,VolumeType=gp3,DeleteOnTermination=true}" \ + --tag-specifications \ + "ResourceType=instance,Tags=[{Key=Name,Value=${JMETER_SG_NAME}-${role}},{Key=project,Value=${EKS_CLUSTER_NAME}},{Key=jmeter-perf-tag,Value=${JMETER_SG_NAME}}]" \ + --query "Instances[0].InstanceId" --output text +} + +echo "==> Launching JMeter EC2s" +CLIENT_ID=$(launch_ec2 "client" "${JMETER_CLIENT_EC2_INSTANCE_TYPE}") +SERVER1_ID=$(launch_ec2 "server-1" "${JMETER_SERVER_EC2_INSTANCE_TYPE}") +SERVER2_ID=$(launch_ec2 "server-2" "${JMETER_SERVER_EC2_INSTANCE_TYPE}") +echo " client: ${CLIENT_ID}" +echo " server-1: ${SERVER1_ID}" +echo " server-2: ${SERVER2_ID}" + +echo "==> Waiting for instances to be running (~60s)..." +${AWS} ec2 wait instance-running --instance-ids "${CLIENT_ID}" "${SERVER1_ID}" "${SERVER2_ID}" + +# Get public + private IPs. +get_ip() { + local id="$1" type="$2" + ${AWS} ec2 describe-instances --instance-ids "${id}" \ + --query "Reservations[0].Instances[0].${type}IpAddress" --output text +} + +CLIENT_PUBLIC_IP=$(get_ip "${CLIENT_ID}" Public) +SERVER1_PUBLIC_IP=$(get_ip "${SERVER1_ID}" Public) +SERVER2_PUBLIC_IP=$(get_ip "${SERVER2_ID}" Public) +SERVER1_PRIVATE_IP=$(get_ip "${SERVER1_ID}" Private) +SERVER2_PRIVATE_IP=$(get_ip "${SERVER2_ID}" Private) + +echo "" +echo " client ${CLIENT_ID} public=${CLIENT_PUBLIC_IP}" +echo " server-1 ${SERVER1_ID} public=${SERVER1_PUBLIC_IP} private=${SERVER1_PRIVATE_IP}" +echo " server-2 ${SERVER2_ID} public=${SERVER2_PUBLIC_IP} private=${SERVER2_PRIVATE_IP}" + +# Write state file for use by main script and cleanup. +cat >"${STATE_FILE}" < State written to ${STATE_FILE}" diff --git a/gateway/perf/api-gateway-eks-perf/eks-cluster-perf.yaml.template b/gateway/perf/api-gateway-eks-perf/eks-cluster-perf.yaml.template new file mode 100644 index 000000000..051dcd803 --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/eks-cluster-perf.yaml.template @@ -0,0 +1,41 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: ${EKS_CLUSTER_NAME} + region: ${AWS_REGION} + version: "${EKS_K8S_VERSION}" + +# OIDC is required for IRSA (EBS CSI driver service account). +iam: + withOIDC: true + +addons: + - name: aws-ebs-csi-driver + wellKnownPolicies: + ebsCSIController: true + +managedNodeGroups: + - name: gateway-ng + instanceType: ${EKS_NODE_INSTANCE_TYPE} + desiredCapacity: ${EKS_GATEWAY_NODE_DESIRED} + minSize: 1 + maxSize: 8 + volumeSize: 50 + privateNetworking: true + labels: + role: gateway-perf + tags: + project: ${EKS_CLUSTER_NAME} + + - name: backend-ng + instanceType: ${EKS_BACKEND_NODE_INSTANCE_TYPE} + desiredCapacity: 1 + minSize: 1 + maxSize: 2 + volumeSize: 30 + privateNetworking: true + labels: + workload: backend + tags: + project: ${EKS_CLUSTER_NAME} diff --git a/gateway/perf/api-gateway-eks-perf/publish-results-pr.sh b/gateway/perf/api-gateway-eks-perf/publish-results-pr.sh new file mode 100755 index 000000000..caff3fb9a --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/publish-results-pr.sh @@ -0,0 +1,236 @@ +#!/bin/bash -e +# Format summary.csv into gateway/perf/README.md and open a GitHub PR. +# +# Required: +# SUMMARY_CSV path to summary-.csv +# GH_TOKEN or GITHUB_TOKEN PAT with repo + pull_request (or `gh auth login`) +# +# Optional: +# RESULTS_PR_REPO default https://github.com/wso2/api-platform.git +# RESULTS_PR_BASE default main +# RESULTS_PR_README default gateway/perf/README.md +# RESULTS_PR_BRANCH default perf-results- +# TEST_ID, GATEWAY_HELM_CHART_VERSION, GATEWAY_NODE_INSTANCE_TYPE, +# GATEWAY_RUNTIME_CPU_LIMIT, GATEWAY_RUNTIME_MEM_LIMIT, +# ROUTER_CONCURRENCY, GOMAXPROCS, GOGC, GOMEMLIMIT, RUN_PERF_OPTS +# +# Jenkins (after Step 15): +# export PUBLISH_RESULTS_PR=1 # or call this script explicitly +# SUMMARY_CSV=.../summary-${TEST_ID}.csv ./api-gateway-eks-perf/publish-results-pr.sh + +set -euo pipefail + +SUMMARY_CSV="${SUMMARY_CSV:?Set SUMMARY_CSV to the summary CSV path}" +[[ -f "${SUMMARY_CSV}" ]] || { echo "ERROR: SUMMARY_CSV not found: ${SUMMARY_CSV}" >&2; exit 1; } + +RESULTS_PR_REPO="${RESULTS_PR_REPO:-https://github.com/wso2/api-platform.git}" +RESULTS_PR_BASE="${RESULTS_PR_BASE:-main}" +RESULTS_PR_README="${RESULTS_PR_README:-gateway/perf/README.md}" +TEST_ID="${TEST_ID:-$(date +%Y%m%d-%H%M%S)}" +RESULTS_PR_BRANCH="${RESULTS_PR_BRANCH:-perf-results-${TEST_ID}}" + +export GH_TOKEN="${GH_TOKEN:-${GITHUB_TOKEN:-}}" +if [[ -z "${GH_TOKEN}" ]] && ! gh auth status >/dev/null 2>&1; then + echo "ERROR: set GH_TOKEN (or GITHUB_TOKEN), or run gh auth login." >&2 + exit 1 +fi +command -v gh >/dev/null || { echo "ERROR: gh CLI required on the Jenkins slave." >&2; exit 1; } +command -v python3 >/dev/null || { echo "ERROR: python3 required." >&2; exit 1; } + +RESULTS_PR_SLUG="$( + python3 - <&2 + exit 1 +} + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/perf-results-pr.XXXXXX")" +cleanup() { rm -rf "${WORK}"; } +trap cleanup EXIT + +echo "==> Sparse-clone ${RESULTS_PR_REPO}@${RESULTS_PR_BASE} ($(dirname "${RESULTS_PR_README}"))" +git clone --depth 1 --filter=tree:0 --sparse \ + --branch "${RESULTS_PR_BASE}" \ + "${RESULTS_PR_REPO}" "${WORK}/repo" +git -C "${WORK}/repo" sparse-checkout set --cone "$(dirname "${RESULTS_PR_README}")" + +README_PATH="${WORK}/repo/${RESULTS_PR_README}" +mkdir -p "$(dirname "${README_PATH}")" +[[ -f "${README_PATH}" ]] || printf '%s\n' '# API Platform Gateway Performance Results' >"${README_PATH}" + +META_FILE="${WORK}/meta.env" +{ + echo "TEST_ID=${TEST_ID}" + echo "DATE_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "GATEWAY_HELM_CHART_VERSION=${GATEWAY_HELM_CHART_VERSION:-}" + echo "GATEWAY_NODE_INSTANCE_TYPE=${GATEWAY_NODE_INSTANCE_TYPE:-}" + echo "GATEWAY_RUNTIME_CPU_LIMIT=${GATEWAY_RUNTIME_CPU_LIMIT:-}" + echo "GATEWAY_RUNTIME_MEM_LIMIT=${GATEWAY_RUNTIME_MEM_LIMIT:-}" + echo "ROUTER_CONCURRENCY=${ROUTER_CONCURRENCY:-}" + echo "GOMAXPROCS=${GOMAXPROCS:-}" + echo "GOGC=${GOGC:-}" + echo "GOMEMLIMIT=${GOMEMLIMIT:-}" + echo "RUN_PERF_OPTS=${RUN_PERF_OPTS:-}" +} >"${META_FILE}" + +echo "==> Format CSV → markdown and update ${RESULTS_PR_README}" +RESULTS_PR_REPO="${RESULTS_PR_REPO}" SUMMARY_CSV="${SUMMARY_CSV}" README_PATH="${README_PATH}" META_FILE="${META_FILE}" \ +python3 <<'PY' +import csv, os +from pathlib import Path + +csv_path = Path(os.environ["SUMMARY_CSV"]) +readme_path = Path(os.environ["README_PATH"]) +meta_path = Path(os.environ["META_FILE"]) +meta = {} +for line in meta_path.read_text().splitlines(): + if "=" in line: + k, _, v = line.partition("=") + meta[k] = v + +keep = [ + "Scenario Name", + "Concurrent Users", + "Throughput (Requests/sec)", + "Average Response Time (ms)", + "Error %", + "90th Percentile of Response Time (ms)", + "99th Percentile of Response Time (ms)", + "# Samples", +] +short = { + "Scenario Name": "Scenario", + "Concurrent Users": "Users", + "Throughput (Requests/sec)": "Throughput", + "Average Response Time (ms)": "Avg Response Time(ms)", + "Error %": "Err %", + "90th Percentile of Response Time (ms)": "p90 (ms)", + "99th Percentile of Response Time (ms)": "p99 (ms)", + "# Samples": "Samples", +} + +with csv_path.open(newline="") as f: + rows = list(csv.DictReader(f)) +if not rows: + raise SystemExit("summary CSV has no data rows") + +headers = [h for h in keep if h in rows[0]] or list(rows[0].keys())[:8] + +def cell(v: str) -> str: + return (v or "").strip().replace("|", "\\|") + +lines = [ + f"### Test ID `{meta.get('TEST_ID', '')}`", + "", + f"- **UTC:** `{meta.get('DATE_UTC', '')}`", +] +if meta.get("GATEWAY_HELM_CHART_VERSION"): + lines.append(f"- **Chart:** `{meta['GATEWAY_HELM_CHART_VERSION']}`") +if meta.get("GATEWAY_NODE_INSTANCE_TYPE"): + lines.append(f"- **Gateway node:** `{meta['GATEWAY_NODE_INSTANCE_TYPE']}`") +cpu, mem = meta.get("GATEWAY_RUNTIME_CPU_LIMIT", ""), meta.get("GATEWAY_RUNTIME_MEM_LIMIT", "") +if cpu or mem: + lines.append(f"- **Runtime resources:** CPU `{cpu or 'n/a'}` / mem `{mem or 'n/a'}`") +tune = [ + f"`{label}={meta[k]}`" + for k, label in ( + ("ROUTER_CONCURRENCY", "ROUTER_CONCURRENCY"), + ("GOMAXPROCS", "GOMAXPROCS"), + ("GOGC", "GOGC"), + ("GOMEMLIMIT", "GOMEMLIMIT"), + ) + if meta.get(k) +] +if tune: + lines.append("- **Tuning:** " + ", ".join(tune)) +if meta.get("RUN_PERF_OPTS"): + lines.append(f"- **RUN_PERF_OPTS:** `{meta['RUN_PERF_OPTS']}`") +lines += [ + "", + "| " + " | ".join(short.get(h, h) for h in headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", +] +for row in rows: + lines.append("| " + " | ".join(cell(row.get(h, "")) for h in headers) + " |") +lines.append("") +lines.append("---") +lines.append("") + +new_section = "\n".join(lines) +start, end = "", "" +text = readme_path.read_text() if readme_path.exists() else "# API Platform Gateway Performance Results\n" + +# Append (newest first): insert this run after START, keep prior runs below. +if start in text and end in text: + pre, rest = text.split(start, 1) + existing, post = rest.split(end, 1) + existing = existing.strip("\n") + # Skip if this TEST_ID was already published (idempotent re-run). + tid = meta.get("TEST_ID", "") + if tid and f"### Test ID `{tid}`" in existing: + print(f"Test ID {tid} already in README — leaving unchanged") + else: + combined = new_section + (existing + "\n" if existing else "") + text = pre.rstrip() + "\n\n" + start + "\n" + combined + end + "\n" + post.lstrip("\n") + readme_path.write_text(text) + print(f"Appended results for {tid} to {readme_path} ({len(rows)} row(s))") +else: + if not text.strip().startswith("#"): + text = "# API Platform Gateway Performance Results\n\n" + text + text = text.rstrip() + "\n\n" + start + "\n" + new_section + end + "\n" + readme_path.write_text(text) + print(f"Created results block in {readme_path} ({len(rows)} row(s))") +PY + +cd "${WORK}/repo" +git config user.name "${GIT_AUTHOR_NAME:-jenkins-perf}" +git config user.email "${GIT_AUTHOR_EMAIL:-jenkins-perf@users.noreply.github.com}" +git checkout -B "${RESULTS_PR_BRANCH}" +git add "${RESULTS_PR_README}" +if git diff --cached --quiet; then + echo "No README changes — nothing to publish." + exit 0 +fi + +git commit -m "$(cat < Push ${RESULTS_PR_BRANCH} → ${RESULTS_PR_SLUG}" +git push -u origin "HEAD:${RESULTS_PR_BRANCH}" + +echo "==> Open PR against ${RESULTS_PR_BASE}" +PR_URL="$( + gh pr list --repo "${RESULTS_PR_SLUG}" --head "${RESULTS_PR_BRANCH}" --json url -q '.[0].url' 2>/dev/null || true +)" +if [[ -z "${PR_URL}" ]]; then + PR_URL="$(gh pr create \ + --repo "${RESULTS_PR_SLUG}" \ + --base "${RESULTS_PR_BASE}" \ + --head "${RESULTS_PR_BRANCH}" \ + --title "(Automated PR): Performance test results for jenkins perf job ${TEST_ID}" \ + --body "$(cat <\` and \`\`. +EOF +)" )" +fi + +echo "PR: ${PR_URL}" +if [[ -n "${JENKINS_JOB_WORKSPACE:-}" ]]; then + echo "${PR_URL}" >"${JENKINS_JOB_WORKSPACE}/results-pr-url.txt" +fi diff --git a/gateway/perf/api-gateway-eks-perf/run-api-gateway-eks-tests.sh b/gateway/perf/api-gateway-eks-perf/run-api-gateway-eks-tests.sh new file mode 100755 index 000000000..a858fc9a6 --- /dev/null +++ b/gateway/perf/api-gateway-eks-perf/run-api-gateway-eks-tests.sh @@ -0,0 +1,626 @@ +#!/bin/bash -e +# Jenkins build script: api-gateway EKS performance test. +# +# === Required Jenkins build parameters (env vars) === +# BUILD_USER_EMAIL e.g. xyz@wso2.com +# AWS_REGION e.g. us-east-1 +# GATEWAY_NODE_INSTANCE_TYPE e.g. m5.2xlarge (EKS gateway pods) +# BACKEND_NODE_INSTANCE_TYPE e.g. m5.xlarge (EKS in-cluster backend) +# JMETER_CLIENT_EC2_INSTANCE_TYPE e.g. c5.2xlarge +# JMETER_SERVER_EC2_INSTANCE_TYPE e.g. c5.2xlarge +# GATEWAY_HELM_CHART_VERSION e.g. 1.2.0-rc +# RUN_PERF_OPTS load/scenario flags only, e.g.: +# "-u 1000 -b 1 -s 0 -d 900 -w 180 -i api_api_plain_get" +# Multiple: -u 100 -u 500 -u 1000 -i api_api_plain_get -i api_api_header_get +# Exclude: -e api_api_jwt_get +# Configurable: -u users, -b message bytes, -s backend sleep ms, -d duration, +# -w warmup, -i include scenario, -e exclude scenario. +# Infra heaps/engines (-n/-m/-j/-k/-l/-r) are set by this script, not RUN_PERF_OPTS. +# +# === Optional Jenkins env (script sources) === +# PERF_SCRIPTS repo@branch:subdir +# default: https://github.com/wso2/api-platform.git@main:gateway/perf/performance-test-scripts +# PERF_COMMON_REPO / PERF_COMMON_BRANCH +# default: performance-common fork used for jtl-splitter +# ROUTER_CONCURRENCY / GOMAXPROCS +# default: 4 / 4 (runtime pod concurrency; keep ≤ GATEWAY_RUNTIME_CPU_LIMIT) +# GOGC / GOMEMLIMIT +# default: 400 / 1500MiB +# PUBLISH_RESULTS_PR +# default: 0 (skip). Set to 1 to open a PR appending summary.csv into RESULTS_PR_README +# GH_TOKEN +# required only when PUBLISH_RESULTS_PR=1 (PAT with repo + PR access) +# RESULTS_PR_REPO / RESULTS_PR_BASE / RESULTS_PR_README +# defaults: wso2/api-platform @ main : gateway/perf/README.md +# +# === Pre-placed on Jenkins slave (one-time setup) === +# ~/keys/apim-perf-test3.pem EC2 SSH key (pem file) +# ~/apache-jmeter-5.6.3.tgz JMeter tarball +# eksctl, kubectl, helm, aws CLI, envsubst, python3, jq, mvn on PATH +# AWS key pair "apim-perf-test3" registered in the target AWS region +# Jenkins slave IAM role: EKS full + EC2 full + SSM read + ELB full +# (Optional) helm registry login ghcr.io — only if using private chart images +# +# === Topology === +# EKS cluster with 2 node groups: gateway-ng + backend-ng +# In-cluster Netty mock backend (perf-mock-backend ClusterIP service) +# Internal AWS NLB for gateway runtime (port 8080, same-VPC access) +# 1 JMeter client EC2 + 2 JMeter server EC2s in EKS VPC (public subnet) + +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# Save Jenkins job workspace (job subdir) before we override WORKSPACE. +JENKINS_JOB_WORKSPACE="${WORKSPACE:-}" +# Always compute from script location — do not use Jenkins $WORKSPACE (it points to job subdir). +WORKSPACE="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# ─── Validate required vars ─────────────────────────────────────────────────── +for v in BUILD_USER_EMAIL AWS_REGION \ + GATEWAY_NODE_INSTANCE_TYPE BACKEND_NODE_INSTANCE_TYPE \ + JMETER_CLIENT_EC2_INSTANCE_TYPE JMETER_SERVER_EC2_INSTANCE_TYPE \ + GATEWAY_HELM_CHART_VERSION RUN_PERF_OPTS; do + [[ -n "${!v:-}" ]] || { echo "ERROR: env var ${v} is not set." >&2; exit 1; } +done + +# Infra flags below are appended by Step 13. run-scenario.sh treats -n/-m/-j/-k/-l/-r as +# repeatable matrix dimensions, so leaving one in RUN_PERF_OPTS silently doubles every +# scenario (e.g. "-r 1" here plus the injected "-r 1" = two identical passes). +sanitize_run_perf_opts() { + local -a toks kept=() + local i=0 + read -ra toks <<<"$1" + while ((i < ${#toks[@]})); do + case "${toks[i]}" in + -n | -m | -j | -k | -l | -r) + echo "WARNING: dropping '${toks[i]} ${toks[i + 1]:-}' from RUN_PERF_OPTS — set by this script." >&2 + i=$((i + 2)) + continue + ;; + esac + kept+=("${toks[i]}") + i=$((i + 1)) + done + echo "${kept[*]}" +} +RUN_PERF_OPTS="$(sanitize_run_perf_opts "${RUN_PERF_OPTS}")" + +# ─── Test identity ───────────────────────────────────────────────────────────── +BUILD_NUMBER="${BUILD_NUMBER:-0}" +TEST_ID="${BUILD_NUMBER}-$(date +%Y%m%d-%H%M%S)" +EKS_CLUSTER_NAME="apigw-jenkins-${TEST_ID}" +export CURRENT_DIR="${CURRENT_DIR:-$(realpath .)}" +export RESULTS_DIR="${RESULTS_DIR:-$(realpath "results-${TEST_ID}")}" +mkdir -p "${RESULTS_DIR}" + +# ─── Constants (override via env if needed) ──────────────────────────────────── +JMETER_KEY="${JMETER_KEY:-${HOME}/keys/apim-perf-test3.pem}" +JMETER_KEY_NAME="${JMETER_KEY_NAME:-apim-perf-test3}" +JMETER_TGZ="${JMETER_TGZ:-${HOME}/apache-jmeter-5.6.3.tgz}" +JMETER_USER="${JMETER_USER:-ec2-user}" +GATEWAY_HELM_CHART="${GATEWAY_HELM_CHART:-oci://ghcr.io/wso2/api-platform/helm-charts/gateway}" +EKS_NAMESPACE="${EKS_NAMESPACE:-api-gateway}" +EKS_RELEASE_NAME="${EKS_RELEASE_NAME:-ap-gateway}" +EKS_K8S_VERSION="${EKS_K8S_VERSION:-1.32}" +GATEWAY_RUNTIME_REPLICAS="${GATEWAY_RUNTIME_REPLICAS:-1}" +BACKEND_PORT=8688 +EKS_STORAGE_CLASS="gp3-csi-auto" +GATEWAY_CONTROLLER_PORT="19090" +GATEWAY_MGMT_API_BASE="/api/management/v1" +JMETER_SERVERS_COUNT=2 +# Results-path / JMeter infra defaults (override via env if needed; not part of RUN_PERF_OPTS). +PERF_HEAP_LABEL="${PERF_HEAP_LABEL:-16G}" +JMETER_SERVER_HEAP="${JMETER_SERVER_HEAP:-4G}" +JMETER_CLIENT_HEAP="${JMETER_CLIENT_HEAP:-2G}" +NETTY_SERVICE_HEAP="${NETTY_SERVICE_HEAP:-4G}" +RESPONSE_SIZE_BYTES="${RESPONSE_SIZE_BYTES:-1}" + +PERF_COMMON_REPO="${PERF_COMMON_REPO:-https://github.com/Milanka00/performance-common.git}" +PERF_COMMON_BRANCH="${PERF_COMMON_BRANCH:-470-ai-api-perf}" +# API Gateway EKS JMeter scripts: single Jenkins param PERF_SCRIPTS=repo@branch:subdir +# (SSH remotes like git@host:org/repo.git@branch:subdir are supported — last @ / last : win.) +# Accept PERF_SCRIPT (singular) as an alias — common Jenkins naming slip. +# Defaults to the fork until gateway/perf/performance-test-scripts is merged into wso2/api-platform. +PERF_SCRIPTS="${PERF_SCRIPTS:-${PERF_SCRIPT:-https://github.com/wso2/api-platform.git@main:gateway/perf/performance-test-scripts}}" +PERF_SCRIPTS_SUBDIR="${PERF_SCRIPTS##*:}" +_perf_scripts_rest="${PERF_SCRIPTS%:*}" +PERF_SCRIPTS_BRANCH="${_perf_scripts_rest##*@}" +PERF_SCRIPTS_REPO="${_perf_scripts_rest%@*}" +unset _perf_scripts_rest +if [[ -z "${PERF_SCRIPTS_REPO}" || -z "${PERF_SCRIPTS_BRANCH}" || -z "${PERF_SCRIPTS_SUBDIR}" \ + || "${PERF_SCRIPTS_REPO}" == "${PERF_SCRIPTS}" ]]; then + echo "ERROR: PERF_SCRIPTS must be repo@branch:subdir (got: ${PERF_SCRIPTS})" >&2 + exit 1 +fi +# Stable local name after clone (rsync'd to JMeter EC2s under this path). +MANUAL_DIR_NAME="${MANUAL_DIR_NAME:-performance-test-scripts}" +MANUAL_DIR="${WORKSPACE}/${MANUAL_DIR_NAME}" +STATE_FILE="${CURRENT_DIR}/jmeter-ec2-state-${TEST_ID}.env" + +# SSH helper used for all JMeter EC2 connections. +SSH_KEY="${JMETER_KEY}" +ssh_cmd() { ssh -o StrictHostKeyChecking=no -o ConnectTimeout=20 -i "${SSH_KEY}" "$@"; } +scp_cmd() { scp -o StrictHostKeyChecking=no -o ConnectTimeout=20 -i "${SSH_KEY}" "$@"; } +rsync_cmd() { + rsync -az -e "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=20 -i ${SSH_KEY}" "$@" +} + +export EKS_CLUSTER_NAME AWS_REGION STATE_FILE EKS_DIR + +echo "===================================================================" +echo " api-gateway EKS performance test" +echo " TEST_ID: ${TEST_ID}" +echo " CLUSTER: ${EKS_CLUSTER_NAME}" +echo " REGION: ${AWS_REGION}" +echo " CHART: ${GATEWAY_HELM_CHART}:${GATEWAY_HELM_CHART_VERSION}" +echo " SCRIPTS: ${PERF_SCRIPTS}" +echo " RUN_OPTS: ${RUN_PERF_OPTS}" +echo "===================================================================" + +# ─── EXIT trap ──────────────────────────────────────────────────────────────── +cleanup_and_archive() { + local rv=$? + echo "" + echo "==> Exit handler (rv=${rv})" + "${SCRIPT_DIR}/cleanup.sh" || true + local archive="${CURRENT_DIR}/archive" + if [[ $rv -eq 0 ]]; then + mkdir -p "${archive}/successful" + [[ -d "${RESULTS_DIR}" ]] && mv -v "${RESULTS_DIR}" "${archive}/successful/" || true + # Keep only the last 5 successful result sets. + ls -1dt "${archive}/successful"/results-* 2>/dev/null | tail -n +6 | xargs rm -rf || true + else + mkdir -p "${archive}/failed" + [[ -d "${RESULTS_DIR}" ]] && mv -v "${RESULTS_DIR}" "${archive}/failed/" || true + # Keep only the last 3 failed result sets. + ls -1dt "${archive}/failed"/results-* 2>/dev/null | tail -n +4 | xargs rm -rf || true + fi +} +trap cleanup_and_archive EXIT + +# ─── Step 1: clone deps (performance-common + gateway/perf scripts) ─────────── +echo "" +echo "==> Step 1a: clone performance-common and build jtl-splitter" +PERF_COMMON_DIR="${WORKSPACE}/performance-common" +if [[ -d "${PERF_COMMON_DIR}/.git" ]]; then + echo " Already present — fetching ${PERF_COMMON_BRANCH}" + git -C "${PERF_COMMON_DIR}" fetch --depth 1 origin "${PERF_COMMON_BRANCH}" 2>&1 | tail -3 || true + git -C "${PERF_COMMON_DIR}" checkout -q FETCH_HEAD 2>/dev/null || \ + git -C "${PERF_COMMON_DIR}" checkout -q "${PERF_COMMON_BRANCH}" 2>/dev/null || true +else + rm -rf "${PERF_COMMON_DIR}" + git clone --depth 1 --branch "${PERF_COMMON_BRANCH}" "${PERF_COMMON_REPO}" "${PERF_COMMON_DIR}" +fi + +echo " Building jtl-splitter JAR (needed for generate-summary.sh)" +mvn -q -f "${PERF_COMMON_DIR}/components/jtl-splitter" package -DskipTests 2>&1 | tail -5 + +echo "" +echo "==> Step 1b: sparse-clone perf scripts only (${PERF_SCRIPTS})" +# Do NOT download the whole api-platform tree: depth-1 + tree:0 + cone sparse-checkout +# fetches only objects needed for PERF_SCRIPTS_SUBDIR (tiny compared to full clone). +PERF_SCRIPTS_CLONE="${WORKSPACE}/.perf-scripts-src" +rm -rf "${PERF_SCRIPTS_CLONE}" +git clone --depth 1 --filter=tree:0 --sparse \ + --branch "${PERF_SCRIPTS_BRANCH}" \ + "${PERF_SCRIPTS_REPO}" "${PERF_SCRIPTS_CLONE}" +git -C "${PERF_SCRIPTS_CLONE}" sparse-checkout set --cone "${PERF_SCRIPTS_SUBDIR}" + +SRC_SCRIPTS="${PERF_SCRIPTS_CLONE}/${PERF_SCRIPTS_SUBDIR}" +[[ -d "${SRC_SCRIPTS}/api-gateway/eks" && -d "${SRC_SCRIPTS}/jmeter" ]] || { + echo "ERROR: ${SRC_SCRIPTS} missing api-gateway/eks or jmeter after sparse clone." >&2 + echo " Check PERF_SCRIPTS=${PERF_SCRIPTS}" >&2 + echo " ${PERF_SCRIPTS_SUBDIR} does not exist on ${PERF_SCRIPTS_REPO}@${PERF_SCRIPTS_BRANCH}." >&2 + echo " Checked out:" >&2 + (cd "${PERF_SCRIPTS_CLONE}" && find . -maxdepth 3 -type d -not -path './.git*' | head -20) >&2 + exit 1 +} +# Flatten into a stable MANUAL_DIR name for remote EC2 paths. +rm -rf "${MANUAL_DIR}" +mkdir -p "${MANUAL_DIR}" +rsync -a --exclude='.git' "${SRC_SCRIPTS}/" "${MANUAL_DIR}/" +EKS_DIR="${MANUAL_DIR}/api-gateway/eks" +export EKS_DIR +echo " MANUAL_DIR=${MANUAL_DIR}" + +export PERF_ROOT="${WORKSPACE}" + +# ─── Step 2: generate env.eks for this run ──────────────────────────────────── +echo "" +echo "==> Step 2: generate env.eks" + +EKS_GATEWAY_NODE_DESIRED="${GATEWAY_RUNTIME_REPLICAS}" +BACKEND_NAMESPACE="${EKS_NAMESPACE}" +MOCK_BACKEND_URL="http://perf-mock-backend.${BACKEND_NAMESPACE}.svc.cluster.local:${BACKEND_PORT}/v1" +PERF_CONFIG_TOML="${MANUAL_DIR}/api-gateway/config.perf-overlay.toml" + +cat >"${EKS_DIR}/env.eks" < Step 3: create EKS cluster ${EKS_CLUSTER_NAME} (15-25 min)" + +CLUSTER_CFG="${SCRIPT_DIR}/.eks-cluster-perf.generated.yaml" +export EKS_GATEWAY_NODE_DESIRED BACKEND_NODE_INSTANCE_TYPE EKS_K8S_VERSION +export EKS_NODE_INSTANCE_TYPE="${GATEWAY_NODE_INSTANCE_TYPE}" +export EKS_BACKEND_NODE_INSTANCE_TYPE="${BACKEND_NODE_INSTANCE_TYPE}" +envsubst <"${SCRIPT_DIR}/eks-cluster-perf.yaml.template" >"${CLUSTER_CFG}" + +eksctl create cluster -f "${CLUSTER_CFG}" +aws eks update-kubeconfig --name "${EKS_CLUSTER_NAME}" --region "${AWS_REGION}" +kubectl get nodes -o wide +echo " Waiting 30s for node daemonsets to initialize..." +sleep 30 + +# ─── Step 4: install gp3 storage class ──────────────────────────────────────── +echo "" +echo "==> Step 4: install gp3-csi-auto storage class" +kubectl apply -f - <<'STORAGECLASS' +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3-csi-auto +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + type: gp3 +reclaimPolicy: Delete +allowVolumeExpansion: true +STORAGECLASS + +# ─── Step 5: deploy in-cluster mock backend ─────────────────────────────────── +echo "" +echo "==> Step 5: deploy in-cluster mock backend" +kubectl create namespace "${EKS_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f "${EKS_DIR}/backend-mock-eks.yaml" +if ! kubectl rollout status deployment/perf-mock-backend -n "${EKS_NAMESPACE}" --timeout=300s; then + echo "ERROR: perf-mock-backend did not become ready. Diagnostics:" + kubectl get nodes -o wide --show-labels || true + kubectl get pods -n "${EKS_NAMESPACE}" -o wide || true + kubectl describe pods -n "${EKS_NAMESPACE}" -l app=perf-mock-backend || true + kubectl get events -n "${EKS_NAMESPACE}" --sort-by='.lastTimestamp' | tail -30 || true + exit 1 +fi +kubectl get pods -n "${EKS_NAMESPACE}" -l app=perf-mock-backend -o wide + +# ─── Step 6: install gateway via Helm ───────────────────────────────────────── +echo "" +echo "==> Step 6: install gateway (Helm ${GATEWAY_HELM_CHART_VERSION})" +(cd "${EKS_DIR}" && source ./env.eks && ./install-gateway.sh) + +# ─── Step 7: deploy minimal APIs ────────────────────────────────────────────── +echo "" +echo "==> Step 7: deploy minimal RestApis (api-plain, api-header, api-jwt)" +(cd "${EKS_DIR}" && source ./env.eks && ./deploy-apis-eks-minimal.sh) + +# ─── Step 8: get NLB hostname ───────────────────────────────────────────────── +echo "" +echo "==> Step 8: wait for gateway NLB hostname" +GATEWAY_HOST="" +for attempt in $(seq 1 36); do + GATEWAY_HOST=$(kubectl get svc "${EKS_RELEASE_NAME}-gateway-runtime" \ + -n "${EKS_NAMESPACE}" \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + [[ -n "${GATEWAY_HOST}" && "${GATEWAY_HOST}" != "None" ]] && break + echo " Attempt ${attempt}/36: NLB pending, waiting 15s..." + sleep 15 +done +[[ -n "${GATEWAY_HOST}" ]] || { echo "ERROR: gateway NLB hostname not available after 9 min." >&2; exit 1; } +echo " GATEWAY_HOST=${GATEWAY_HOST}" + +# ─── Step 9: create JMeter EC2s ─────────────────────────────────────────────── +echo "" +echo "==> Step 9: create JMeter EC2s in EKS VPC" +export JMETER_SG_TAG="jmeter-${TEST_ID}" +export JMETER_CLIENT_EC2_INSTANCE_TYPE JMETER_SERVER_EC2_INSTANCE_TYPE JMETER_KEY_NAME +"${SCRIPT_DIR}/create-jmeter-ec2s.sh" +# shellcheck source=/dev/null +source "${STATE_FILE}" + +echo "==> Waiting 90s for EC2 cloud-init and SSH to be available..." +sleep 90 + +wait_for_ssh() { + local host="$1" + for attempt in $(seq 1 15); do + ssh_cmd "${JMETER_USER}@${host}" "echo ssh-ok" 2>/dev/null && return 0 + echo " SSH to ${host}: attempt ${attempt}/15, retrying..." + sleep 15 + done + echo "ERROR: SSH to ${host} timed out." >&2; return 1 +} + +wait_for_ssh "${CLIENT_PUBLIC_IP}" +wait_for_ssh "${SERVER1_PUBLIC_IP}" +wait_for_ssh "${SERVER2_PUBLIC_IP}" + +# ─── Step 10: sync perf repo and setup JMeter on all 3 EC2s ────────────────── +echo "" +echo "==> Step 10: rsync perf repo + JMeter tarball to EC2s" + +REMOTE_PERF_ROOT="~/perf-scripts" +REMOTE_PERF_HOME="~/perf-api-gateway-manual" + +for host in "${CLIENT_PUBLIC_IP}" "${SERVER1_PUBLIC_IP}" "${SERVER2_PUBLIC_IP}"; do + echo " rsync → ${JMETER_USER}@${host}" + rsync_cmd \ + --exclude='.git' --exclude='*.log' --exclude='results-*' --exclude='archive' \ + "${WORKSPACE}/" "${JMETER_USER}@${host}:${REMOTE_PERF_ROOT}/" + echo " scp JMeter tarball → ${host}" + scp_cmd "${JMETER_TGZ}" "${JMETER_USER}@${host}:~/" +done + +echo "" +echo "==> Step 10b: run setup-jmeter.sh on all EC2s" +for host in "${CLIENT_PUBLIC_IP}" "${SERVER1_PUBLIC_IP}" "${SERVER2_PUBLIC_IP}"; do + echo " setup-jmeter on ${JMETER_USER}@${host}" + ssh_cmd "${JMETER_USER}@${host}" \ + "PERF_ROOT=${REMOTE_PERF_ROOT} PERF_HOME=${REMOTE_PERF_HOME} \ + JMETER_TGZ=~/$(basename "${JMETER_TGZ}") \ + ${REMOTE_PERF_ROOT}/${MANUAL_DIR_NAME}/jmeter/setup-jmeter.sh" +done + +# ─── Step 11: start JMeter servers ──────────────────────────────────────────── +# jmeter-server-start.sh requires -n and -i . +# RMI hostname MUST be the private IP (same as -R list used by the client). +echo "" +echo "==> Step 11: start JMeter server processes on server EC2s" +declare -a _jmeter_server_public=("${SERVER1_PUBLIC_IP}" "${SERVER2_PUBLIC_IP}") +declare -a _jmeter_server_private=("${SERVER1_PRIVATE_IP}" "${SERVER2_PRIVATE_IP}") +for ix in "${!_jmeter_server_public[@]}"; do + host="${_jmeter_server_public[$ix]}" + private_ip="${_jmeter_server_private[$ix]}" + echo " Starting JMeter server on ${host} (RMI hostname=${private_ip})" + # Do not wrap in outer nohup/& — the start script already nohups jmeter-server + # and exits non-zero if ApacheJMeter.jar does not come up. + ssh_cmd "${JMETER_USER}@${host}" \ + "${REMOTE_PERF_HOME}/jmeter/jmeter-server-start.sh -n ${private_ip} -i \$HOME -m ${JMETER_SERVER_HEAP}" \ + | tee "${RESULTS_DIR}/jmeter-server-${private_ip}.log" || { + echo "ERROR: JMeter server failed to start on ${host}." >&2 + ssh_cmd "${JMETER_USER}@${host}" \ + "tail -80 ~/server.out 2>/dev/null; tail -40 ~/jmeter-server.log 2>/dev/null; pgrep -af ApacheJMeter || true" || true + exit 1 + } +done + +# Check JMeter RMI port from the client EC2 (not slave — port 1099 is only open within the JMeter SG). +echo " Waiting for JMeter RMI port (1099) on server EC2s (checked from client)..." +for private_ip in "${SERVER1_PRIVATE_IP}" "${SERVER2_PRIVATE_IP}"; do + ready=0 + for attempt in $(seq 1 24); do + if ssh_cmd "${JMETER_USER}@${CLIENT_PUBLIC_IP}" \ + "nc -z -w3 ${private_ip} 1099 2>/dev/null"; then + echo " ${private_ip}:1099 open" + ready=1 + break + fi + echo " ${private_ip}:1099 not ready, attempt ${attempt}/24, waiting 5s..." + sleep 5 + done + if [[ "${ready}" -ne 1 ]]; then + echo "ERROR: ${private_ip}:1099 never opened after ~2 min." >&2 + exit 1 + fi +done + +# ─── Step 12: configure JMeter client (SSH aliases + env.jmeter) ────────────── +echo "" +echo "==> Step 12: configure JMeter client distributed SSH and env" + +# Copy EC2 key to client for server-to-server SSH (needed by JMeter RMI setup). +scp_cmd "${SSH_KEY}" "${JMETER_USER}@${CLIENT_PUBLIC_IP}:~/.ssh/jmeter-servers.pem" +ssh_cmd "${JMETER_USER}@${CLIENT_PUBLIC_IP}" "chmod 600 ~/.ssh/jmeter-servers.pem" + +# Write ~/.ssh/config on client with jmeter1/jmeter2 aliases (private IPs). +ssh_cmd "${JMETER_USER}@${CLIENT_PUBLIC_IP}" bash <>~/.ssh/config <<'CFGENTRY' + +Host jmeter1 + HostName ${SERVER1_PRIVATE_IP} + User ${JMETER_USER} + IdentityFile ~/.ssh/jmeter-servers.pem + StrictHostKeyChecking no + +Host jmeter2 + HostName ${SERVER2_PRIVATE_IP} + User ${JMETER_USER} + IdentityFile ~/.ssh/jmeter-servers.pem + StrictHostKeyChecking no +CFGENTRY +chmod 600 ~/.ssh/config +SSHCONFIG + +# Generate env.jmeter locally and scp to client (avoids remote heredoc quoting issues). +# IMPORTANT: also write env.jmeter.api — run-scenario.sh sources that profile AFTER +# env.jmeter and would otherwise fall back to env.jmeter.api.example (hardcoded EC2 IPs). +ENV_JMETER_TMP=$(mktemp) +cat >"${ENV_JMETER_TMP}" < Step 13: run perf scenarios (RUN_PERF_OPTS: ${RUN_PERF_OPTS})" +echo " Infra defaults: -n ${JMETER_SERVERS_COUNT} -m ${PERF_HEAP_LABEL} -j ${JMETER_SERVER_HEAP} -k ${JMETER_CLIENT_HEAP} -l ${NETTY_SERVICE_HEAP} -r ${RESPONSE_SIZE_BYTES}" + +# Write run script on client to avoid shell-quoting issues with RUN_PERF_OPTS. +# RUN_PERF_OPTS should only carry -u/-b/-s/-d/-w/-i/-e (and repeats of those). +ssh_cmd "${JMETER_USER}@${CLIENT_PUBLIC_IP}" bash <&1 | tee /tmp/perf-run.log +RUNSCRIPT + +# ─── Step 14: generate summary on JMeter client ─────────────────────────────── +echo "" +echo "==> Step 14: generate summary CSV on JMeter client" +ssh_cmd "${JMETER_USER}@${CLIENT_PUBLIC_IP}" bash < Step 15: download results to ${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}/jmeter-results" + +rsync_cmd \ + "${JMETER_USER}@${CLIENT_PUBLIC_IP}:${REMOTE_PERF_HOME}/jmeter/results/" \ + "${RESULTS_DIR}/jmeter-results/" 2>/dev/null || true + +rsync_cmd \ + "${JMETER_USER}@${CLIENT_PUBLIC_IP}:${REMOTE_PERF_HOME}/jmeter/summary.csv" \ + "${RESULTS_DIR}/" 2>/dev/null || true + +rsync_cmd \ + "${JMETER_USER}@${CLIENT_PUBLIC_IP}:/tmp/perf-run.log" \ + "${RESULTS_DIR}/" 2>/dev/null || true + +if [[ -f "${RESULTS_DIR}/summary.csv" ]]; then + echo "" + echo "===================================================================" + echo " RESULTS SUMMARY" + echo "===================================================================" + column -t -s ',' "${RESULTS_DIR}/summary.csv" + echo "===================================================================" + # Copy to Jenkins job workspace so "Archive the artifacts" can expose it. + if [[ -n "${JENKINS_JOB_WORKSPACE}" ]]; then + cp "${RESULTS_DIR}/summary.csv" \ + "${JENKINS_JOB_WORKSPACE}/summary-${TEST_ID}.csv" + echo " Archived to Jenkins workspace: summary-${TEST_ID}.csv" + fi +fi + +echo "" +echo "===================================================================" +echo " Performance test complete." +echo " Summary: ${RESULTS_DIR}/summary.csv" +echo " Results: ${RESULTS_DIR}/jmeter-results/" +echo "===================================================================" + +# ─── Optional: publish formatted results PR ─────────────────────────────────── +# Set PUBLISH_RESULTS_PR=1 (and GH_TOKEN) on the Jenkins job to open a PR that +# updates RESULTS_PR_README (default gateway/perf/README.md) in RESULTS_PR_REPO. +if [[ "${PUBLISH_RESULTS_PR:-0}" == "1" ]]; then + echo "" + echo "==> Step 16: publish results PR" + SUMMARY_CSV="${JENKINS_JOB_WORKSPACE}/summary-${TEST_ID}.csv" + [[ -f "${SUMMARY_CSV}" ]] || SUMMARY_CSV="${RESULTS_DIR}/summary.csv" + SUMMARY_CSV="${SUMMARY_CSV}" \ + TEST_ID="${TEST_ID}" \ + JENKINS_JOB_WORKSPACE="${JENKINS_JOB_WORKSPACE}" \ + "${SCRIPT_DIR}/publish-results-pr.sh" || { + echo "WARNING: publish-results-pr.sh failed (perf run itself succeeded)." >&2 + } +fi + +# EXIT trap runs cleanup_and_archive → cleanup.sh diff --git a/gateway/perf/performance-test-scripts/README.md b/gateway/perf/performance-test-scripts/README.md new file mode 100644 index 000000000..6b75e4aa1 --- /dev/null +++ b/gateway/perf/performance-test-scripts/README.md @@ -0,0 +1,178 @@ +# Performance-test-scripts — API Gateway EKS performance scripts + +Minimal scripts for **Jenkins API Gateway EKS** performance tests. + + +Supports the following RestApis and scenarios: + +| RestApi (deploy) | Policy | JMeter scenario (`-i`) | Path (8 routes: base + `/r1`–`/r7`) | +|------------------|--------|------------------------|------| +| `perf-api-plain` | none (plain) | `api_api_plain_get` | `/api-plain/1.0.0/chat/completions` | +| `perf-api-header` | set-headers | `api_api_header_get` | `/api-header/1.0.0/chat/completions` | +| `perf-api-jwt` | jwt-auth | `api_api_jwt_get` | `/api-jwt/1.0.0/chat/completions` | + +Each RestApi is deployed with `-r 8` (16 operations: GET+POST × 8 paths). JMeter randomizes among the same 8 path suffixes via `resourceSuffixes=,/r1,...,/r7`. + +--- + +## How Jenkins gets these scripts + +`api-gateway-eks-perf/run-api-gateway-eks-tests.sh` sparse-clones this tree via one Jenkins param: + +| Env | Format / default | +|-----|------------------| +| `PERF_SCRIPTS` | `repo@branch:subdir` → `https://github.com/wso2/api-platform.git@main:gateway/perf/performance-test-scripts` | + +After clone it copies into workspace `performance-test-scripts/`. Fork testing: + +```bash +export PERF_SCRIPTS=https://github.com//api-platform.git@my-perf-branch:gateway/perf/performance-test-scripts +``` + +--- + +## Workspace layout (Jenkins slave after job start) + +``` +product-performance-test/ # PERF_ROOT +├── api-gateway-eks-perf/ # orchestrator (on slave) +├── performance-test-scripts/ # cloned from gateway/perf/performance-test-scripts +├── performance-common/ # cloned for jtl-splitter +└── .perf-scripts-src/ # sparse clone cache (ephemeral) +``` + +--- + +## Jenkins job parameters + +| Parameter | Example | Notes | +|-----------|---------|-------| +| `RUN_PERF_OPTS` | `-u 1000 -b 1 -s 0 -d 900 -w 180 -i api_api_plain_get` | **Load/scenario only** (see below) | +| `GATEWAY_HELM_CHART_VERSION` | `1.2.0-rc` | Pin in Jenkins; see Helm upgrades below | +| `GATEWAY_NODE_INSTANCE_TYPE` | `c5.2xlarge` | One node ≈ one 4-CPU runtime pod | +| `ROUTER_CONCURRENCY` | `4` | Runtime Envoy/router concurrency (default 4) | +| `GOMAXPROCS` | `4` | Go max procs (default 4; keep ≤ CPU limit) | +| `JWT_OAUTH_*` | Asgardeo client creds | Required for JWT token mint; `JWT_OAUTH_TOKEN_URL` also sets the gateway's JWT issuer | + +### `RUN_PERF_OPTS` (configure these) + +| Flag | Meaning | Multiple? | +|------|---------|-----------| +| `-u` | Concurrent users | yes (`-u 100 -u 500 -u 1000`) | +| `-b` | Message size bytes | yes | +| `-s` | Backend sleep ms | yes | +| `-d` | Test duration seconds (default 900) | no | +| `-w` | Warm-up seconds (default 300) | no | +| `-i` | Include scenario | yes (`-i api_api_plain_get -i api_api_header_get`) | +| `-e` | Exclude scenario | yes | + +Scenarios: `api_api_plain_get`, `api_api_header_get`, `api_api_jwt_get`. + +Example multi-user / multi-scenario: + +```text +-u 500 -u 1000 -b 1 -s 0 -d 900 -w 180 -i api_api_plain_get -i api_api_header_get +``` + +--- + +## Gateway runtime tuning (where to change) + +Jenkins writes `performance-test-scripts/api-gateway/eks/env.eks` each run from +`api-gateway-eks-perf/run-api-gateway-eks-tests.sh` (Step 2). Edit that block for job defaults: + +```bash +export GATEWAY_RUNTIME_REPLICAS="${GATEWAY_RUNTIME_REPLICAS:-1}" +export GATEWAY_RUNTIME_CPU_LIMIT="4" +export GATEWAY_RUNTIME_MEM_LIMIT="2Gi" +export ROUTER_CONCURRENCY="${ROUTER_CONCURRENCY:-4}" +export GOMAXPROCS="${GOMAXPROCS:-4}" +export GOGC="${GOGC:-400}" +export GOMEMLIMIT="${GOMEMLIMIT:-1500MiB}" +export LOG_LEVEL="error" +export POLICY_ENGINE_METRICS_ENABLED="false" +``` + +Flow: `env.eks` → `install-gateway.sh` → `eks-common.sh` → `.generated-values.yaml` → Helm → live Deployment. + +Manual EKS (outside Jenkins): edit `api-gateway/eks/env.eks` from `env.eks.example`, then `source env.eks && ./install-gateway.sh`. + +Do **not** edit `.generated-values.yaml` by hand — regenerated each install. + +--- + +## Adding a new RestApi + JMeter scenario + +Example: add `api-ratelimit` with basic-ratelimit policy. + +### 1. Deploy script — `api-gateway/deploy/create-rest-perf-api.sh` + +- Add a new `-m` mode if the policy combination is new (or reuse `plain` / `add_headers` / `jwt_auth`). +- Implement policy YAML in the `case "$api_mode"` block. + +### 2. EKS deploy — `api-gateway/eks/deploy-apis-eks-minimal.sh` + +- Add the metadata name to `keep_apis=(...)`. +- Add a deploy line, e.g. `"${DEPLOY_DIR}/create-rest-perf-api.sh" -n api-ratelimit -m plain -r 8 -l`. +- Keep `-r` in sync with `PERF_API_ROUTE_COUNT` / `_perf_api_route_suffixes` in `gateway-scenarios.sh` (default **8** = base + `/r1`–`/r7`). + +### 3. JMeter scenario — `jmeter/gateway-scenarios.sh` + +- Add `test_scenarioN` with matching `[path]`, `[jmx]`, and register in `test_scenario` map. +- If auth tokens needed, add entries to `scenario_api_key_file` / `scenario_auth_header`. + +### 4. JMX (if needed) + +- Reuse `api-api-test-gateway-plain.jmx` for GET without body auth quirks. +- Use `api-api-test-gateway-jwt-plain.jmx` when `Authorization: Bearer` header is required. + +### 5. Config overlay — `api-gateway/config.perf-overlay.toml` + +- Add policy system config (keymanager, ratelimit backend, etc.) if the new policy needs it. +- JWT keymanager name must match `JWT_KEYMANAGER_NAME` in `env.eks` (default `test`). + +### 6. Jenkins + +- Document new `-i ` in the job parameter help. +- No orchestrator change unless deploy script name or paths change. + +--- + +## Upgrading Helm / gateway chart version + +When moving to a new chart (e.g. `1.2.0-rc` → `1.3.0`): + +1. **Jenkins parameter** — set `GATEWAY_HELM_CHART_VERSION`. +2. **`env.eks.example`** — update default chart version comment if you maintain it. +3. **`run-api-gateway-eks-tests.sh`** — verify `GATEWAY_MGMT_API_BASE` (v1 vs v0.9) still correct. +4. **`eks-common.sh`** — run a test install; check for chart value renames in upstream `values.yaml`. +5. **`values.perf.yaml`** — merge any new required Helm keys from the new chart defaults. +6. **`config.perf-overlay.toml`** — confirm overlay keys still valid (some keys are chart/version-specific; comments in file note unsupported sections). +7. **Controller port** — `GATEWAY_CONTROLLER_PORT` (default `19090`) if the new chart changes management port. +8. **Images** — official chart pulls `ghcr.io/wso2/api-platform/gateway-*:`; override with `GATEWAY_RUNTIME_IMAGE` only for custom builds. +9. **Re-run** `deploy-apis-eks-minimal.sh` after upgrade — RestApi CRD/API shape may change between versions. + +After upgrade, smoke test from slave: + +```bash +curl -sf "http://${NLB_HOST}:8080/api-plain/1.0.0/chat/completions" -o /dev/null -w '%{http_code}\n' +``` + +--- + +## File map + +| Path | Role | +|------|------| +| `api-gateway/eks/install-gateway.sh` | Helm install/upgrade | +| `api-gateway/eks/eks-common.sh` | Generates Helm values from `env.eks` | +| `api-gateway/eks/deploy-apis-eks-minimal.sh` | Deploy 3 RestApis, delete others | +| `api-gateway/eks/backend-mock-eks.yaml` | In-cluster Netty mock | +| `api-gateway/eks/values.perf.yaml` | Static Helm overrides | +| `api-gateway/config.perf-overlay.toml` | Embedded gateway config (access logs off, JWT, ratelimit) | +| `api-gateway/deploy/create-rest-perf-api.sh` | RestApi YAML builder (plain / header / jwt) | +| `jmeter/run-scenario.sh` | Distributed test driver | +| `jmeter/gateway-scenarios.sh` | Scenario definitions (3 only) | +| `jmeter/generate-jwt-tokens.sh` | Asgardeo OAuth → `jwt-tokens.csv` | +| `jmeter/*.jmx` | JMeter test plans | + diff --git a/gateway/perf/performance-test-scripts/api-gateway/config.perf-overlay.toml b/gateway/perf/performance-test-scripts/api-gateway/config.perf-overlay.toml new file mode 100644 index 000000000..f2f2b40c0 --- /dev/null +++ b/gateway/perf/performance-test-scripts/api-gateway/config.perf-overlay.toml @@ -0,0 +1,108 @@ + +# analytics.enabled must be true for the controller to attach Envoy → PE ALS (gRPC +# access log). Without ALS, [traffic_logging] never receives events. +# Leave enabled_publishers empty so Moesif does not need an application_id. +[analytics] +enabled = false +enabled_publishers = [] + +[router] +gateway_host = "*" + +[router.access_logs] +enabled = false + +[controller.server] +gateway_id = "platform-gateway-id" + +[policy_engine.logging] +level = "error" + +[policy_engine.metrics] +enabled = false + +[controller.logging] +level = "error" + +[controller.controlplane] +insecure_skip_verify = true +gateway_name = "default" + +[controller.auth.basic] +enabled = true + +[[controller.auth.basic.users]] +username = "admin" +password = "admin" +roles = ["admin"] + +[immutable_gateway] +enabled = false +artifacts_dir = "/etc/api-platform-gateway/immutable_gateway/artifacts" + +[policy_configurations.jwtauth_v1] +# Must exceed test duration (warmup + run). 5m default caused a 401 burst at T+5min until runtime restart. +jwkscachettl = "24h" +jwksfetchtimeout = "5s" +jwksfetchretrycount = 5 +jwksfetchretryinterval = "2s" + +# jwt-auth v1.2.1 token verdict cache (policy-definition.yaml systemParameters). +# Caches signature-verified verdicts; bounds JWKS re-fetch vs repeat verification cost. +tokencaching = true +tokencachettl = "5m" +negativecachettl = "60s" +cachemaxsize = 100000 + +# Superseded jwt-auth token cache settings: +# tokencaching = true +# tokencachettl = "5m" +# negativecachettl = "60s" +# cachemaxsize = 1000 + +allowedalgorithms = ["RS256", "ES256"] +leeway = "30s" +authheaderscheme = "Bearer" +headername = "Authorization" +onfailurestatuscode = 401 +errormessageformat = "json" +errormessage = "Authentication failed" +validateissuer = true + +[[policy_configurations.jwtauth_v1.keymanagers]] +name = "test" +# Replace YOUR_TENANT before JWT scenarios, or override via a local overlay. +issuer = "https://api.asgardeo.io/t/YOUR_TENANT/oauth2/token" + +[policy_configurations.jwtauth_v1.keymanagers.jwks.remote] +uri = "https://api.asgardeo.io/t/YOUR_TENANT/oauth2/jwks" +skipTlsVerify = false + +[api_key] +api_keys_per_user_per_api = 10 + +# basic-ratelimit / advanced-ratelimit shared system config (in-memory, single gateway instance). +# See: https://github.com/wso2/gateway-controllers/blob/main/docs/basic-ratelimit/v1.0/docs/basic-ratelimit.md +[policy_configurations.ratelimit_v1] +algorithm = "fixed-window" +backend = "memory" + +[policy_configurations.ratelimit_v1.memory] +max_entries = 100000 +cleanup_interval = "5m" + +[collector] +request_body = false +response_body = false +request_headers = false +response_headers = false + +[traffic_logging] +enabled = false +masked_headers = ["authorization", "x-api-key", "x-jwt-assertion"] +max_payload_size = 2048 +request_headers = false +response_headers = false +request_body = false +response_body = false +exclude_fields = [] diff --git a/gateway/perf/performance-test-scripts/api-gateway/deploy/create-rest-perf-api.sh b/gateway/perf/performance-test-scripts/api-gateway/deploy/create-rest-perf-api.sh new file mode 100755 index 000000000..548ad8573 --- /dev/null +++ b/gateway/perf/performance-test-scripts/api-gateway/deploy/create-rest-perf-api.sh @@ -0,0 +1,182 @@ +#!/bin/bash -e +# Deploy RestApi for API Gateway EKS perf (plain, set-headers, jwt-auth only). + +script_dir=$(dirname "$0") +controller_host="${GATEWAY_CONTROLLER_HOST:-127.0.0.1}" +controller_port="${GATEWAY_CONTROLLER_PORT:-9090}" +mgmt_user="${GATEWAY_MGMT_USER:-admin}" +mgmt_pass="${GATEWAY_MGMT_PASS:-admin}" +mgmt_base="${GATEWAY_MGMT_API_BASE:-/api/management/v1}" +upstream_url="${MOCK_BACKEND_URL:-http://127.0.0.1:${BACKEND_PORT:-8688}/v1}" +api_name="" +api_mode="plain" +api_version="1.0.0" +route_count="8" +chat_path="/chat/completions" +enable_ratelimit="0" +operations_mode="both" +ratelimit_requests="${RATELIMIT_REQUESTS:-1000000000}" +ratelimit_duration="${RATELIMIT_DURATION:-24h}" + +function usage() { + echo "" + echo "Usage: $0 -n [-m ] [-r ] [-l] [-o get|both] [-h]" + echo "" + echo "-n: API context prefix (e.g. api-plain). Gateway path: //${chat_path}" + echo "-m: plain | add_headers | jwt_auth" + echo "-r: Route count (default 8): 1 = base only; 8 = base + /r1-/r7" + echo "-l: Attach basic-ratelimit v1 (high quota for perf)" + echo "-o: get (GET only) | both (GET+POST, default)" + echo "" +} + +while getopts "n:m:r:lo:h" opt; do + case "${opt}" in + n) api_name=${OPTARG} ;; + m) api_mode=${OPTARG} ;; + r) route_count=${OPTARG} ;; + l) enable_ratelimit="1" ;; + o) operations_mode=${OPTARG} ;; + h) usage; exit 0 ;; + \?) usage; exit 1 ;; + esac +done + +if [[ ! "$route_count" =~ ^[0-9]+$ ]] || [[ "$route_count" -lt 1 ]]; then + echo "Invalid -r ${route_count}. Use a positive integer (e.g. 1, 8)." + exit 1 +fi + +resource_suffixes=("") +if [[ "$route_count" -gt 1 ]]; then + for ((i = 1; i < route_count; i++)); do + resource_suffixes+=("/r${i}") + done +fi + +if [[ "$operations_mode" != "get" && "$operations_mode" != "both" ]]; then + echo "Invalid -o ${operations_mode}. Use get or both." + exit 1 +fi + +if [[ -z $api_name ]]; then + echo "Please provide -n " + exit 1 +fi + +if [[ $api_mode != "plain" && $api_mode != "add_headers" && $api_mode != "jwt_auth" ]]; then + echo "Invalid -m. Use: plain, add_headers, jwt_auth" + exit 1 +fi + +metadata_name="perf-${api_name}" +api_context="/${api_name}/${api_version}" +mgmt_url="http://${controller_host}:${controller_port}${mgmt_base}/rest-apis" +auth="${mgmt_user}:${mgmt_pass}" +yaml_file="/tmp/${metadata_name}-$$.yaml" +curl_timeout_args=(--connect-timeout "${GATEWAY_CURL_CONNECT_TIMEOUT:-5}" --max-time "${GATEWAY_CURL_MAX_TIME:-30}") + +cat >"$yaml_file" <>"$yaml_file" <>"$yaml_file" <>"$yaml_file" <>"$yaml_file" <>"$yaml_file" <>"$yaml_file" < api_api_plain_get +# perf-api-header -> api_api_header_get +# perf-api-jwt -> api_api_jwt_get +# +# Usage: +# source env.eks && ./deploy-apis-eks-minimal.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=eks-common.sh +source "${SCRIPT_DIR}/eks-common.sh" +eks_load_env +eks_require_cmd kubectl curl aws python3 +eks_kubecontext + +cleanup() { eks_stop_port_forward_controller; } +trap cleanup EXIT + +keep_apis=(perf-api-plain perf-api-header perf-api-jwt) + +echo "==> Port-forward controller :${GATEWAY_CONTROLLER_PORT}" +eks_port_forward_controller + +export GATEWAY_CONTROLLER_HOST="127.0.0.1" +export GATEWAY_CONTROLLER_PORT +export GATEWAY_MGMT_USER="${GATEWAY_MGMT_USER:-admin}" +export GATEWAY_MGMT_PASS="${GATEWAY_MGMT_PASS:-admin}" +# After port-forward, eks_wait_controller_ready may have auto-detected the live base. +export GATEWAY_MGMT_API_BASE="${GATEWAY_MGMT_API_BASE:-/api/management/v1}" +export MOCK_BACKEND_URL +export JWT_KEYMANAGER_NAME +export RATELIMIT_REQUESTS +export RATELIMIT_DURATION + +echo "==> Management API: ${GATEWAY_MGMT_API_BASE}" +mgmt_url="http://${GATEWAY_CONTROLLER_HOST}:${GATEWAY_CONTROLLER_PORT}${GATEWAY_MGMT_API_BASE}/rest-apis" +auth="${GATEWAY_MGMT_USER}:${GATEWAY_MGMT_PASS}" +DEPLOY_DIR="${API_GATEWAY_DIR}/deploy" + +echo "==> Upstream: ${MOCK_BACKEND_URL}" + +list_json=$(curl -sf -u "$auth" "$mgmt_url") +api_names=() +while IFS= read -r name; do + [[ -n "$name" ]] && api_names+=("$name") +done < <(python3 -c " +import json, sys +data = json.load(sys.stdin) +apis = data.get('apis', data.get('items', [])) +for api in apis: + name = api.get('metadata', {}).get('name') or api.get('name') + if name: + print(name) +" <<<"$list_json") + +api_count=0 +for _n in "${api_names[@]-}"; do + [[ -n "$_n" ]] && api_count=$((api_count + 1)) +done +echo "==> Before: ${api_count} RestApi(s)" +deleted=0 +for name in "${api_names[@]-}"; do + [[ -n "$name" ]] || continue + keep_it=0 + for k in "${keep_apis[@]}"; do + [[ "$name" == "$k" ]] && keep_it=1 && break + done + [[ "$keep_it" -eq 1 ]] && continue + code=$(curl -s -o /tmp/delete-rest-api-eks.json -w "%{http_code}" \ + -u "$auth" -X DELETE "${mgmt_url}/${name}") + if [[ "$code" == "200" || "$code" == "204" ]]; then + echo " deleted ${name}" + deleted=$((deleted + 1)) + else + echo " FAILED ${name} (HTTP ${code})" >&2 + cat /tmp/delete-rest-api-eks.json >&2 + exit 1 + fi +done +echo "==> Deleted ${deleted} RestApi(s)" + +echo "==> Deploy 3 RestApis (8 paths × GET+POST = 16 operations: base + /r1-/r7)" +"${DEPLOY_DIR}/create-rest-perf-api.sh" -n api-plain -m plain -r 8 -o both +"${DEPLOY_DIR}/create-rest-perf-api.sh" -n api-header -m add_headers -r 8 -o both +"${DEPLOY_DIR}/create-rest-perf-api.sh" -n api-jwt -m jwt_auth -r 8 -o both + +echo "==> Waiting for routes to propagate..." +sleep 8 + +remaining=$(curl -sf -u "$auth" "$mgmt_url" | python3 -c " +import json, sys +data = json.load(sys.stdin) +apis = data.get('apis', data.get('items', [])) +for api in apis: + print(api.get('metadata', {}).get('name') or api.get('name')) +") +echo "==> After:" +echo "$remaining" | sed 's/^/ /' + +echo "==> Done." diff --git a/gateway/perf/performance-test-scripts/api-gateway/eks/eks-common.sh b/gateway/perf/performance-test-scripts/api-gateway/eks/eks-common.sh new file mode 100755 index 000000000..e4bfd0ec8 --- /dev/null +++ b/gateway/perf/performance-test-scripts/api-gateway/eks/eks-common.sh @@ -0,0 +1,563 @@ +#!/bin/bash +# Shared helpers for EKS gateway perf scripts. + +eks_require_cmd() { + local cmd + for cmd in "$@"; do + command -v "$cmd" >/dev/null 2>&1 || { + echo "Missing required command: ${cmd}" >&2 + exit 1 + } + done +} + +eks_load_env() { + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[1]}")" && pwd)" + if [[ -f "${script_dir}/env.eks" ]]; then + # shellcheck source=env.eks + source "${script_dir}/env.eks" + elif [[ -f "${script_dir}/env.eks.example" ]]; then + echo "WARNING: using env.eks.example — copy to env.eks and edit" >&2 + # shellcheck source=env.eks.example + source "${script_dir}/env.eks.example" + else + echo "Missing env.eks (copy from env.eks.example)" >&2 + exit 1 + fi + export EKS_DIR="${EKS_DIR:-${script_dir}}" +} + +eks_clear_localhost_proxy_if_needed() { + local _p _v + [[ -z "${EKS_KEEP_PROXY:-}" ]] || return 0 + for _p in HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy; do + _v="${!_p:-}" + if [[ "$_v" =~ ^https?://127\.0\.0\.1: ]] || [[ "$_v" =~ ^socks5?://127\.0\.0\.1: ]]; then + unset "$_p" + fi + done +} + +eks_helm_chart_ok() { + if [[ "${GATEWAY_HELM_CHART}" == oci://* ]]; then + return 0 + fi + [[ -f "${GATEWAY_HELM_CHART}/Chart.yaml" ]] || { + echo "Helm chart not found: ${GATEWAY_HELM_CHART}" >&2 + exit 1 + } +} + +eks_kubecontext() { + eks_clear_localhost_proxy_if_needed + local _args=(--name "${EKS_CLUSTER_NAME}" --region "${AWS_REGION}") + [[ -n "${AWS_PROFILE:-}" ]] && _args+=(--profile "${AWS_PROFILE}") + aws eks describe-cluster "${_args[@]}" >/dev/null 2>&1 || { + echo "EKS cluster '${EKS_CLUSTER_NAME}' not found in ${AWS_REGION}" >&2 + echo "Create it in AWS Console, then: ./connect-kubeconfig.sh" >&2 + exit 1 + } + aws eks update-kubeconfig "${_args[@]}" >/dev/null +} + +# Build helm chart ref for install/upgrade (bash 3.2 safe — no mapfile). +# Usage: helm_args=(-n "$EKS_NAMESPACE"); eks_helm_append_chart_ref helm_args; helm_args+=(-f ...) +eks_helm_append_chart_ref() { + local _var="$1" + eval "${_var}+=(\"${EKS_RELEASE_NAME}\" \"${GATEWAY_HELM_CHART}\")" + if [[ -n "${GATEWAY_HELM_CHART_VERSION:-}" ]]; then + eval "${_var}+=(--version \"${GATEWAY_HELM_CHART_VERSION}\")" + fi +} + +eks_controller_svc() { + echo "${EKS_RELEASE_NAME}-controller" +} + +eks_runtime_svc() { + echo "${EKS_RELEASE_NAME}-gateway-runtime" +} + +# Validate selected storage class and ensure it is CSI-backed. +eks_validate_storage_class() { + local sc prov + sc="${EKS_STORAGE_CLASS:-}" + [[ -n "$sc" ]] || { + echo "EKS_STORAGE_CLASS is empty. Set a CSI-backed class in env.eks (e.g., gp3-csi-auto)." >&2 + return 1 + } + prov="$(kubectl get storageclass "$sc" -o jsonpath='{.provisioner}' 2>/dev/null || true)" + [[ -n "$prov" ]] || { + echo "StorageClass '${sc}' not found. Create one and set EKS_STORAGE_CLASS." >&2 + return 1 + } + if [[ "$prov" == "kubernetes.io/aws-ebs" ]]; then + echo "StorageClass '${sc}' uses legacy provisioner kubernetes.io/aws-ebs." >&2 + echo "Use CSI-backed class (provisioner ebs.csi.eks.amazonaws.com) to avoid Pending PVC." >&2 + return 1 + fi +} + +# Patch controller PVC storage class if it exists with empty storageClassName. +eks_fix_controller_pvc_storage_class() { + local pvc sc current_sc + pvc="${EKS_RELEASE_NAME}-controller-data" + sc="${EKS_STORAGE_CLASS:-}" + [[ -n "$sc" ]] || return 0 + if ! kubectl get pvc "$pvc" -n "${EKS_NAMESPACE}" >/dev/null 2>&1; then + return 0 + fi + current_sc="$(kubectl get pvc "$pvc" -n "${EKS_NAMESPACE}" -o jsonpath='{.spec.storageClassName}' 2>/dev/null || true)" + if [[ -z "$current_sc" ]]; then + echo "==> Patching PVC ${pvc} storageClassName=${sc}" + kubectl patch pvc "$pvc" -n "${EKS_NAMESPACE}" -p "{\"spec\":{\"storageClassName\":\"${sc}\"}}" >/dev/null || true + fi +} + +# The committed overlay ships a YOUR_TENANT placeholder in the jwt-auth issuer/JWKS URLs. +# Resolve it from JWT_TENANT, or derive it from JWT_OAUTH_TOKEN_URL (the same Asgardeo app +# that mints the test tokens) so the validator and the token source cannot drift apart. +eks_jwt_tenant() { + if [[ -n "${JWT_TENANT:-}" ]]; then + echo "${JWT_TENANT}" + return 0 + fi + # https://api.asgardeo.io/t//oauth2/token -> + if [[ "${JWT_OAUTH_TOKEN_URL:-}" =~ /t/([^/]+)/ ]]; then + echo "${BASH_REMATCH[1]}" + fi +} + +# Sections from config.perf-overlay.toml safe to append after Helm-generated config.toml +# (no duplicate [router], [controller], [policy_engine], etc.). +eks_append_perf_config_toml() { + local overlay="${PERF_CONFIG_TOML:?Set PERF_CONFIG_TOML}" + [[ -f "$overlay" ]] || { + echo "Perf overlay not found: ${overlay}" >&2 + return 1 + } + local tenant + tenant="$(eks_jwt_tenant)" + if [[ -z "${tenant}" ]] && grep -q 'YOUR_TENANT' "$overlay"; then + # Every jwt-auth request 401s when the issuer stays on the placeholder. + echo "WARNING: overlay still has YOUR_TENANT and no tenant resolved." >&2 + echo " Set JWT_TENANT or JWT_OAUTH_TOKEN_URL, else jwt scenarios return 401." >&2 + fi + awk ' + /^\[router\.upstream\.circuit_breakers\]/ { emit = 1 } + /^\[policy_configurations/ { emit = 1; in_policy = 1 } + /^\[\[policy_configurations/ { emit = 1; in_policy = 1 } + /^\[immutable_gateway\]/ { emit = 0; in_policy = 0; next } + /^\[/ { + if (emit && !in_policy && $0 !~ /^\[router\.upstream\.circuit_breakers\]/) { + emit = 0 + } + if (emit && in_policy && $0 !~ /policy_configurations/) { + in_policy = 0 + emit = 0 + } + } + emit { print } + ' "$overlay" \ + | if [[ -n "${tenant}" ]]; then sed "s#YOUR_TENANT#${tenant}#g"; else cat; fi +} + +eks_generated_values_path() { + echo "${EKS_DIR}/.generated-values.yaml" +} + +# Build runtime extraEnv lines (GOGC / GOMEMLIMIT when set — Jenkins defaults both on). +eks_runtime_extra_env_yaml() { + if [[ -n "${GOGC:-}" ]]; then + echo " - name: GOGC" + echo " value: \"${GOGC}\"" + fi + if [[ -n "${GOMEMLIMIT:-}" ]]; then + echo " - name: GOMEMLIMIT" + echo " value: \"${GOMEMLIMIT}\"" + fi +} + +# Recreate runtime Deployment before helm upgrade (clears kubectl-patch SSA conflicts). +eks_reset_runtime_deployment_for_helm() { + local deploy + deploy="${EKS_RELEASE_NAME}-gateway-runtime" + if kubectl get deployment "$deploy" -n "${EKS_NAMESPACE}" >/dev/null 2>&1; then + echo "==> Recreating ${deploy} (clear field-manager conflicts from kubectl patch)" + kubectl delete deployment "$deploy" -n "${EKS_NAMESPACE}" --wait=true + fi +} + +# Patch runtime Deployment args (custom images only; e.g. --rtr.disable-hot-restart). +eks_patch_runtime_args() { + local deploy extra + deploy="${EKS_RELEASE_NAME}-gateway-runtime" + extra="${GATEWAY_RUNTIME_EXTRA_ARGS:-}" + [[ -n "$extra" ]] || return 0 + + kubectl patch deployment "$deploy" -n "${EKS_NAMESPACE}" --type=strategic -p " +spec: + template: + spec: + containers: + - name: gateway-runtime + args: + - --pol.config + - /etc/policy-engine/config.toml + - ${extra} +" + kubectl rollout status -n "${EKS_NAMESPACE}" "deployment/${deploy}" --timeout=300s +} + +# Optional YAML fragment for image override (omit when unset → chart defaults). +eks_helm_controller_image_yaml() { + [[ -n "${GATEWAY_CONTROLLER_IMAGE:-}" ]] || return 0 + cat <&2 + exit 1 + } + if ! grep -q '"ghcr.io"' "$docker_config" 2>/dev/null; then + echo "ERROR: no ghcr.io credentials in ${docker_config} — run 'docker login ghcr.io'" >&2 + exit 1 + fi + echo "==> Ensure image pull secret: ${secret_name}" + kubectl create secret generic "${secret_name}" \ + --from-file=.dockerconfigjson="${docker_config}" \ + --type=kubernetes.io/dockerconfigjson \ + -n "${EKS_NAMESPACE}" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +eks_helm_image_pull_secrets_yaml() { + local secret_name + eks_uses_private_ghcr_images || return 0 + secret_name="${EKS_GHCR_PULL_SECRET:-ghcr-pull}" + cat </dev/null 2>&1; then + return 0 + fi + + tmp_dir="$(mktemp -d)" + key_file="${tmp_dir}/default-aesgcm256-v1.bin" + python3 - "$key_file" <<'PY' +import os +import sys +with open(sys.argv[1], "wb") as f: + f.write(os.urandom(32)) +PY + + kubectl create secret generic "${secret_name}" \ + --from-file=default-aesgcm256-v1.bin="${key_file}" \ + -n "${EKS_NAMESPACE}" \ + --dry-run=client -o yaml | kubectl apply -f - + + rm -rf "${tmp_dir}" +} + +# Build Helm values aligned with docker-compose.perf.yaml + config.perf-overlay.toml. +eks_write_generated_values() { + local out replicas ctrl_cpu ctrl_mem ctrl_pvc_size ctrl_sqlite_path rt_cpu rt_mem lb_type extra_env + local ctrl_image_yaml rt_image_yaml pull_secrets_yaml + local ctrl_encryption_secret + out="$(eks_generated_values_path)" + replicas="${GATEWAY_RUNTIME_REPLICAS:-1}" + ctrl_cpu="${GATEWAY_CONTROLLER_CPU_LIMIT:-1}" + ctrl_mem="${GATEWAY_CONTROLLER_MEM_LIMIT:-2Gi}" + ctrl_pvc_size="${GATEWAY_CONTROLLER_PVC_SIZE:-1Gi}" + ctrl_sqlite_path="${GATEWAY_CONTROLLER_SQLITE_PATH:-/app/data/gateway.db}" + rt_cpu="${GATEWAY_RUNTIME_CPU_LIMIT:-4}" + rt_mem="${GATEWAY_RUNTIME_MEM_LIMIT:-2Gi}" + lb_type="nlb" + extra_env="$(eks_runtime_extra_env_yaml)" + ctrl_image_yaml="$(eks_helm_controller_image_yaml)" + rt_image_yaml="$(eks_helm_runtime_image_yaml)" + pull_secrets_yaml="$(eks_helm_image_pull_secrets_yaml)" + ctrl_encryption_secret="$(eks_controller_encryption_secret_name)" + log_level="${LOG_LEVEL:-info}" + policy_engine_metrics="${POLICY_ENGINE_METRICS_ENABLED:-true}" + + cat >"$out" <>"$out" +} + +eks_port_forward_controller() { + local pidfile="${EKS_DIR}/.controller-port-forward.pid" + local logfile="${EKS_DIR}/.controller-port-forward.log" + local port i + port="${GATEWAY_CONTROLLER_PORT:-9090}" + if [[ -f "$pidfile" ]] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + eks_wait_controller_ready && return 0 + fi + eks_stop_port_forward_controller + : >"$logfile" + kubectl port-forward -n "${EKS_NAMESPACE}" \ + "svc/$(eks_controller_svc)" \ + "${port}:9090" \ + >>"$logfile" 2>&1 & + echo $! >"$pidfile" + disown + eks_wait_controller_ready +} + +# Controller Service exposes REST :9090 only (admin :9092 is pod-local, not on the Service). +eks_wait_controller_ready() { + local port i user pass base_paths base status + port="${GATEWAY_CONTROLLER_PORT:-9090}" + user="${GATEWAY_MGMT_USER:-admin}" + pass="${GATEWAY_MGMT_PASS:-admin}" + # Prefer chart-aligned base first, then probe both (1.1.0=v0.9, 1.2+=v1). + base_paths=("${GATEWAY_MGMT_API_BASE:-}" "/api/management/v0.9" "/api/management/v1") + # Deduplicate empty / repeats while preserving order + local -a uniq=() + local b seen + for b in "${base_paths[@]}"; do + [[ -n "$b" ]] || continue + seen=0 + for u in "${uniq[@]-}"; do + [[ "$u" == "$b" ]] && { seen=1; break; } + done + [[ "$seen" -eq 0 ]] && uniq+=("$b") + done + base_paths=("${uniq[@]}") + for i in $(seq 1 30); do + for base in "${base_paths[@]}"; do + status="$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 2 --max-time 5 \ + -u "${user}:${pass}" "http://127.0.0.1:${port}${base}/rest-apis" || true)" + if [[ "${status}" == "200" ]]; then + export GATEWAY_MGMT_API_BASE="${base}" + return 0 + fi + done + sleep 1 + done + echo "Controller not reachable via port-forward (REST :${port})" >&2 + local logfile="${EKS_DIR}/.controller-port-forward.log" + [[ -f "$logfile" ]] && tail -20 "$logfile" >&2 + return 1 +} + +eks_stop_port_forward_controller() { + local pidfile="${EKS_DIR}/.controller-port-forward.pid" + local pid + [[ -f "$pidfile" ]] || return 0 + pid="$(cat "$pidfile")" + kill "$pid" 2>/dev/null || true + sleep 1 + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" 2>/dev/null || true + rm -f "$pidfile" +} + +eks_runtime_hostname() { + local svc ip host + svc="$(eks_runtime_svc)" + ip=$(kubectl get svc -n "${EKS_NAMESPACE}" "$svc" \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + if [[ -n "$ip" ]]; then + echo "$ip" + return 0 + fi + ip=$(kubectl get svc -n "${EKS_NAMESPACE}" "$svc" \ + -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) + [[ -n "$ip" ]] && echo "$ip" +} + +eks_wait_runtime_lb() { + local i host + echo "Waiting for runtime LoadBalancer hostname..." + for i in $(seq 1 60); do + host="$(eks_runtime_hostname)" + if [[ -n "$host" ]]; then + echo "$host" + return 0 + fi + sleep 10 + done + echo "Timed out waiting for LoadBalancer" >&2 + return 1 +} diff --git a/gateway/perf/performance-test-scripts/api-gateway/eks/env.eks.example b/gateway/perf/performance-test-scripts/api-gateway/eks/env.eks.example new file mode 100644 index 000000000..3c1ce3511 --- /dev/null +++ b/gateway/perf/performance-test-scripts/api-gateway/eks/env.eks.example @@ -0,0 +1,111 @@ +# Copy to env.eks and edit, then: source env.eks +# +# EKS perf gateway — separate from EC2 docker-compose flow (env.gateway / env.scaled). + +# Copy to env.eks and edit, then load env (use bash — zsh breaks BASH_SOURCE / glob in comments): +# bash -c 'source env.eks && env | grep -E "^EKS_|^AWS_|^GATEWAY_"' +# Or: set -a && source env.eks && set +a (from bash shell only) + +if [[ -n "${BASH_SOURCE[0]:-}" ]]; then + _EKS_ENV_SELF="${BASH_SOURCE[0]}" +else + _EKS_ENV_SELF="${0}" +fi +MANUAL_DIR="$(cd "$(dirname "${_EKS_ENV_SELF}")/../.." && pwd)" +export PERF_ROOT="${PERF_ROOT:-$(cd "${MANUAL_DIR}/.." && pwd)}" +export API_GATEWAY_DIR="${MANUAL_DIR}/api-gateway" +export EKS_DIR="${API_GATEWAY_DIR}/eks" + +# Official Helm chart (OCI). Docs: https://wso2.com/api-platform/docs/api-gateway/deployment/deployment-modes/kubernetes/kubernetes-standalone/ +export GATEWAY_HELM_CHART="oci://ghcr.io/wso2/api-platform/helm-charts/gateway" +export GATEWAY_HELM_CHART_VERSION="1.2.0-rc" + +# Optional: local chart from api-platform clone instead of OCI. +# export GATEWAY_HELM_CHART="${HOME}/wso2-apim/api-platform/kubernetes/helm/gateway-helm-chart" + +# AWS CLI profile for account +export AWS_PROFILE="${AWS_PROFILE:-apip-perf}" + +export AWS_REGION="${AWS_REGION:-eu-north-1}" +export EKS_CLUSTER_NAME="${EKS_CLUSTER_NAME:-perf-api-gateway}" +export EKS_K8S_VERSION="${EKS_K8S_VERSION:-1.29}" + +# Gateway Auto Mode NodePool (gateway-perf) — e.g. m5.2xlarge (8 vCPU, 32 GiB). +# Change live nodes via NodePool patch, not classic managed node groups. +export EKS_NODE_INSTANCE_TYPE="${EKS_NODE_INSTANCE_TYPE:-m5.2xlarge}" +# Nodes: at least one per runtime replica (controller shares a node). Override if you use larger nodes. +export GATEWAY_RUNTIME_REPLICAS="1" +_replicas_for_nodes="${GATEWAY_RUNTIME_REPLICAS}" +[[ "${_replicas_for_nodes}" =~ ^[0-9]+$ ]] || _replicas_for_nodes=1 +export EKS_NODE_DESIRED="${EKS_NODE_DESIRED:-$((_replicas_for_nodes + 1))}" +unset _replicas_for_nodes +export EKS_NODE_MIN="${EKS_NODE_MIN:-1}" +export EKS_NODE_MAX="${EKS_NODE_MAX:-8}" + +# Reuse existing VPC where backend + JMeter EC2 live (recommended). Leave empty to let eksctl create a new VPC. +export EKS_VPC_ID="${EKS_VPC_ID:-}" +export EKS_PRIVATE_SUBNET_IDS="${EKS_PRIVATE_SUBNET_IDS:-}" # comma-separated subnet-xxx,subnet-yyy +export EKS_PUBLIC_SUBNET_IDS="${EKS_PUBLIC_SUBNET_IDS:-}" # for internal NLB you still need private subnets for nodes + +# --- Helm release --- +export EKS_NAMESPACE="${EKS_NAMESPACE:-api-gateway}" +export EKS_RELEASE_NAME="${EKS_RELEASE_NAME:-ap-gateway}" +export GATEWAY_CONTROLLER_ENCRYPTION_SECRET="${GATEWAY_CONTROLLER_ENCRYPTION_SECRET:-${EKS_RELEASE_NAME}-controller-encryption-keys}" + +# Container images: leave unset to use Helm chart defaults +# (ghcr.io/wso2/api-platform/gateway-controller:1.1.0 and gateway-runtime:1.1.0). +# Override only for custom builds: +# export GATEWAY_CONTROLLER_IMAGE=ghcr.io/example/gateway-controller:1.2.0 +# export GATEWAY_RUNTIME_IMAGE=ghcr.io/example/gateway-runtime:1.2.0 + +# docker-compose.perf.yaml: controller 1 CPU / 2g, runtime 4 CPU / 2g. +export GATEWAY_CONTROLLER_CPU_LIMIT="1" +export GATEWAY_CONTROLLER_MEM_LIMIT="2Gi" +export GATEWAY_CONTROLLER_PVC_SIZE="${GATEWAY_CONTROLLER_PVC_SIZE:-1Gi}" +# Set to a CSI-backed storage class in your cluster (example: gp3-csi-auto). +# Legacy in-tree classes (provisioner kubernetes.io/aws-ebs) will stay Pending. +export EKS_STORAGE_CLASS="${EKS_STORAGE_CLASS:-}" +export GATEWAY_CONTROLLER_SQLITE_PATH="${GATEWAY_CONTROLLER_SQLITE_PATH:-/app/data/gateway.db}" +export GATEWAY_RUNTIME_CPU_LIMIT="4" +export GATEWAY_RUNTIME_MEM_LIMIT="2Gi" +export ROUTER_CONCURRENCY="${ROUTER_CONCURRENCY:-4}" +export GOMAXPROCS="${GOMAXPROCS:-4}" +export LOG_LEVEL="${LOG_LEVEL:-info}" +export GOGC="${GOGC:-400}" +export GOMEMLIMIT="${GOMEMLIMIT:-1500MiB}" + +# NLB: internal (same-VPC JMeter) or internet-facing. +export EKS_LB_SCHEME="${EKS_LB_SCHEME:-internal}" # internal | internet-facing + +# --- Backend (preferred: in-cluster mock backend Service) --- +export BACKEND_IN_CLUSTER="${BACKEND_IN_CLUSTER:-1}" +export BACKEND_NAMESPACE="${BACKEND_NAMESPACE:-${EKS_NAMESPACE}}" +export BACKEND_SERVICE_NAME="${BACKEND_SERVICE_NAME:-perf-mock-backend}" +export BACKEND_PORT="${BACKEND_PORT:-8688}" +# Fallback (EC2 backend) used when BACKEND_IN_CLUSTER=0 +export BACKEND_PRIVATE_IP="${BACKEND_PRIVATE_IP:-}" # required when BACKEND_IN_CLUSTER=0 +if [[ "${BACKEND_IN_CLUSTER}" == "1" ]]; then + export MOCK_BACKEND_URL="${MOCK_BACKEND_URL:-http://${BACKEND_SERVICE_NAME}.${BACKEND_NAMESPACE}.svc.cluster.local:${BACKEND_PORT}/v1}" +else + export MOCK_BACKEND_URL="${MOCK_BACKEND_URL:-http://${BACKEND_PRIVATE_IP}:${BACKEND_PORT}/v1}" +fi + +# --- API deploy / artifacts (reuses ../deploy-apis.sh) --- +export GATEWAY_CONTROLLER_HOST="${GATEWAY_CONTROLLER_HOST:-127.0.0.1}" +# Avoid 9090 — often bound by Rancher Desktop / local tunnels (wrong controller). +export GATEWAY_CONTROLLER_PORT="${GATEWAY_CONTROLLER_PORT:-19090}" +# 1.1.0 = /api/management/v0.9 ; 1.2.0+ = /api/management/v1 +# Hard-set with chart pin so a prior `source` cannot leave the wrong base stuck. +export GATEWAY_MGMT_API_BASE="/api/management/v1" +export API_KEY_COUNT="${API_KEY_COUNT:-10}" +export ARTIFACTS_DIR="${ARTIFACTS_DIR:-${HOME}/perf-artifacts}" +export JWT_KEYMANAGER_NAME="${JWT_KEYMANAGER_NAME:-test}" +export RATELIMIT_REQUESTS="${RATELIMIT_REQUESTS:-1000000000}" +export RATELIMIT_DURATION="${RATELIMIT_DURATION:-24h}" + +# Perf overlay — full file embedded into Helm config.toml (same as EC2 volume mount). +export PERF_CONFIG_TOML="${PERF_CONFIG_TOML:-${API_GATEWAY_DIR}/config.perf-overlay.toml}" + +# Runtime extra arg from docker-compose.perf.yaml (patched after Helm install). +# Matches EC2 perf: --rtr.disable-hot-restart. Set empty to disable. +export GATEWAY_RUNTIME_EXTRA_ARGS="${GATEWAY_RUNTIME_EXTRA_ARGS:---rtr.disable-hot-restart}" diff --git a/gateway/perf/performance-test-scripts/api-gateway/eks/install-gateway.sh b/gateway/perf/performance-test-scripts/api-gateway/eks/install-gateway.sh new file mode 100755 index 000000000..038d84de5 --- /dev/null +++ b/gateway/perf/performance-test-scripts/api-gateway/eks/install-gateway.sh @@ -0,0 +1,83 @@ +#!/bin/bash -e +# Install / upgrade API Platform Gateway on EKS via official OCI Helm chart. +# +# Prerequisite: EKS cluster created (AWS Console) and kubeconfig connected. +# Usage: +# cp env.eks.example env.eks && edit && source env.eks +# ./connect-kubeconfig.sh +# ./install-gateway.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=eks-common.sh +source "${SCRIPT_DIR}/eks-common.sh" +eks_load_env +eks_require_cmd kubectl helm aws +eks_kubecontext + +echo "==> Namespace: ${EKS_NAMESPACE}" +kubectl create namespace "${EKS_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +eks_ensure_ghcr_pull_secret +eks_ensure_controller_encryption_key_secret +eks_validate_storage_class +eks_fix_controller_pvc_storage_class + +eks_write_generated_values +gen_values="$(eks_generated_values_path)" +base_values="${EKS_DIR}/values.perf.yaml" + +helm_args=(-n "${EKS_NAMESPACE}") +eks_helm_append_chart_ref helm_args +if [[ -f "$base_values" ]]; then + helm_args+=(-f "$base_values") +fi +helm_args+=(-f "$gen_values") + +echo "==> Helm chart: ${GATEWAY_HELM_CHART}" +[[ -n "${GATEWAY_HELM_CHART_VERSION:-}" ]] && echo " version: ${GATEWAY_HELM_CHART_VERSION}" +echo "==> Runtime replicas: ${GATEWAY_RUNTIME_REPLICAS}" +echo "==> Runtime CPU/mem: ${GATEWAY_RUNTIME_CPU_LIMIT} / ${GATEWAY_RUNTIME_MEM_LIMIT}" +echo "==> Config: ${PERF_CONFIG_TOML}" +echo "==> Images: $(eks_helm_images_label)" + +if helm status "${EKS_RELEASE_NAME}" -n "${EKS_NAMESPACE}" >/dev/null 2>&1; then + eks_reset_runtime_deployment_for_helm + helm upgrade "${helm_args[@]}" +else + helm install "${helm_args[@]}" +fi + +echo "==> Enforce replica counts (controller=1, runtime=${GATEWAY_RUNTIME_REPLICAS:-1})" +kubectl scale deployment "${EKS_RELEASE_NAME}-controller" -n "${EKS_NAMESPACE}" --replicas=1 +kubectl scale deployment "${EKS_RELEASE_NAME}-gateway-runtime" -n "${EKS_NAMESPACE}" \ + --replicas="${GATEWAY_RUNTIME_REPLICAS:-1}" + +echo "==> Patch runtime args" +if [[ -n "${GATEWAY_RUNTIME_EXTRA_ARGS:-}" ]]; then + eks_patch_runtime_args +else + echo " (skipped — GATEWAY_RUNTIME_EXTRA_ARGS unset; required for official chart images)" +fi + +echo "==> Waiting for pods..." +kubectl wait -n "${EKS_NAMESPACE}" \ + --for=condition=ready pod \ + -l "app.kubernetes.io/instance=${EKS_RELEASE_NAME}" \ + --timeout=300s || true + +kubectl get pods -n "${EKS_NAMESPACE}" -o wide +kubectl get svc -n "${EKS_NAMESPACE}" + +host="$(eks_wait_runtime_lb || true)" +if [[ -n "${host:-}" ]]; then + echo "" + echo "Gateway runtime LB: http://${host}:8080" + echo "Save for JMeter: export GATEWAY_HOST=${host}" +fi + +cat < + + + + + false + false + + + + + + + + continue + + false + -1 + + ${__P(users)} + ${__P(rampUpPeriod,60)} + 1499334866000 + 1499334866000 + true + ${__P(duration)} + + + + + ${__P(tokens)} + + token + , + false + true + false + shareMode.all + false + + + + dual-gateway-port-v2 + + + if (!"1".equals(props.get("dualGateway"))) { + return +} +def hosts = (props.get("dualGatewayHosts") ?: "").split(",")*.trim().findAll { it } +def ports = (props.get("dualGatewayPorts") ?: "").split(",")*.trim().findAll { it } +if (ports.isEmpty()) { + return +} +def ip = java.net.InetAddress.getLocalHost().getHostAddress() +def engineIdx = hosts.findIndexOf { it == ip } +if (engineIdx < 0) { + engineIdx = 0 +} +def nEngines = Math.max(hosts.size(), 1) +def nPorts = ports.size() +if (nEngines >= nPorts) { + if (engineIdx < nPorts) { + props.put("port", ports[engineIdx]) + } +} else { + def portsPerEngine = (int) Math.ceil(nPorts / (double) nEngines) + def baseIdx = engineIdx * portsPerEngine + def slot = (ctx.getThreadNum() + engineIdx) % portsPerEngine + def portIdx = baseIdx + slot + if (portIdx < nPorts) { + props.put("port", ports[portIdx]) + } +} + groovy + + + + + + ${__P(authHeaderName,Authorization)} + ${token} + + + accept + application/json + + + Content-Type + application/json + + + + + + jwt-plain-resource-suffix-v1 + + + def raw = props.get("resourceSuffixes") +if (raw == null || raw.trim().isEmpty()) { + vars.put("resourceSuffix", "") + return +} +def items = raw.split(",") +def idx = new java.util.Random().nextInt(items.length) +vars.put("resourceSuffix", items[idx]) + groovy + + + + + + + false + ${__P(requestBody,)} + = + false + + + + ${__P(host,localhost)} + ${__P(port,8080)} + ${__P(protocol,http)} + + ${__P(path)}${resourceSuffix} + ${__P(method,GET)} + true + false + true + false + true + + HttpClient4 + 10000 + 600000 + + + + + 200 + + Assertion.response_code + false + 8 + + + + + + + true + true + + + diff --git a/gateway/perf/performance-test-scripts/jmeter/api-api-test-gateway-plain.jmx b/gateway/perf/performance-test-scripts/jmeter/api-api-test-gateway-plain.jmx new file mode 100644 index 000000000..ef7bc4d8f --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/api-api-test-gateway-plain.jmx @@ -0,0 +1,137 @@ + + + + + + false + false + + + + + + + + continue + + false + -1 + + ${__P(users)} + ${__P(rampUpPeriod,60)} + 1499334866000 + 1499334866000 + true + ${__P(duration)} + + + + + dual-gateway-port-v2 + + + if (!"1".equals(props.get("dualGateway"))) { + return +} +def hosts = (props.get("dualGatewayHosts") ?: "").split(",")*.trim().findAll { it } +def ports = (props.get("dualGatewayPorts") ?: "").split(",")*.trim().findAll { it } +if (ports.isEmpty()) { + return +} +def ip = java.net.InetAddress.getLocalHost().getHostAddress() +def engineIdx = hosts.findIndexOf { it == ip } +if (engineIdx < 0) { + engineIdx = 0 +} +def nEngines = Math.max(hosts.size(), 1) +def nPorts = ports.size() +if (nEngines >= nPorts) { + if (engineIdx < nPorts) { + props.put("port", ports[engineIdx]) + } +} else { + def portsPerEngine = (int) Math.ceil(nPorts / (double) nEngines) + def baseIdx = engineIdx * portsPerEngine + def slot = (ctx.getThreadNum() + engineIdx) % portsPerEngine + def portIdx = baseIdx + slot + if (portIdx < nPorts) { + props.put("port", ports[portIdx]) + } +} + groovy + + + + + + accept + application/json + + + Content-Type + application/json + + + + + + plain-resource-suffix-v1 + + + def raw = props.get("resourceSuffixes") +if (raw == null || raw.trim().isEmpty()) { + vars.put("resourceSuffix", "") + return +} +def items = raw.split(",") +def idx = new java.util.Random().nextInt(items.length) +vars.put("resourceSuffix", items[idx]) + groovy + + + + + + + false + ${__P(requestBody,)} + = + false + + + + ${__P(host,localhost)} + ${__P(port,8080)} + ${__P(protocol,http)} + + ${__P(path)}${resourceSuffix} + ${__P(method,GET)} + true + false + true + false + true + + HttpClient4 + 10000 + 600000 + + + + + 200 + + Assertion.response_code + false + 8 + + + + + + + true + true + + + diff --git a/gateway/perf/performance-test-scripts/jmeter/env.example b/gateway/perf/performance-test-scripts/jmeter/env.example new file mode 100644 index 000000000..6987a30e6 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/env.example @@ -0,0 +1,22 @@ +# Source after copying to env.jmeter on the JMeter EC2 (Jenkins generates this). + +_JMETER_ENV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MANUAL_DIR="$(cd "${_JMETER_ENV_DIR}/.." && pwd)" +export PERF_ROOT="${PERF_ROOT:-$(cd "${MANUAL_DIR}/.." && pwd)}" +# shellcheck source=../common.env.example +source "${MANUAL_DIR}/common.env.example" + +export MANUAL_DIR +export PERF_GATEWAY_TYPE="${PERF_GATEWAY_TYPE:-api-gateway}" +# Left empty for setup-only scripts (setup-jmeter.sh runs before env.jmeter exists); +# run-scenario.sh fails fast if still unset at test time. +export GATEWAY_HOST="${GATEWAY_HOST:-${GATEWAY_PRIVATE_IP:-}}" +export BACKEND_HOST="${BACKEND_HOST:-127.0.0.1}" +export BACKEND_IN_CLUSTER="${BACKEND_IN_CLUSTER:-1}" +export BACKEND_IMPL="${BACKEND_IMPL:-docker}" +export BACKEND_PORT="${BACKEND_PORT:-8688}" +export PERF_HOME="${PERF_HOME:-${HOME}/perf-api-gateway-manual}" +export PERF_LOCAL_SCRIPTS="${MANUAL_DIR}/jmeter" +export JMETER_ALIAS_PREFIX="${JMETER_ALIAS_PREFIX:-jmeter}" +export SKIP_JTL_SPLIT="${SKIP_JTL_SPLIT:-true}" +export SKIP_PAYLOAD_GENERATION="${SKIP_PAYLOAD_GENERATION:-true}" diff --git a/gateway/perf/performance-test-scripts/jmeter/env.jmeter.api.example b/gateway/perf/performance-test-scripts/jmeter/env.jmeter.api.example new file mode 100644 index 000000000..5d7698d28 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/env.jmeter.api.example @@ -0,0 +1,31 @@ +# Copy to env.jmeter.api on the JMeter controller, or let the Jenkins job upload one. +# API Gateway profile — placeholders only (do not ship lab EC2 IPs here). + +_JMETER_ENV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export PERF_ROOT="${PERF_ROOT:-${HOME}/perf}" +export MANUAL_DIR="${MANUAL_DIR:-${PERF_ROOT}/performance-test-scripts}" +# shellcheck source=../common.env.example +source "${MANUAL_DIR}/common.env.example" + +# --- Shared --- +export BACKEND_HOST="${BACKEND_HOST:-127.0.0.1}" +export BACKEND_IMPL="${BACKEND_IMPL:-docker}" +export BACKEND_PORT="${BACKEND_PORT:-8688}" +export BACKEND_IN_CLUSTER="${BACKEND_IN_CLUSTER:-1}" +export BACKEND_SSH="${BACKEND_SSH:-}" +export BACKEND_SSH_KEY="${BACKEND_SSH_KEY:-}" +export GATEWAY_ARTIFACTS="${GATEWAY_ARTIFACTS:-${HOME}/perf-artifacts}" +export JMETER_ALIAS_PREFIX="${JMETER_ALIAS_PREFIX:-jmeter}" +export PERF_LOCAL_SCRIPTS="${MANUAL_DIR}/jmeter" + +# --- API Gateway (EKS NLB hostname or EC2 private IP) --- +export PERF_GATEWAY_TYPE=api-gateway +export GATEWAY_HOST="${GATEWAY_HOST:-REPLACE_WITH_GATEWAY_HOST}" +export GATEWAY_PORT="${GATEWAY_PORT:-8080}" +export PERF_HOME="${PERF_HOME:-${HOME}/perf-api-gateway-manual}" + +# --- JWT (Asgardeo OAuth for api_api_jwt_*) --- +# [[ -f "${_JMETER_ENV_DIR}/env.jmeter.jwt" ]] && source "${_JMETER_ENV_DIR}/env.jmeter.jwt" +# export JWT_OAUTH_TOKEN_URL='https://api.asgardeo.io/t//oauth2/token' +# export JWT_OAUTH_CLIENT_ID='...' +# export JWT_OAUTH_CLIENT_SECRET='...' diff --git a/gateway/perf/performance-test-scripts/jmeter/gateway-scenarios.sh b/gateway/perf/performance-test-scripts/jmeter/gateway-scenarios.sh new file mode 100755 index 000000000..f9e559bfe --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/gateway-scenarios.sh @@ -0,0 +1,92 @@ +# API Gateway EKS perf — three baseline scenarios only. +# Must match deploy-apis-eks-minimal.sh (api-plain, api-header, api-jwt). +# Default 8 routes: base + /r1-/r7 (PERF_API_ROUTE_COUNT must match deploy -r). + +function _perf_api_route_suffixes() { + local routes="${1:-${PERF_API_ROUTE_COUNT:-8}}" + local i + local s="" + for ((i = 1; i < routes; i++)); do + s="${s},/r${i}" + done + echo "${s}" +} + +function load_perf_gateway_scenarios() { + unset test_scenario0 test_scenario1 test_scenario2 test_scenario 2>/dev/null || true + unset scenario_api_key_file scenario_auth_header 2>/dev/null || true + declare -gA scenario_api_key_file + declare -gA scenario_auth_header + + if [[ "${PERF_GATEWAY_TYPE:-api-gateway}" != "api-gateway" ]]; then + echo "ERROR: performance-test-scripts supports api-gateway only (got: ${PERF_GATEWAY_TYPE})" >&2 + exit 1 + fi + + local backend_port="${BACKEND_PORT:-8688}" + local backend_flags="--port ${backend_port}" + + scenario_api_key_file=( + [api_api_jwt_get]="jwt-tokens.csv" + ) + scenario_auth_header=( + [api_api_jwt_get]="Authorization" + ) + + declare -gA test_scenario0=( + [name]="api_api_plain_get" + [display_name]="API Gateway Plain GET" + [description]="RestApi plain GET — no policies (api-plain)." + [jmx]="api-api-test-gateway-plain.jmx" + [protocol]="http" + [path]="/api-plain/1.0.0/chat/completions" + [resource_suffixes]="$(_perf_api_route_suffixes "${PERF_API_ROUTE_COUNT:-8}")" + [host_type]="gateway" + [port]="8080" + [use_apim]=false + [use_backend]=true + [backend_flags]="${backend_flags}" + [skip]=false + [method]="GET" + ) + declare -gA test_scenario1=( + [name]="api_api_header_get" + [display_name]="API Gateway Header Policy GET" + [description]="set-headers policy GET (api-header)." + [jmx]="api-api-test-gateway-plain.jmx" + [protocol]="http" + [path]="/api-header/1.0.0/chat/completions" + [resource_suffixes]="$(_perf_api_route_suffixes "${PERF_API_ROUTE_COUNT:-8}")" + [host_type]="gateway" + [port]="8080" + [use_apim]=false + [use_backend]=true + [backend_flags]="${backend_flags}" + [skip]=false + [method]="GET" + ) + declare -gA test_scenario2=( + [name]="api_api_jwt_get" + [display_name]="API Gateway JWT GET" + [description]="jwt-auth policy GET (api-jwt)." + [jmx]="api-api-test-gateway-jwt-plain.jmx" + [protocol]="http" + [path]="/api-jwt/1.0.0/chat/completions" + [resource_suffixes]="$(_perf_api_route_suffixes "${PERF_API_ROUTE_COUNT:-8}")" + [host_type]="gateway" + [port]="8080" + [use_apim]=false + [use_backend]=true + [backend_flags]="${backend_flags}" + [skip]=false + [method]="GET" + ) + + declare -gA test_scenario=( + [test_scenario0]=1 + [test_scenario1]=1 + [test_scenario2]=1 + ) +} + +load_perf_gateway_scenarios diff --git a/gateway/perf/performance-test-scripts/jmeter/generate-jwt-tokens.sh b/gateway/perf/performance-test-scripts/jmeter/generate-jwt-tokens.sh new file mode 100755 index 000000000..909d6d6e5 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/generate-jwt-tokens.sh @@ -0,0 +1,134 @@ +#!/bin/bash -e +# Pre-generate Bearer tokens for api_api_jwt_get (Asgardeo OAuth on Jenkins/EKS). +# export JWT_OAUTH_TOKEN_URL='https://api.asgardeo.io/t//oauth2/token' +# export JWT_OAUTH_CLIENT_ID='...' +# export JWT_OAUTH_CLIENT_SECRET='...' +# JWT_REGENERATE=1 ./generate-jwt-tokens.sh +# +# Single token (JWKS/JWT cache test): +# JWT_SINGLE_TOKEN=1 ./generate-jwt-tokens.sh + +set -euo pipefail + +_out="${JWT_TOKENS_FILE:-${HOME}/jwt-tokens.csv}" +_jwt_count="${JWT_TOKEN_COUNT:-500}" +if [[ "${JWT_SINGLE_TOKEN:-}" == "1" ]]; then + _jwt_count=1 +fi + +_fetch_oauth_token() { + local token_url="${JWT_OAUTH_TOKEN_URL:?Set JWT_OAUTH_TOKEN_URL}" + local client_id="${JWT_OAUTH_CLIENT_ID:?Set JWT_OAUTH_CLIENT_ID}" + local client_secret="${JWT_OAUTH_CLIENT_SECRET:?Set JWT_OAUTH_CLIENT_SECRET}" + curl -sf --connect-timeout 15 --location "${token_url}" \ + --header 'Content-Type: application/x-www-form-urlencoded' \ + --data-urlencode 'grant_type=client_credentials' \ + --data-urlencode "client_id=${client_id}" \ + --data-urlencode "client_secret=${client_secret}" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' +} + +_fetch_mock_token() { + local token_url="$1" + curl -sf --connect-timeout 10 "${token_url}" +} + +_fetch_mock_tokens_remote() { + local jwt_count="$1" + local token_url="$2" + local out="$3" + local gw_user="${GATEWAY_SCP_USER:-${GATEWAY_SSH%%@*}}" + gw_user="${gw_user:-ec2-user}" + local gw_host="${GATEWAY_PRIVATE_IP:-${GATEWAY_HOST:?Set GATEWAY_HOST or GATEWAY_PRIVATE_IP}}" + local ssh_opts=(-o StrictHostKeyChecking=no -o ConnectTimeout=10) + + if [[ -n "${GATEWAY_SSH_KEY:-}" ]]; then + [[ -f "${GATEWAY_SSH_KEY}" ]] || { + echo "ERROR: GATEWAY_SSH_KEY not found: ${GATEWAY_SSH_KEY}" >&2 + return 1 + } + ssh_opts+=(-i "${GATEWAY_SSH_KEY}") + fi + + if ! ssh "${ssh_opts[@]}" "${gw_user}@${gw_host}" \ + "curl -sf 'http://127.0.0.1:8088/jwks' >/dev/null || (cd ~/perf/ai-gateway-manual/api-gateway && ./start-mock-jwks.sh)" \ + 2>/dev/null; then + echo "ERROR: Could not reach gateway or start mock-jwks (${gw_user}@${gw_host})" >&2 + return 1 + fi + + : >"${out}" + if [[ "${jwt_count}" -eq 1 ]]; then + token=$(ssh "${ssh_opts[@]}" "${gw_user}@${gw_host}" "curl -sf '${token_url}'") || return 1 + echo "Bearer ${token}" >>"${out}" + return 0 + fi + + echo "==> Fetching ${jwt_count} mock tokens via SSH ..." + ssh "${ssh_opts[@]}" "${gw_user}@${gw_host}" bash -s -- "${jwt_count}" "${token_url}" <<'REMOTE' >"${out}" +set -euo pipefail +count=$1 +url=$2 +for ((i = 1; i <= count; i++)); do + token=$(curl -sf "${url}") + printf 'Bearer %s\n' "${token}" +done +REMOTE +} + +generate_jwt_tokens() { + local out="$_out" + local jwt_count="$_jwt_count" + local i token + + : >"${out}" + + if [[ -n "${JWT_OAUTH_TOKEN_URL:-}" ]]; then + if [[ "${jwt_count}" -eq 1 ]]; then + echo "Generating 1 OAuth token at ${out} (all threads reuse it) ..." + else + echo "Generating ${jwt_count} OAuth tokens at ${out} (${JWT_OAUTH_TOKEN_URL}) ..." + fi + for ((i = 1; i <= jwt_count; i++)); do + token=$(_fetch_oauth_token) || break + echo "Bearer ${token}" >>"${out}" + done + else + local issuer="http://172.17.0.1:8088/token" + local token_url="http://127.0.0.1:8088/token?issuer=${issuer}" + if [[ "${jwt_count}" -eq 1 ]]; then + echo "Generating 1 mock JWT at ${out} (all threads reuse it) ..." + else + echo "Generating ${jwt_count} mock JWT tokens at ${out} (issuer=${issuer}) ..." + fi + if [[ -n "${GATEWAY_HOST:-}" ]] && curl -sf --connect-timeout 3 \ + "http://${GATEWAY_HOST}:8088/token?issuer=${issuer}" >/dev/null 2>&1; then + echo "==> Fetching mock tokens via HTTP ${GATEWAY_HOST}:8088" + token_url="http://${GATEWAY_HOST}:8088/token?issuer=${issuer}" + for ((i = 1; i <= jwt_count; i++)); do + token=$(_fetch_mock_token "${token_url}") || break + echo "Bearer ${token}" >>"${out}" + done + else + echo "==> Port 8088 not reachable — fetching mock tokens via SSH" + _fetch_mock_tokens_remote "${jwt_count}" "${token_url}" "${out}" || true + fi + fi + + local lines + lines=$(wc -l <"${out}" | tr -d ' ') + if [[ "${lines}" -lt 1 ]]; then + echo "ERROR: Failed to generate JWT tokens." >&2 + if [[ -n "${JWT_OAUTH_TOKEN_URL:-}" ]]; then + echo " Check JWT_OAUTH_TOKEN_URL, JWT_OAUTH_CLIENT_ID, JWT_OAUTH_CLIENT_SECRET" >&2 + else + echo " Set JWT_OAUTH_TOKEN_URL, JWT_OAUTH_CLIENT_ID, JWT_OAUTH_CLIENT_SECRET (Asgardeo OAuth)." >&2 + fi + return 1 + fi + echo "==> Wrote ${lines} JWT tokens to ${out}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + generate_jwt_tokens +fi diff --git a/gateway/perf/performance-test-scripts/jmeter/generate-summary.sh b/gateway/perf/performance-test-scripts/jmeter/generate-summary.sh new file mode 100755 index 000000000..1307d6966 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/generate-summary.sh @@ -0,0 +1,131 @@ +#!/bin/bash -e +# Build summary.csv from JMeter results (trimmed column set for API Gateway EKS). + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MANUAL_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=env.example +source "${SCRIPT_DIR}/env.example" +# shellcheck source=../lib/common.sh +source "${MANUAL_DIR}/lib/common.sh" +require_bash4 + +PERF_HOME="${PERF_HOME:-${HOME}/perf-api-gateway-manual}" +JMETER_DIR="${PERF_HOME}/jmeter" +RESULTS_DIR="${JMETER_DIR}/results" +WARMUP="${WARMUP:-}" +JTL_SPLITTER_JAR="${JTL_SPLITTER_JAR:-${PERF_ROOT}/performance-common/components/jtl-splitter/target/jtl-splitter-0.4.6-SNAPSHOT.jar}" +CREATE_SUMMARY="${PERF_ROOT}/performance-common/distribution/scripts/jmeter/create-summary-csv.sh" + +export JTL_SPLITTER_JAR + +if [[ -z "$WARMUP" && -f "${RESULTS_DIR}/test-metadata.json" ]]; then + WARMUP=$(jq -r '.warmup_time // empty' "${RESULTS_DIR}/test-metadata.json") +fi +WARMUP="${WARMUP:-300}" + +echo "==> Results: ${RESULTS_DIR}" +echo "==> Warmup: ${WARMUP}s" +[[ -d "$RESULTS_DIR" ]] || { echo "No results dir. Run a perf test first."; exit 1; } + +ensure_jtl_for_split() { + local run_dir="$1" + if [[ -f "${run_dir}/results.jtl" ]]; then + printf '%s\n' "${run_dir}/results.jtl" + return 0 + fi + if [[ -f "${run_dir}/jtls.zip" ]]; then + echo " extracting results.jtl from ${run_dir}/jtls.zip ..." >&2 + unzip -p "${run_dir}/jtls.zip" results.jtl >"${run_dir}/results.jtl" + printf '%s\n' "${run_dir}/results.jtl" + return 0 + fi + return 1 +} + +echo "==> Splitting JTLs and writing *-measurement-summary.json ..." +split_count=0 +while IFS= read -r run_dir; do + [[ -f "${run_dir}/results-measurement-summary.json" ]] && continue + jtl=$(ensure_jtl_for_split "$run_dir") || continue + echo " split: ${jtl} (large files may take several minutes)" >&2 + "${PERF_HOME}/jtl-splitter/jtl-splitter.sh" -m 2g -- \ + -f "$jtl" -d -t "$WARMUP" -u SECONDS -s + split_count=$((split_count + 1)) +done < <(find "$RESULTS_DIR" \( -name 'jtls.zip' -o -name 'results.jtl' \) | xargs -I{} dirname {} | sort -u) + +summary_json_count=$(find "$RESULTS_DIR" -name 'results-measurement-summary.json' | wc -l | tr -d ' ') +echo "==> Found ${summary_json_count} measurement summary JSON file(s) (split ops: ${split_count})" +if [[ "$summary_json_count" -eq 0 ]]; then + echo "ERROR: No results-measurement-summary.json under ${RESULTS_DIR}." >&2 + echo " Ensure a perf test completed and jtl-splitter JAR exists (mvn package in performance-common)." >&2 + exit 1 +fi + +cd "$JMETER_DIR" +rm -f summary.csv summary.full.csv + +# Path layout: ...//_heap/_users/B/B_response/ms_sleep +# create-summary-csv.sh locates the scenario directory by counting back one level per -c/-r +# pair, so every level below the scenario must be declared here even though the Python step +# below keeps only a subset. Omitting any pair shifts the window and yields the wrong +# Scenario Name plus N/A for Concurrent Users. +"$BASH" "$CREATE_SUMMARY" \ + -n "API Gateway" \ + -d "$RESULTS_DIR" \ + -j 1 \ + -p jmeter \ + -o summary.full.csv \ + -c "Heap Size" -r '([0-9]+[a-zA-Z])_heap' \ + -c "Concurrent Users" -r '([0-9]+)_users' \ + -c "Message Size (Bytes)" -r '^([0-9]+)B$' \ + -c "Response Size (Bytes)" -r '([0-9]+)B_response' \ + -c "Back-end Service Delay (ms)" -r '([0-9]+)ms_sleep' + +python3 - <<'PY' +import csv +from pathlib import Path + +src = Path("summary.full.csv") +dst = Path("summary.csv") +keep = [ + "Scenario Name", + "Concurrent Users", + "Throughput (Requests/sec)", + "Average Response Time (ms)", + "# Samples", + "Error Count", + "Error %", + "Average Users in the System", + "Standard Deviation of Response Time (ms)", + "Minimum Response Time (ms)", + "Maximum Response Time (ms)", + "75th Percentile of Response Time (ms)", + "90th Percentile of Response Time (ms)", + "95th Percentile of Response Time (ms)", + "98th Percentile of Response Time (ms)", + "99th Percentile of Response Time (ms)", + "99.9th Percentile of Response Time (ms)", + "Received (KB/sec)", + "Sent (KB/sec)", +] +with src.open(newline="") as f: + reader = csv.DictReader(f) + missing = [c for c in keep if c not in (reader.fieldnames or [])] + if missing: + raise SystemExit(f"summary.full.csv missing columns: {missing}") + rows = [{c: row.get(c, "") for c in keep} for row in reader] +with dst.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=keep) + writer.writeheader() + writer.writerows(rows) +print(f"==> Trimmed {len(rows)} row(s) -> summary.csv ({len(keep)} columns)") +PY + +rm -f summary.full.csv + +data_rows=$(($(wc -l Wrote ${JMETER_DIR}/summary.csv (${data_rows} data row(s))" +if [[ "$data_rows" -eq 0 ]]; then + echo "WARNING: CSV has headers only. Check results-measurement-summary.json files." >&2 + exit 1 +fi diff --git a/gateway/perf/performance-test-scripts/jmeter/jmeter-server-start.sh b/gateway/perf/performance-test-scripts/jmeter/jmeter-server-start.sh new file mode 100755 index 000000000..16e12aef9 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/jmeter-server-start.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e +# Delegates to performance-common JMeter server starter. + +# Follow symlinks: $HOME/jmeter -> perf-ai-gateway-manual/jmeter -> .../ai-gateway-manual/jmeter +SCRIPT_DIR="$(cd -P "$(dirname "$0")" && pwd)" +PERF_ROOT="$(cd -P "${SCRIPT_DIR}/../.." && pwd)" +exec "${PERF_ROOT}/performance-common/distribution/scripts/jmeter/jmeter-server-start.sh" "$@" diff --git a/gateway/perf/performance-test-scripts/jmeter/lib/gateway-profile.sh b/gateway/perf/performance-test-scripts/jmeter/lib/gateway-profile.sh new file mode 100755 index 000000000..bed6ae9d6 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/lib/gateway-profile.sh @@ -0,0 +1,17 @@ +# shellcheck shell=bash +# Load env.jmeter. (or .example) after PERF_GATEWAY_TYPE is set. + +load_jmeter_gateway_profile() { + local script_dir="$1" + local profile="${script_dir}/env.jmeter.${PERF_GATEWAY_TYPE%-gateway}" + if [[ -f "$profile" ]]; then + # shellcheck source=/dev/null + source "$profile" + elif [[ -f "${profile}.example" ]]; then + # shellcheck source=/dev/null + source "${profile}.example" + else + echo "Missing ${profile} (and no .example). Copy env.jmeter.api.example to env.jmeter.api" >&2 + exit 1 + fi +} diff --git a/gateway/perf/performance-test-scripts/jmeter/run-scenario.sh b/gateway/perf/performance-test-scripts/jmeter/run-scenario.sh new file mode 100755 index 000000000..163f0774f --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/run-scenario.sh @@ -0,0 +1,361 @@ +#!/bin/bash -e +# Distributed API Gateway perf runner (plain / header / jwt scenarios). + +PERF_GATEWAY_TYPE="${PERF_GATEWAY_TYPE:-api-gateway}" +_cli_gateway_type="" +_forward_args=() +while [[ $# -gt 0 ]]; do + case "$1" in + -g) + PERF_GATEWAY_TYPE="$2" + _cli_gateway_type="$2" + shift 2 + ;; + *) + _forward_args+=("$1") + shift + ;; + esac +done +set -- "${_forward_args[@]}" +export PERF_GATEWAY_TYPE + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MANUAL_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +if [[ -f "${SCRIPT_DIR}/env.jmeter" ]]; then + # shellcheck source=env.jmeter + source "${SCRIPT_DIR}/env.jmeter" +else + # shellcheck source=env.example + source "${SCRIPT_DIR}/env.example" +fi +# CLI -g wins over PERF_GATEWAY_TYPE in env.jmeter +if [[ -n "${_cli_gateway_type}" ]]; then + export PERF_GATEWAY_TYPE="${_cli_gateway_type}" +fi +# shellcheck source=lib/gateway-profile.sh +source "${SCRIPT_DIR}/lib/gateway-profile.sh" +load_jmeter_gateway_profile "${SCRIPT_DIR}" +# shellcheck source=../lib/common.sh +source "${MANUAL_DIR}/lib/common.sh" +require_bash4 + +if [[ "${PERF_GATEWAY_TYPE}" != "api-gateway" ]]; then + echo "ERROR: performance-test-scripts supports -g api-gateway only" >&2 + exit 1 +fi +PERF_HOME="${PERF_HOME:-${HOME}/perf-api-gateway-manual}" +export PERF_HOME PERF_ROOT +export PERF_LOCAL=true +export PERF_LOCAL_SCRIPTS="${SCRIPT_DIR}" +export SKIP_JTL_SPLIT="${SKIP_JTL_SPLIT:-true}" +export SKIP_PAYLOAD_GENERATION="${SKIP_PAYLOAD_GENERATION:-true}" +export AI_CHAT_COMPLETION_MIN_RESPONSE_BYTES=279 + +: "${GATEWAY_HOST:?Set GATEWAY_HOST in env.jmeter}" +: "${BACKEND_HOST:?Set BACKEND_HOST in env.jmeter}" + +# api-gateway EC2 perf: docker Netty on :8688; jar backend on host uses :3000 +if [[ "${BACKEND_IMPL:-jar}" == "docker" || "${BACKEND_IMPL:-jar}" == "compose" ]]; then + export BACKEND_PORT="${BACKEND_PORT:-8688}" +else + export BACKEND_PORT="${BACKEND_PORT:-3000}" +fi + +echo "==> ${PERF_GATEWAY_TYPE}: JMeter -> ${GATEWAY_HOST}:8080 (backend ${BACKEND_HOST}:${BACKEND_PORT}, impl=${BACKEND_IMPL:-jar})" + +mkdir -p "$PERF_HOME" +[[ -d "${PERF_HOME}/jmeter" ]] || "${SCRIPT_DIR}/setup-jmeter.sh" + +cd "${PERF_HOME}/jmeter" +common_dir="${PERF_ROOT}/performance-common/distribution/scripts/common" +perf_test_common="${PERF_ROOT}/performance-common/distribution/scripts/jmeter/perf-test-common.sh" + +api_keys_file_name="api-keys.csv" + +_parse_jmeter_users() { + local prev="" u="" + while [[ $# -gt 0 ]]; do + if [[ "$prev" == "-u" ]]; then + echo "$1" + return 0 + fi + prev="$1" + shift + done + return 0 +} + +_jmeter_users="$(_parse_jmeter_users "$@")" +export API_KEY_START="${API_KEY_START:-1}" +# API_KEY_COUNT comes from env.jmeter (pregenerated once on gateway). Not overridden per run. + +function start_netty_backend_local() { + local _heap="$1" + local sleep_time="$2" + local backend_flags="$3" + local response_size="${rsize:-10240}" + # EKS / in-cluster mock: gateway reaches ClusterIP; no SSH to a backend EC2. + if [[ "${BACKEND_IN_CLUSTER:-}" == "1" || -z "${BACKEND_SSH:-}" ]]; then + echo "Skipping remote backend start (BACKEND_IN_CLUSTER=${BACKEND_IN_CLUSTER:-0}, BACKEND_SSH=${BACKEND_SSH:-})" + return 0 + fi + local ssh_opts=(-o StrictHostKeyChecking=no -o ConnectTimeout=10) + [[ -f "${BACKEND_SSH_KEY}" ]] && ssh_opts+=(-i "${BACKEND_SSH_KEY}") + if [[ "${BACKEND_IMPL:-jar}" == "compose" || "${BACKEND_IMPL:-jar}" == "docker" ]]; then + echo "Ensuring container backend is up (impl=${BACKEND_IMPL:-jar}, port=${BACKEND_PORT}) — skip jar reconfigure" + ssh "${ssh_opts[@]}" "${BACKEND_SSH}" \ + "cd ${PERF_ROOT}/ai-gateway-manual/backend && \ + [[ -f env.backend ]] && source env.backend; \ + export BACKEND_IMPL='${BACKEND_IMPL:-jar}' BACKEND_PORT='${BACKEND_PORT}' NETTY_PORT='${BACKEND_PORT}'; \ + ./start-backend.sh" \ + || echo "WARNING: backend container start failed" + return 0 + fi + echo "Reconfiguring backend: delay=${sleep_time}ms response_size=${response_size} impl=${BACKEND_IMPL:-jar} port=${BACKEND_PORT}" + ssh "${ssh_opts[@]}" "${BACKEND_SSH}" \ + "cd ${PERF_ROOT}/ai-gateway-manual/backend && \ + [[ -f env.backend ]] && source env.backend; \ + export BACKEND_IMPL='${BACKEND_IMPL:-jar}' BACKEND_PORT='${BACKEND_PORT}' NETTY_PORT='${BACKEND_PORT}' \ + MOCK_BACKEND_DELAY='${sleep_time}' MOCK_RESPONSE_SIZE='${response_size}' NETTY_HEAP='${NETTY_HEAP:-4g}'; \ + ./reconfigure-backend.sh -d ${sleep_time} -r ${response_size}" \ + || echo "WARNING: backend SSH reconfigure failed — ensure backend/reconfigure-backend.sh is on backend EC2" +} +export -f start_netty_backend_local + +. "${common_dir}/common.sh" +. "${perf_test_common}" + +# High-TPS runs: skip per-cell GC logs (saves GiB when disk is tight). Set SKIP_JMETER_GC_LOG=0 to enable. +if [[ "${SKIP_JMETER_GC_LOG:-true}" == "true" ]]; then + function jmeter_gc_log_args() { :; } +fi + +function collect_server_metrics() { return 0; } +function write_server_metrics() { return 0; } +function download_file() { return 0; } +export -f collect_server_metrics write_server_metrics download_file + +# shellcheck source=generate-jwt-tokens.sh +source "${SCRIPT_DIR}/generate-jwt-tokens.sh" + +function initialize() { + if [[ "${JWT_SINGLE_TOKEN:-}" == "1" ]]; then + export JWT_TOKEN_COUNT=1 + generate_jwt_tokens || exit 1 + elif [[ ! -s "${HOME}/jwt-tokens.csv" || "${JWT_REGENERATE:-}" == "1" ]]; then + generate_jwt_tokens || exit 1 + fi + + [[ "${jmeter_servers:-1}" -le 1 ]] && return 0 + + local key_files=() + local f + for f in "${HOME}"/jwt-tokens*.csv; do + [[ -f "$f" ]] && key_files+=("$f") + done + if [[ ${#key_files[@]} -eq 0 ]]; then + echo "ERROR: No jwt-tokens*.csv in ${HOME}. Set JWT_OAUTH_* or run generate-jwt-tokens.sh" >&2 + exit 1 + fi + + local host + for host in "${jmeter_ssh_hosts[@]}"; do + echo "Copying JWT tokens to ${host}..." + scp -o StrictHostKeyChecking=no "${key_files[@]}" "${host}:${HOME}/" + done +} +export -f initialize + +script_dir="$(pwd -P)" + +# shellcheck source=gateway-scenarios.sh +source "${MANUAL_DIR}/jmeter/gateway-scenarios.sh" + +function before_execute_test_scenario() { + local service_host="${GATEWAY_HOST}" + local response_size=${rsize:-10240} + local keys_file="${HOME}/${api_keys_file_name}" + local request_body_mode="" + local request_body="" + + if [[ ${scenario[host_type]} == "backend" ]]; then + service_host="${BACKEND_HOST}" + elif [[ -n "${scenario_api_key_file[${scenario[name]}]:-}" ]]; then + keys_file="${HOME}/${scenario_api_key_file[${scenario[name]}]}" + [[ -f "$keys_file" ]] || keys_file="${PERF_HOME}/${scenario_api_key_file[${scenario[name]}]}" + [[ -f "$keys_file" ]] || { + echo "Missing ${keys_file}. Regenerate JWT tokens (JWT_REGENERATE=1)." >&2 + exit 1 + } + local key_lines + key_lines=$(wc -l <"$keys_file" | tr -d ' ') + if [[ -n "${users:-}" && "${users}" =~ ^[0-9]+$ && "${key_lines}" -lt "${users}" ]]; then + echo "WARNING: ${keys_file} has ${key_lines} keys but test uses ${users} users (keys will recycle)." + echo " Pre-generate more: ./pregenerate-api-keys.sh ${users} (one-time on gateway + fetch)" + fi + if [[ "${scenario_auth_header[${scenario[name]}]:-}" == "Authorization" && ! -s "$keys_file" ]]; then + echo "ERROR: ${keys_file} is empty. Regenerate with JWT_REGENERATE=1 ./run-scenario.sh ..." + exit 1 + fi + fi + + jmeter_params+=("host=$service_host" "port=${scenario[port]}" "path=${scenario[path]}") + if [[ -n "${GATEWAY_DUAL_PORTS:-}" ]]; then + jmeter_params+=("dualGateway=1") + jmeter_params+=("dualGatewayHosts=$(IFS=,; echo "${jmeter_hosts[*]}")") + jmeter_params+=("dualGatewayPorts=${GATEWAY_DUAL_PORTS}") + fi + jmeter_params+=("protocol=${scenario[protocol]}") + jmeter_params+=("resourceSuffixes=${scenario[resource_suffixes]:-}") + jmeter_params+=("method=${scenario[method]:-POST}") + + request_body_mode="${scenario[request_body_mode]:-}" + if [[ "${scenario[method]:-GET}" == "POST" ]]; then + request_body="${scenario[request_body]:-{\"messages\":[{\"role\":\"user\",\"content\":\"perf-post\"}]}}" + jmeter_params+=("requestBody=${request_body}") + fi + if [[ ${scenario[host_type]} == "gateway" ]]; then + jmeter_params+=("response_size=${response_size}") + if [[ -n "${scenario_api_key_file[${scenario[name]}]:-}" ]]; then + jmeter_params+=("tokens=${keys_file}") + jmeter_params+=("authHeaderName=${scenario_auth_header[${scenario[name]}]:-X-API-Key}") + fi + fi + + if [[ -n "${rsize}" && "${rsize}" -lt "${AI_CHAT_COMPLETION_MIN_RESPONSE_BYTES}" ]]; then + echo "==> Response ${response_size}B < ${AI_CHAT_COMPLETION_MIN_RESPONSE_BYTES}B — Netty echo mode (tiny GET body, weather-like)" + scenario[backend_flags]="--port ${BACKEND_PORT}" + else + scenario[backend_flags]="--port ${BACKEND_PORT} --ai-chat-completion-response --ai-chat-completion-response-size ${response_size}" + fi +} + +function after_execute_test_scenario() { return 0; } + +print_load_plan() { + local users="" servers=1 + while [[ $# -gt 0 ]]; do + case "$1" in + -u) users="$2"; shift 2 ;; + -n) servers="$2"; shift 2 ;; + *) shift ;; + esac + done + [[ -z "$users" || "$servers" -le 1 ]] && return 0 + if ! [[ "$users" =~ ^[0-9]+$ && "$servers" =~ ^[0-9]+$ ]]; then + return 0 + fi + local per=$((users / servers)) rem=$((users % servers)) + if [[ "$rem" -ne 0 ]]; then + echo "ERROR: -u ${users} must divide evenly by -n ${servers}" >&2 + exit 1 + fi + echo "" + echo "==> Gateway type: ${PERF_GATEWAY_TYPE}" + if [[ -n "${GATEWAY_DUAL_PORTS:-}" ]]; then + IFS=',' read -ra _dual_ports <<< "${GATEWAY_DUAL_PORTS}" + local n_ports=${#_dual_ports[@]} + local ix _port _base _end + if [[ "$servers" -ge "$n_ports" ]]; then + for ix in $(seq 0 $((servers - 1))); do + _port="${_dual_ports[$ix]:-${_dual_ports[0]}}" + echo "==> Engine ${ix}: ${GATEWAY_HOST}:${_port} (${per} threads)" + done + echo "==> Scaled gateway: ${users} TOTAL users => ${per} threads × ${servers} engines (one port per engine)" + else + local _ppe=$(( (n_ports + servers - 1) / servers )) + for ix in $(seq 0 $((servers - 1))); do + _base=$((ix * _ppe)) + _end=$((_base + _ppe - 1)) + [[ "$_end" -ge "$n_ports" ]] && _end=$((n_ports - 1)) + if [[ "$_base" -lt "$n_ports" ]]; then + echo "==> Engine ${ix}: ${GATEWAY_HOST}:${_dual_ports[_base]}–${_dual_ports[_end]} (${per} threads, round-robin)" + fi + done + echo "==> Scaled gateway: ${users} TOTAL users => ${per} threads × ${servers} engines (${n_ports} ports total)" + fi + echo " Single JMeter client on jmeter1; combined summary on this host" + else + echo "==> Gateway target: ${GATEWAY_HOST}:8080" + echo "==> Load plan: ${users} TOTAL users => ${per} threads × ${servers} engines" + fi + echo " Gateway sees HTTP from ${servers} JMeter host IPs (not ${users}×${servers})" + if [[ "${PERF_GATEWAY_TYPE}" == "api-gateway" && "${users}" -gt 1000 ]]; then + echo "WARNING: api-gateway plain baseline is known to plateau around ~20-25k TPS at 1000 users." + echo " Higher users can trigger ext_proc overflow (HTTP 500 / JMeter code 1000)." + fi + echo "" +} + +print_load_plan "$@" + +prune_old_result_backups() { + local keep="${1:-2}" + local d + for d in $(ls -dt results.backup-* 2>/dev/null | tail -n +$((keep + 1))); do + echo "==> Removing old backup: ${d}" + rm -rf "${d}" + done + for d in $(ls -t results.zip.backup-* 2>/dev/null | tail -n +$((keep + 1))); do + echo "==> Removing old backup: ${d}" + rm -f "${d}" + done +} + +preflight_jmeter_disk() { + local avail_kb min_free_kb=2097152 + avail_kb=$(df "${PERF_HOME}" 2>/dev/null | awk 'NR==2 {print $4}') + if [[ -z "${avail_kb}" ]]; then + avail_kb=$(df / | awk 'NR==2 {print $4}') + fi + echo "==> Disk free: $(( avail_kb / 1024 )) MiB on $(df / | awk 'NR==2 {print $1}')" + if [[ -d "${PERF_HOME}/jmeter/results" ]]; then + echo "==> Current results/: $(du -sh "${PERF_HOME}/jmeter/results" 2>/dev/null | awk '{print $1}')" + fi + prune_old_result_backups 1 + avail_kb=$(df / | awk 'NR==2 {print $4}') + if [[ "${avail_kb}" -lt "${min_free_kb}" ]]; then + echo "WARNING: < 2 GiB free — deleting results/ and old backups (no archive)" + rm -rf "${PERF_HOME}/jmeter/results" "${PERF_HOME}/jmeter/results.backup-"* 2>/dev/null || true + rm -f "${PERF_HOME}/jmeter/results.zip" "${PERF_HOME}/jmeter/results.zip.backup-"* 2>/dev/null || true + avail_kb=$(df / | awk 'NR==2 {print $4}') + fi + if [[ "${avail_kb}" -lt 524288 ]]; then + echo "ERROR: < 512 MiB free on JMeter-1. Run: cd ${SCRIPT_DIR} && ./recover-stuck-test.sh --aggressive" >&2 + exit 1 + fi + if [[ "${avail_kb}" -lt 1048576 ]]; then + echo "WARNING: < 1 GiB free after cleanup — long runs may fail; consider ./recover-stuck-test.sh --aggressive or expand EBS" + fi +} + +backup_previous_results() { + local stamp + stamp=$(date +%Y%m%d-%H%M%S) + local avail_kb + avail_kb=$(df / | awk 'NR==2 {print $4}') + if [[ -d results ]]; then + if [[ "${avail_kb}" -lt 2097152 ]]; then + echo "==> Disk low (< 2 GiB) — deleting results/ instead of backup" + rm -rf results + else + echo "Backing up existing results/ -> results.backup-${stamp}" + mv results "results.backup-${stamp}" + fi + fi + if [[ -f results.zip ]]; then + if [[ "${avail_kb}" -lt 2097152 ]]; then + rm -f results.zip + else + echo "Backing up existing results.zip -> results.zip.backup-${stamp}" + mv results.zip "results.zip.backup-${stamp}" + fi + fi + rm -f test-duration.json test-metadata.json +} + +preflight_jmeter_disk +backup_previous_results + +test_scenarios diff --git a/gateway/perf/performance-test-scripts/jmeter/setup-jmeter.sh b/gateway/perf/performance-test-scripts/jmeter/setup-jmeter.sh new file mode 100755 index 000000000..9e2badee2 --- /dev/null +++ b/gateway/perf/performance-test-scripts/jmeter/setup-jmeter.sh @@ -0,0 +1,56 @@ +#!/bin/bash -e +# JMeter EC2 setup: Java 17, JMeter tarball, PERF_HOME symlinks, payloads. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=env.example +source "${SCRIPT_DIR}/env.example" +# shellcheck source=../lib/common.sh +source "${MANUAL_DIR}/lib/common.sh" + +install_java17_and_maven +install_linux_packages unzip bc jq + +if ! command -v jmeter >/dev/null; then + JMETER_TGZ="${JMETER_TGZ:-${HOME}/apache-jmeter-5.6.3.tgz}" + [[ -f "$JMETER_TGZ" ]] || { + echo "Download JMeter 5.6.3 to ${JMETER_TGZ} or install jmeter on PATH" >&2 + exit 1 + } + tar -xzf "$JMETER_TGZ" -C "$HOME" + JMETER_HOME="${HOME}/apache-jmeter-5.6.3" + cp "${PERF_ROOT}/performance-common/distribution/scripts/jmeter/user.properties" \ + "${JMETER_HOME}/bin/user.properties" + grep -q 'apache-jmeter-5.6.3/bin' "${HOME}/.bashrc" 2>/dev/null || \ + echo 'export PATH=${HOME}/apache-jmeter-5.6.3/bin:$PATH' >>"${HOME}/.bashrc" + export PATH="${HOME}/apache-jmeter-5.6.3/bin:$PATH" +fi + +# Always refresh user.properties from perf repo so log tuning and RMI settings stay in sync. +JMETER_HOME="${JMETER_HOME:-$(dirname "$(dirname "$(command -v jmeter)")")}" +if [[ -n "${JMETER_HOME}" && -f "${JMETER_HOME}/bin/user.properties" ]]; then + cp "${PERF_ROOT}/performance-common/distribution/scripts/jmeter/user.properties" \ + "${JMETER_HOME}/bin/user.properties" +fi + +COMMON_SCRIPTS="${PERF_ROOT}/performance-common/distribution/scripts" +mkdir -p "$PERF_HOME" +ln -sfn "${MANUAL_DIR}/jmeter" "${PERF_HOME}/jmeter" +ln -sfn "${COMMON_SCRIPTS}/payloads" "${PERF_HOME}/payloads" +ln -sfn "${COMMON_SCRIPTS}/jtl-splitter" "${PERF_HOME}/jtl-splitter" +ln -sfn "${COMMON_SCRIPTS}/common" "${PERF_HOME}/common" +ln -sfn "${PERF_HOME}/jmeter" "${HOME}/jmeter" +ln -sfn "${PERF_HOME}/payloads" "${HOME}/payloads" +ln -sfn "${PERF_HOME}/jtl-splitter" "${HOME}/jtl-splitter" + +if [[ "${SKIP_PAYLOAD_GENERATION:-true}" != "true" ]]; then + ( + cd "$PERF_HOME" + ./payloads/generate-payloads.sh -a -s 1024 -s 10240 + ) + cp "${PERF_HOME}"/ai_*.json "${HOME}/" 2>/dev/null || true +fi + +echo "JMeter setup OK. PERF_HOME=${PERF_HOME}" +echo "Next: source env.jmeter && ./run-scenario.sh -g api-gateway ..." diff --git a/gateway/perf/performance-test-scripts/lib/common.sh b/gateway/perf/performance-test-scripts/lib/common.sh new file mode 100755 index 000000000..c7bb05691 --- /dev/null +++ b/gateway/perf/performance-test-scripts/lib/common.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# Shared helpers for performance-test-scripts. +# Usage: source "${MANUAL_DIR}/lib/common.sh" (after env.example) + +# Re-exec with bash 4+ (macOS default bash is 3.2). +require_bash4() { + if ((BASH_VERSINFO[0] < 4)); then + for _bash in /opt/homebrew/bin/bash /usr/local/bin/bash /usr/bin/bash; do + if [[ -x "$_bash" ]] && "$_bash" -c '((BASH_VERSINFO[0] >= 4))' 2>/dev/null; then + exec "$_bash" "$0" "$@" + fi + done + echo "Error: bash 4+ required (install: brew install bash)" >&2 + exit 1 + fi +} + +# True on Ubuntu EC2 (apt available). +is_ubuntu() { + [[ "$(uname -s)" == "Linux" ]] && command -v apt-get >/dev/null 2>&1 +} + +# True on Amazon Linux EC2 (ec2-user, yum/dnf). +is_amazon_linux() { + [[ -f /etc/os-release ]] && grep -qiE 'amazon linux' /etc/os-release +} + +linux_pkg_manager() { + if is_ubuntu; then + echo apt + elif command -v dnf >/dev/null 2>&1; then + echo dnf + elif command -v yum >/dev/null 2>&1; then + echo yum + else + echo "" + fi +} + +# Install packages on Ubuntu or Amazon Linux EC2; no-op on macOS. +install_linux_packages() { + local pm + pm="$(linux_pkg_manager)" + [[ -n "$pm" ]] || return 0 + if [[ "$pm" == apt ]]; then + sudo apt-get update + sudo apt-get install -y "$@" + else + sudo "$pm" install -y "$@" + fi +} + +# Backward-compatible alias. +install_ubuntu_packages() { + install_linux_packages "$@" +} + +# Java 17 + Maven for backend/jmeter setup on EC2. +install_java17_and_maven() { + if is_ubuntu; then + install_linux_packages openjdk-17-jdk maven curl netcat-openbsd + elif is_amazon_linux; then + # curl-minimal is preinstalled on AL2023; installing curl often conflicts. + install_linux_packages java-17-amazon-corretto-devel maven + install_linux_packages nmap-ncat 2>/dev/null || true + else + echo "Install Java 17 and Maven manually, then re-run setup." >&2 + return 1 + fi + command -v java >/dev/null || { echo "java not on PATH after install" >&2; return 1; } + java -version +} + +# Path to shaded Netty mock JAR after Maven build. +resolve_netty_jar() { + find "${PERF_ROOT}/performance-common/components/netty-http-echo-service/target" \ + -maxdepth 1 -name 'netty-http-echo-service*.jar' ! -name 'original-*' 2>/dev/null | head -1 +} + +# Run docker compose (plugin) or legacy docker-compose. +docker_compose() { + if docker compose version >/dev/null 2>&1; then + docker compose "$@" + elif command -v docker-compose >/dev/null 2>&1; then + docker-compose "$@" + else + echo "Docker Compose not found. On Amazon Linux: sudo dnf install -y docker-compose-plugin" >&2 + exit 1 + fi +} + +# Install docker compose v2 plugin (AL2023 has no docker-compose-plugin RPM). +install_docker_compose_plugin() { + if docker compose version >/dev/null 2>&1; then + return 0 + fi + + local arch plugin_dir plugin_path url + arch="$(uname -m)" + for plugin_dir in /usr/libexec/docker/cli-plugins /usr/local/lib/docker/cli-plugins; do + sudo mkdir -p "$plugin_dir" + plugin_path="${plugin_dir}/docker-compose" + url="https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${arch}" + echo "Installing docker compose plugin -> ${plugin_path}" >&2 + if sudo curl -fsSL "$url" -o "$plugin_path"; then + sudo chmod +x "$plugin_path" + if docker compose version >/dev/null 2>&1; then + docker compose version + return 0 + fi + fi + done + + # Fallback: standalone docker-compose binary + if ! command -v docker-compose >/dev/null 2>&1; then + sudo curl -fsSL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${arch}" \ + -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + fi + docker-compose version >/dev/null 2>&1 || docker compose version >/dev/null 2>&1 || { + echo "Failed to install docker compose" >&2 + return 1 + } +} + +# Docker + Compose on EC2 gateway host. +install_docker_ec2() { + if is_amazon_linux; then + install_linux_packages docker jq + sudo systemctl enable --now docker + install_docker_compose_plugin + else + install_linux_packages docker.io docker-compose-plugin jq curl + install_docker_compose_plugin 2>/dev/null || true + fi + sudo usermod -aG docker "${USER:-$(whoami)}" 2>/dev/null || true +} + +# Build Netty mock JAR if missing. +ensure_netty_jar() { + local jar + jar="$(resolve_netty_jar)" + if [[ -n "$jar" && -f "$jar" ]]; then + echo "$jar" + return 0 + fi + echo "Building Netty mock JAR..." >&2 + (cd "${PERF_ROOT}/performance-common" && mvn -q package -DskipTests -Dfindbugs.skip=true \ + -pl components/netty-http-echo-service -am) + resolve_netty_jar +}