diff --git a/.env.example b/.env.example
index d0c09f0..f7497b2 100644
--- a/.env.example
+++ b/.env.example
@@ -12,6 +12,7 @@
BASE_URL=https://dev.parakh.civicdataspace.in
ENVIRONMENT=development # development | staging | production
GRAPHQL_URL=https://dev.api.parakh.civicdataspace.in/graphql/
+CDS_URL=https://dev.civicdataspace.in
# ── Keycloak SSO ──────────────────────────────────────────────────────────────
# Leave blank if the app and auth share the same origin.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8b86028..6a98615 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -22,6 +22,7 @@ Run the suite locally before opening a PR:
ruff check . # lint must be green
pytest --collect-only # all tests must collect
pytest -m smoke -v # fast sanity subset
+pytest -m "not load" # normal full suite without stress scenarios
```
---
@@ -51,6 +52,9 @@ and route through the page object — not to inline it.
`@pytest.mark.auth`.
5. For tests that mutate platform state, use the `regression_write` marker —
they auto-skip unless `SANDBOX_ORG_SLUG` is set.
+6. Use `security` for security-focused HTTP/session checks and `load` for
+ explicit stress or concurrency scenarios. Load tests must target a sandbox,
+ never production.
### Selector conventions
@@ -76,8 +80,10 @@ This tolerates minor UI changes without churning the test suite.
| `e2e` | Browser UI end-to-end tests |
| `accessibility` | WCAG / axe-core tests |
| `visual` | Screenshot regression tests |
-| `api` | HTTP-layer tests (no browser) |
-| `performance` | Load/timing metric tests |
+| `api` | HTTP/API tests; some security checks also inspect a browser session |
+| `performance` | Page timing and resource-budget tests |
+| `load` | Explicit load/stress tests; run against dev/staging sandbox |
+| `security` | Cookie, auth, IDOR, CORS, error-handling, and sanitisation checks |
| `smoke` | Fast sanity subset (PR checks) |
| `regression` | Full regression suite |
| `regression_write` | Tests that mutate platform state — sandbox org only |
diff --git a/README.md b/README.md
index 679ad46..27967f5 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,10 @@ Production-ready Python + Playwright test automation framework for the **[Parakh
| E2E UI | Playwright + pytest | Auth, homepage, navigation, feature tabs, AI models, evaluations list + detail, New Evaluation wizard (draft, automated & manual modes, cancel paths), evaluators management, evaluator role, auditor flows, prompt libraries, mobile viewport |
| Accessibility | axe-playwright-python (WCAG 2.1 AA) | Axe scans, alt text, ARIA, keyboard, skip links, social-icon labels |
| Visual Regression | Pillow pixel-diff | Desktop / tablet / mobile viewports |
-| API / HTTP | requests | Status codes, headers, response time, GraphQL contracts |
-| Performance | CDP + Navigation Timing API | Load time, TTFB, LCP, mobile 3G, authenticated-route budgets |
+| API / HTTP | requests | Status codes, headers, response time, public and authenticated GraphQL contracts |
+| Security | requests + Playwright | Cookie flags, browser storage, GraphQL auth/error handling, IDOR, CORS, and input sanitisation |
+| Performance | CDP + Navigation Timing API | Load time, TTFB, LCP, mobile 3G, authenticated-route and resource-size budgets |
+| Load | requests + concurrent workers | Concurrent GraphQL reads, draft creation, authentication, pagination, and UI degradation |
---
@@ -61,8 +63,9 @@ pytest
pytest tests/e2e/ -m e2e # E2E browser tests
pytest tests/accessibility/ -m accessibility # WCAG / axe-core
pytest tests/visual/ -m visual # screenshot regression
-pytest tests/api/ -m api # HTTP-layer (no browser)
+pytest tests/api/ -m api # HTTP/API + security checks
pytest tests/performance/ -m performance # load & timing metrics
+pytest tests/load/ -m load # explicit load/stress suite
```
### By marker
@@ -71,8 +74,15 @@ pytest -m smoke -v # fast CI sanity subset
pytest -m auth -v # authenticated tests only (needs TEST_EMAIL_1)
pytest -m mobile -v # mobile-viewport tests (390px)
pytest -m regression_write # write-side tests (needs SANDBOX_ORG_SLUG)
+pytest -m security -v # security checks; cookie checks also need TEST_EMAIL_1
+pytest -m load -v # load tests; mutation scenarios need SANDBOX_ORG_SLUG
```
+Load tests are not intended for routine PR validation. Run them explicitly
+against the dev/staging sandbox, not production. The complete unfiltered
+`pytest` command includes them; use `pytest -m "not load"` for a normal local
+full-suite run without stress scenarios.
+
### Single file / test
```bash
pytest tests/e2e/test_homepage.py -v
@@ -157,8 +167,10 @@ pytest tests/visual/test_visual_regression.py::TestHomepageVisual::test_homepage
| `@pytest.mark.e2e` | Browser UI end-to-end tests |
| `@pytest.mark.accessibility` | WCAG / axe-core tests |
| `@pytest.mark.visual` | Screenshot regression tests |
-| `@pytest.mark.api` | HTTP-layer tests (no browser) |
-| `@pytest.mark.performance` | Load/timing metric tests |
+| `@pytest.mark.api` | HTTP/API tests; some security checks also inspect a browser session |
+| `@pytest.mark.performance` | Page timing and resource-budget tests |
+| `@pytest.mark.load` | Explicit load/stress tests, including concurrent reads and mutations |
+| `@pytest.mark.security` | Security-focused HTTP and browser-session checks |
| `@pytest.mark.mobile` | Mobile-viewport tests (390×844) |
| `@pytest.mark.smoke` | Fast sanity subset for PR checks |
| `@pytest.mark.regression` | Full regression suite |
@@ -267,8 +279,11 @@ After each run, reports are written to `reports/`:
| `reports/e2e_summary.md` | Merged Markdown posted to GitHub Step Summary |
| `reports/a11y_report.html` | Accessibility HTML report |
| `reports/accessibility_report.json` | Structured axe violations JSON |
+| `reports/accessibility_*_report.json` | Authenticated-route axe results |
| `reports/performance_metrics.json` | Public-page timing metrics |
| `reports/performance_metrics_auth.json` | Authenticated-route timing metrics |
+| `reports/load_metrics.json` | Mutation, authentication, wizard, and UI load metrics |
+| `reports/load_metrics_graphql.json` | Concurrent GraphQL read and pagination metrics |
| `screenshots/FAIL_*.png` | Failure screenshots |
| `screenshots/DIFF_*.png` | Visual regression diff images |
@@ -334,14 +349,17 @@ ParakhAI_test/
│ ├── ci.yml # PR + push pipeline (lint → parallel suites → summary)
│ └── scheduled.yml # Nightly regression (02:00 UTC)
├── tests/
-│ ├── e2e/ # Browser UI tests (~310 tests)
+│ ├── e2e/ # Browser UI tests (377 collected tests)
│ │ ├── test_auth.py
│ │ ├── test_homepage.py
+│ │ ├── test_smoke_critical.py
+│ │ ├── test_functional.py
│ │ ├── test_navigation.py
│ │ ├── test_feature_tabs.py
│ │ ├── test_models.py
│ │ ├── test_evaluations.py
│ │ ├── test_evaluation_detail.py
+│ │ ├── test_regression_new_features.py
│ │ ├── test_new_evaluation_smoke.py
│ │ ├── test_new_evaluation_regression.py
│ │ ├── test_new_evaluation_full_flow.py
@@ -360,12 +378,21 @@ ParakhAI_test/
│ │ ├── test_prompt_libraries.py
│ │ ├── test_user_flows.py
│ │ └── test_mobile.py
-│ ├── accessibility/ # WCAG / axe-core tests
+│ ├── accessibility/ # Public + authenticated WCAG / axe-core tests
+│ │ ├── test_accessibility.py
+│ │ └── test_accessibility_auth.py
│ ├── visual/ # Screenshot regression
│ ├── api/ # HTTP-layer tests
+│ │ ├── test_api.py
│ │ ├── test_graphql.py
-│ │ └── test_graphql_authenticated.py
-│ ├── performance/ # Load & timing tests
+│ │ ├── test_graphql_authenticated.py
+│ │ └── test_security.py
+│ ├── performance/ # Public/auth route timing + resource budgets
+│ │ ├── test_performance.py
+│ │ └── test_performance_auth.py
+│ ├── load/ # Concurrent API, auth, mutation, and UI load tests
+│ │ ├── test_load.py
+│ │ └── test_load_graphql.py
│ ├── data/
│ │ └── test_data.py # GraphQL queries/mutations + sandbox constants
│ └── conftest.py # All shared fixtures
@@ -374,12 +401,12 @@ ParakhAI_test/
│ ├── home_page.py
│ ├── login_page.py
│ ├── dashboard_page.py
-│ ├── ai_maker_dashboard_page.py
+│ ├── ai_maker_page.py
│ ├── models_page.py
│ ├── evaluations_page.py
│ ├── evaluation_detail_page.py
│ ├── new_evaluation_page.py
-│ ├── evaluation_workspace_page.py
+│ ├── workspace_page.py
│ ├── evaluators_page.py
│ ├── evaluator_role_page.py
│ ├── auditor_model_detail_page.py
diff --git a/docs/app_bugs.md b/docs/app_bugs.md
index b801118..ecfe75e 100644
--- a/docs/app_bugs.md
+++ b/docs/app_bugs.md
@@ -11,10 +11,15 @@ Format: append-only. When a bug is fixed in the app, mark `status: fixed` and th
| 3 | dev-env stability | Dev frontend (`https://dev.parakh.civicdataspace.in`) intermittently returns genuine HTTP 4xx responses mid-run, and Keycloak login form sometimes fails to render. **Rescoped 2026-05-23:** the majority of Phase 1 skips (9/9 on 2026-05-08) were caused by `networkidle` misuse on authenticated routes that have background polling — those routes never reach `networkidle`, so the login fixture timed out and bailed. Those tests are now fixed (switched to `load` + `wait_for_app_ready()`). The remaining genuine infra issue is: under load or on VPN-gated IP ranges, the dev nginx occasionally returns 503/504, causing the `home_page.py` skip guard ("check VPN / IP allowlist") to fire for any test in the run. This is an infra/ops issue, not a frontend bug. | infra | Run `pytest -m smoke` from an office IP (not VPN); mid-run some requests return 503. With pytest-xdist some workers get a working login, others don't — visible as partial skip pattern. Phase 1 API run 2026-05-08 produced 9 skips: all now attributed to networkidle misuse (fixed), not 4xx. | All `e2e` + `smoke` tests transitively (skip guard in `home_page.py`); `tests/api/test_graphql_authenticated.py` (skip guard in `_do_login`) | 2026-05-07 |
| 4 | a11y / homepage | Homepage `/` has no skip-to-main-content link. WCAG 2.4.1 (Bypass Blocks, Level A). Keyboard users cannot bypass the navigation. Frontend should add `Skip to main content` as the first child of `
`, and ensure the `` element has `id="main-content"`. | open | Load `https://dev.parakh.civicdataspace.in/`. Run `document.querySelectorAll("a[href='#main-content'], a[href='#main']").length` → returns 0. Confirmed via Playwright MCP 2026-05-08. | `tests/accessibility/test_accessibility.py::TestStructure::test_skip_link_exists` | 2026-05-08 |
| 5 | a11y / keycloak (third-party) | Keycloak login page at `opub-kc.civicdatalab.in/auth/realms/DataSpace/...` has 2 serious axe violations: `color-contrast` (WCAG 1.4.3) on form labels/placeholders, and `link-name` (WCAG 4.1.2 / 2.4.4) on 4 social-media icon links in the footer (icons-only ``s with no `aria-label` or visible text). Owned by the Keycloak team — not the Parakh frontend. File upstream with whoever maintains the `opub-kc` realm theme. | open (third-party) | Open the Keycloak login URL directly. Run `Array.from(document.querySelectorAll('a')).filter(a => !(a.getAttribute('aria-label') \|\| a.textContent\|\|'').trim()).length` → 4 (the GitHub/LinkedIn/Twitter/Facebook icons). Color-contrast confirmed via axe-core scan in test. Reproduced via Playwright MCP 2026-05-08. | `tests/accessibility/test_accessibility.py::TestAxeScans::test_login_page_has_no_critical_axe_violations` | 2026-05-08 |
-| 6 | navigation / dashboard | List click-throughs across the AI Maker dashboard do not navigate. Four confirmed surfaces: (a) View buttons on AI Models list render `aria-disabled="true"` and their `onClick` is a no-op; (b) clicking a row in the Evaluations list (incl. COMPLETED rows) does not open the detail page; (c) clicking an evaluation link in a model's Past Evaluations table does not open the eval detail; (d) the "Back to List" button on the Evaluation Detail page click-fires but the URL stays at `/evaluations/{id}` (added 2026-05-21 — strict-mode locator drift was masking this; once the click succeeded the no-op surfaced). Direct navigation to `/ai-models/{id}` and `/evaluations/{id}` works — only the from-the-list / back-from-detail click-throughs are broken. Suggests a shared opub-ui Button/Table interaction regression rather than separate bugs. | open | Log in → AI Maker → CivicDataLab. (a) Click any model's "View" → URL unchanged. (b) Evaluations list → click any COMPLETED row → URL unchanged. (c) Open any model's detail → Past Evaluations → click eval name → URL unchanged. (d) Open any evaluation detail directly via URL → click "Back to List" → URL unchanged. Confirmed via test reruns 2026-05-18 and 2026-05-21. | `tests/e2e/test_models.py::TestModelDetailPage::test_clicking_model_card_from_list_navigates_to_detail`; `tests/e2e/test_user_flows.py::TestFlow02_DashboardToModelDetail::test_past_evaluation_link_navigates_to_detail`; `tests/e2e/test_user_flows.py::TestFlow04_EvaluationDetailAndReport::test_navigate_to_completed_evaluation_detail`; `tests/e2e/test_user_flows.py::TestFlow02_DashboardToModelDetail::test_navigate_from_dashboard_to_model_detail`; `tests/e2e/test_evaluation_detail.py::TestEvaluationDetailBackNavigation::test_back_button_removes_eval_id_from_url` | 2026-05-18 |
+| 6 | navigation / dashboard | List click-throughs across the AI Maker dashboard do not navigate. Four confirmed surfaces: (a) View buttons on AI Models list render `aria-disabled="true"` and their `onClick` is a no-op; (b) clicking a row in the Evaluations list (incl. COMPLETED rows) does not open the detail page; (c) clicking an evaluation link in a model's Past Evaluations table does not open the eval detail; (d) the "Back to List" button on the Evaluation Detail page click-fires but the URL stays at `/evaluations/{id}` (added 2026-05-21 — strict-mode locator drift was masking this; once the click succeeded the no-op surfaced). Direct navigation to `/ai-models/{id}` and `/evaluations/{id}` works — only the from-the-list / back-from-detail click-throughs are broken. Suggests a shared opub-ui Button/Table interaction regression rather than separate bugs. | fixed 2026-05-23 | Log in → AI Maker → CivicDataLab. (a) Click any model's "View" → URL unchanged. (b) Evaluations list → click any COMPLETED row → URL unchanged. (c) Open any model's detail → Past Evaluations → click eval name → URL unchanged. (d) Open any evaluation detail directly via URL → click "Back to List" → URL unchanged. Confirmed via test reruns 2026-05-18 and 2026-05-21. | `tests/e2e/test_models.py::TestModelDetailPage::test_clicking_model_card_from_list_navigates_to_detail`; `tests/e2e/test_user_flows.py::TestFlow02_DashboardToModelDetail::test_past_evaluation_link_navigates_to_detail`; `tests/e2e/test_user_flows.py::TestFlow04_EvaluationDetailAndReport::test_navigate_to_completed_evaluation_detail`; `tests/e2e/test_user_flows.py::TestFlow02_DashboardToModelDetail::test_navigate_from_dashboard_to_model_detail`; `tests/e2e/test_evaluation_detail.py::TestEvaluationDetailBackNavigation::test_back_button_removes_eval_id_from_url` | 2026-05-18 |
| 7 | evaluation detail | The Evaluation Detail page (`/dashboard/ai-maker/{org}/evaluations/{eval_id}`) renders every COMPLETED evaluation as if it were a DRAFT. Status badge shows `DRAFT` instead of `COMPLETED`, mode badge is empty, the `Completed`, `Modules`, `Evaluation Scope`, and `Evaluation Type` overview fields all show `--`, and the Summary / Pass Rate / Risk Levels / Module-wise Results / Sample Issues sections are never rendered. The Download Report button is greyed out. Confirmed against two distinct COMPLETED audits whose GraphQL `audit(id)` payload reports `status: "COMPLETED"`, `progressPercentage: 100`, `totalTests > 0`, and a non-null `completedAt` — so the data exists on the API. The frontend likely fetches the wrong query or fails to map the response. **Update 2026-05-22:** Re-verified against eval 595 (newest COMPLETED on dev): status badge correctly shows `COMPLETED`, Evaluation Summary + TOTAL PASS RATE + per-module section all render. Test Cases tab, Results tab, and Sample Issues accordion are still broken (tabs never become visible — same locator times out). Bug is partial; the badge / Summary / Pass Rate / Risk sub-defects appear fixed since 2026-05-20. Frontend should re-scope this row to "Test Cases + Results tabs + Sample Issues fail to render on COMPLETED evals" once a code search confirms the remaining defect. | open (partial — see 2026-05-22 update) | Log in as TEST_EMAIL_1 → AI Maker → CivicDataLab → Evaluations. In a second shell, run `python -c "from scripts._api_client import get_access_token,graphql; t=get_access_token(); print(graphql(t,'1','query{audits(status:\"COMPLETED\"){id name}}'))"` to find a COMPLETED eval id (e.g. 469 "Agriculture Queries in Hindi", 464 "Healthcare prompts"). Navigate directly to `https://dev.parakh.civicdataspace.in/dashboard/ai-maker/1/evaluations/469`. Observe: status badge says `DRAFT`, all overview fields after `Evaluation ID` show `--`, no results sections below the overview. Confirmed via Playwright runs 2026-05-20. | `tests/e2e/test_evaluations.py::TestEvaluationDetail::*` (all assertions about COMPLETED-specific content); `tests/e2e/test_evaluations.py::TestEvaluationsListPage::test_completed_evaluations_are_listed`, `test_status_badge_colors_are_distinct`, `test_clicking_completed_evaluation_navigates_to_detail` (list-side rendering of COMPLETED rows is broken in the same way — same query/mapping defect surfacing in the list view) | 2026-05-20 |
| 9 | a11y / homepage footer | Homepage footer (`https://dev.parakh.civicdataspace.in/`) contains icon-only social-media links (Twitter/X, LinkedIn, GitHub, YouTube, etc.) with no `aria-label`, visible text, or `title` attribute. Violates WCAG 4.1.2 (Name, Role, Value) and 2.4.4 (Link Purpose). Screen readers announce these as unlabelled links. Frontend should add `aria-label="Follow us on Twitter"` (or equivalent) to each social icon `` in the footer. Same class of defect as bug #5 on the Keycloak page, but owned by the Parakh frontend team. | open | Load `https://dev.parakh.civicdataspace.in/`. Scroll to footer. Run `Array.from(document.querySelectorAll('footer a')).filter(a => !a.getAttribute('aria-label') && !a.textContent.trim() && !a.getAttribute('title')).map(a => a.getAttribute('href'))` — returns the unlabelled icon link hrefs. Confirmed via Playwright test `test_social_links_have_accessible_names` on 2026-05-20. | `tests/accessibility/test_accessibility.py::TestImagesAndLinks::test_social_links_have_accessible_names` | 2026-05-20 |
-| 8 | navigation / evaluator sidebar | Clicking **Assigned Models** or **Evaluations** from the Evaluator-role sidebar (`/dashboard/auditor`) highlights the link visually but does not navigate. URL stays on `/dashboard/auditor`. Direct navigation to `/dashboard/auditor/assignments` and `/dashboard/auditor/evaluations` works — only the sidebar click-throughs are broken. Same shape as bug #6 (AI-Maker list click-throughs) — likely the same shared Sidebar / Link interaction regression, just on the Evaluator-role layout. The "View Assignments" CTA in the evaluations empty state suffers the same defect. | open | Log in as `TEST_EMAIL_1` → AI Maker dashboard → Switch Roles → Evaluator. On `/dashboard/auditor` click "Assigned Models" in the left sidebar → URL unchanged, content unchanged. Repeat with "Evaluations". Then navigate directly to `/dashboard/auditor/evaluations`, observe the empty-state "View Assignments" link → click it → URL stays on `/dashboard/auditor/evaluations`. Confirmed via test runs 2026-05-20. | `tests/e2e/test_evaluator_role.py::TestEvaluatorAssignedModels::test_sidebar_link_navigates_to_assignments`; `tests/e2e/test_evaluator_role.py::TestEvaluatorEvaluations::test_sidebar_link_navigates_to_evaluations`; `tests/e2e/test_evaluator_role.py::TestEvaluatorEvaluations::test_view_assignments_link_navigates_to_assignments`; `tests/e2e/test_user_flows.py::TestFlow07_RoleSwitching::test_evaluator_assignments_page_accessible`; `tests/e2e/test_user_flows.py::TestFlow07_RoleSwitching::test_evaluator_can_navigate_to_their_evaluations` | 2026-05-20 |
+| 8 | navigation / evaluator sidebar | Clicking **Assigned Models** or **Evaluations** from the Evaluator-role sidebar (`/dashboard/auditor`) highlights the link visually but does not navigate. URL stays on `/dashboard/auditor`. Direct navigation to `/dashboard/auditor/assignments` and `/dashboard/auditor/evaluations` works — only the sidebar click-throughs are broken. Same shape as bug #6 (AI-Maker list click-throughs) — likely the same shared Sidebar / Link interaction regression, just on the Evaluator-role layout. The "View Assignments" CTA in the evaluations empty state suffers the same defect. | fixed 2026-05-23 | Log in as `TEST_EMAIL_1` → AI Maker dashboard → Switch Roles → Evaluator. On `/dashboard/auditor` click "Assigned Models" in the left sidebar → URL unchanged, content unchanged. Repeat with "Evaluations". Then navigate directly to `/dashboard/auditor/evaluations`, observe the empty-state "View Assignments" link → click it → URL stays on `/dashboard/auditor/evaluations`. Confirmed via test runs 2026-05-20. | `tests/e2e/test_evaluator_role.py::TestEvaluatorAssignedModels::test_sidebar_link_navigates_to_assignments`; `tests/e2e/test_evaluator_role.py::TestEvaluatorEvaluations::test_sidebar_link_navigates_to_evaluations`; `tests/e2e/test_evaluator_role.py::TestEvaluatorEvaluations::test_view_assignments_link_navigates_to_assignments`; `tests/e2e/test_user_flows.py::TestFlow07_RoleSwitching::test_evaluator_assignments_page_accessible`; `tests/e2e/test_user_flows.py::TestFlow07_RoleSwitching::test_evaluator_can_navigate_to_their_evaluations` | 2026-05-20 |
+
+| 10 | wizard / cancel | The **Cancel Evaluation** button (`"button:has-text('Cancel Evaluation')"` and its fallback selectors) is not found in the wizard after clicking **Start** from the modal. The wizard Configuration tab renders successfully (`WIZARD_TAB_CONFIGURATION` is visible) but the Cancel Evaluation button never appears within 5 seconds. Previously hidden by a `pytest.skip` guard — now surfaces as a failure after the guard was converted to `assert`. Could be a timing issue (the button renders after a further delay) or the button text / element has changed. | open | Log in → AI Maker → CivicDataLab → New Evaluation → Start → wait for wizard. Inspect DOM for `button` elements containing "Cancel" — the button may be absent or have a different label. | `tests/e2e/test_evaluations.py::TestNewEvaluationWizard::test_cancel_evaluation_returns_to_list` | 2026-05-23 |
+| 12 | auth / keycloak-dev | **Keycloak dev instance rate-limits ≥5 concurrent logins from the same account.** Under a 5-parallel-login load test, 3/5 workers fail with "login form not rendered" even after a 15 s wait — the form simply never appears. The 3-concurrent test (same account) passes consistently. Root causes (any combination): Keycloak brute-force-protection window, single-node session-table lock contention, or the dev nginx upstream thread pool exhaustion. **Not a bug in the Parakh frontend.** Upgrade path: HA Keycloak with session affinity disabled and per-user rate-limit tuning. | open (infra) | Run `pytest tests/load/test_load.py::TestConcurrentAuthentication -v -s` and observe 3/5 workers timing out on the login form. The 3-concurrent variant in the same class passes. | `tests/load/test_load.py::TestConcurrentAuthentication::test_5_concurrent_logins_success_rate` | 2026-05-23 |
+| 11 | wizard / manual-workspace | The manual-evaluation workspace Test Cases tab fails to render its expected UI elements when reached via the Domain type flow: (a) module cards appear (counter confirmed) but clicking the first card does not open an input/output panel — neither `MANUAL_INPUT_TEXTAREA` nor `MANUAL_CHANGE_MODULE_LINK` becomes visible; (b) the minimum test-cases hint ("Evaluate at least 3 test cases per module…") is not rendered on the Test Cases tab. These were previously skipped with `pytest.skip` guards; they now surface as failures. | open | Log in → AI Maker → CivicDataLab → New Evaluation → Start → select Domain type → fill Objective → click Test Cases tab. Click any module card and observe whether an input panel appears. Also check whether the min-cases hint text is rendered below the module list. | `tests/e2e/test_evaluation_workspace_manual.py::TestManualWorkspaceTestEntry::test_module_click_opens_input_panel`; `tests/e2e/test_evaluation_workspace_manual.py::TestManualWorkspaceTestEntry::test_min_test_cases_note_visible` | 2026-05-23 |
+| 13 | evaluations list / backend perf | The frontend `GetAudits` query that powers the AI-Maker Evaluations list (`/dashboard/ai-maker/{org}/evaluations`) **hangs and never returns** on dev — the page sits on "Loading evaluations…" indefinitely (still up at 90 s). Root-caused 2026-06-12 via live Playwright + direct GraphQL probing on org 1 (981 total audits): requesting the **light** field set (`id name status modelId`, `limit:100`) returns in ~10.6 s, and `metrics`-only or `modules`-only at `limit:100` each return in ~16–18 s, but the **full field set the frontend actually requests** (`name modelName status modules metrics evaluationMode auditType totalTests passedTests failedTests createdAt startedAt completedAt`) **times out past 30 s even at `limit:5`** — so it is the combined per-row resolver cost (likely N+1 on `metrics`/`modules`/test-count fields), not row volume. Backend should batch/precompute these fields or paginate server-side. Also observed intermittent `ERR_NETWORK_CHANGED` / "Backend server is not available" GraphQL errors on dev during the same session (degraded backend). **Not a test defect** — no client-side timeout bump can help since the query never completes. | open | Log in → AI Maker → CivicDataLab → Evaluations. Page stays on "Loading evaluations…". In DevTools, the `GetAudits` POST to `dev.api.parakh.civicdataspace.in/graphql/` has no response after 90 s. Confirmed via Playwright MCP 2026-06-12: full-field `audits` query aborts at 30 s even with `limit:5`, while light-field returns in ~10 s. | `tests/e2e/test_evaluations.py::TestStatusFilterTabs::*` (all 6 — tabs/column/rows render only after the list query resolves); transitively any test that asserts on rendered evaluations-list content for org 1 | 2026-06-12 |
## Conventions
diff --git a/locators/add_model_flow_locators.py b/locators/add_model_flow_locators.py
new file mode 100644
index 0000000..617e081
--- /dev/null
+++ b/locators/add_model_flow_locators.py
@@ -0,0 +1,26 @@
+"""Selectors for the Add Model cross-platform flow (ParakhAI → CivicDataSpace)."""
+
+
+class AddModelFlowLocators:
+ ADD_NEW_MODEL_BUTTON = "button:has-text('Add A New Model'), button:has-text('Add New AI Model')"
+ REDIRECT_DIALOG = "[role='dialog']:has-text('CivicDataSpace'), [role='alertdialog']"
+ REDIRECT_CONFIRM_BTN = (
+ "[role='dialog'] button:has-text('Go'), "
+ "[role='dialog'] button:has-text('CivicDataSpace'), "
+ "[role='dialog'] button:has-text('Proceed')"
+ )
+ REDIRECT_CANCEL_BTN = "[role='dialog'] button:has-text('Cancel')"
+
+ CDS_ADD_MODEL_BTN = "button:has-text('Add New AI Model'), button:has-text('Add AI Model')"
+ CDS_STEP1_TITLE_INPUT = (
+ "input[name='title'], input[placeholder*='title'], input[placeholder*='model name']"
+ )
+ CDS_NEXT_BTN = "button:has-text('Next'), button:has-text('Save & Continue')"
+ CDS_AUTOSAVE_INDICATOR = "text=All Changes Saved, text=Saved"
+ CDS_STEP_INDICATOR = "[aria-label*='step'], .step-indicator, [data-step]"
+ CDS_PUBLISH_BTN = "button:has-text('Publish')"
+ CDS_ADD_ACCESS_METHOD_BTN = "button:has-text('Add New Access Method')"
+ CDS_API_KEY_INPUT = (
+ "input[type='password'][name*='key'], input[name*='apiKey'], input[placeholder*='API']"
+ )
+ CDS_QUILL_BULLET_BTN = ".ql-bullet, .ql-list[value='bullet']"
diff --git a/locators/ai_maker_locators.py b/locators/ai_maker_locators.py
index 746491e..cce9f01 100644
--- a/locators/ai_maker_locators.py
+++ b/locators/ai_maker_locators.py
@@ -31,6 +31,17 @@ class AIMakerLocators:
STAT_ISSUES_FLAGGED = "text=Issues Flagged"
STAT_CARD = "[class*='stat'], [class*='card'], [class*='Card']"
+ # ── Org selection page: Add Organisation button (Jun 2026) ───────────────
+ ADD_ORGANISATION_BUTTON = (
+ "button:has-text('Add Organisation'), a:has-text('Add Organisation')"
+ )
+ # AlertDialog opened by Add Organisation / Add A New Model / Edit model buttons.
+ # The dialog title and "Yes, continue" CTA are stable text anchors.
+ EXTERNAL_REDIRECT_DIALOG = (
+ "[role='alertdialog'], [role='dialog'], [class*='AlertDialog'], [class*='alertdialog']"
+ )
+ EXTERNAL_REDIRECT_CONFIRM = "button:has-text('Yes, continue')"
+
# ── Models section on home ─────────────────────────────────────────────────
MODELS_SECTION_HEADING = "text=Models"
# Playwright's `text=` engine treats commas as literal text; the original
diff --git a/locators/evaluations_locators.py b/locators/evaluations_locators.py
index 162c79c..8a064ca 100644
--- a/locators/evaluations_locators.py
+++ b/locators/evaluations_locators.py
@@ -17,6 +17,29 @@ class EvaluationsLocators:
EVAL_TESTS_COL = "th:text('Tests'), :text('Tests')"
EVAL_COMPLETED_COL = "th:text('Completed'), :text('Completed')"
+ # ── Status filter tabs (StatusFilterTabs component — Jun 2026) ───────────
+ # As of late Jun 2026 the component renders 9 tabs:
+ # All | Draft | Queued | Running | In Progress | Pending Review | Completed | Failed | Cancelled
+ # Use :has-text for substring match so count badges ("Draft(40)") still match.
+ STATUS_TAB_ALL = "button:has-text('All'), [role='tab']:has-text('All')"
+ STATUS_TAB_DRAFT = "button:has-text('Draft'), [role='tab']:has-text('Draft')"
+ # "Pending" is now split into "Queued" and "Pending Review" — keep old selector
+ # as a broad fallback and add the specific new ones.
+ STATUS_TAB_PENDING = "button:has-text('Pending'), [role='tab']:has-text('Pending')"
+ STATUS_TAB_QUEUED = "button:has-text('Queued'), [role='tab']:has-text('Queued')"
+ STATUS_TAB_IN_PROGRESS = "button:has-text('In Progress'), [role='tab']:has-text('In Progress')"
+ STATUS_TAB_PENDING_REVIEW = "button:has-text('Pending Review'), [role='tab']:has-text('Pending Review')"
+ STATUS_TAB_RUNNING = "button:has-text('Running'), [role='tab']:has-text('Running')"
+ STATUS_TAB_COMPLETED = "button:has-text('Completed'), [role='tab']:has-text('Completed')"
+ STATUS_TAB_FAILED = "button:has-text('Failed'), [role='tab']:has-text('Failed')"
+ STATUS_TAB_CANCELLED = "button:has-text('Cancelled'), [role='tab']:has-text('Cancelled')"
+
+ # ── Pagination controls (Jun 2026) ────────────────────────────────────────
+ PAGINATION_NEXT = (
+ "button:has-text('Next'), [aria-label='Go to next page'], [class*='pagination'] button:last-child"
+ )
+ PAGINATION_CONTAINER = "[class*='pagination'], [class*='Pagination'], [aria-label*='pagination']"
+
# Status badges and mode labels live inside the evaluations table — scope
# the selectors to table cells so they don't match unrelated text elsewhere
# on the page (e.g. the Mode dropdown options literally contain "Automated"
@@ -48,7 +71,11 @@ class EvaluationsLocators:
MODAL_MODEL_DROPDOWN = "select, [class*='select'], [role='combobox']"
MODAL_VERSION_DROPDOWN = "select, [class*='select'], [role='combobox']"
MODAL_START_BUTTON = "button:has-text('Start')"
- MODAL_CANCEL_BUTTON = "button:has-text('Cancel')"
+ MODAL_CANCEL_BUTTON = (
+ "[role='dialog'] button:has-text('Cancel'), "
+ "[class*='modal'] button:has-text('Cancel'), "
+ "[class*='Modal'] button:has-text('Cancel')"
+ )
# Modal dropdown option lists — at least one