diff --git a/.github/workflows/44-railway-tests.yml b/.github/workflows/44-railway-tests.yml index ad537b7ac1..e5b0dc7988 100644 --- a/.github/workflows/44-railway-tests.yml +++ b/.github/workflows/44-railway-tests.yml @@ -252,7 +252,7 @@ jobs: echo "Waiting for ${label}: ${url}" for attempt in $(seq 1 30); do - if curl -sfL -o /dev/null "$url" 2>/dev/null; then + if curl -sfL --max-time 10 --connect-timeout 5 -o /dev/null "$url" 2>/dev/null; then echo "${label} is ready." return 0 fi diff --git a/web/ee/tests/playwright/acceptance/members/index.ts b/web/ee/tests/playwright/acceptance/members/index.ts index 4ea83d5ed6..383e8a26d7 100644 --- a/web/ee/tests/playwright/acceptance/members/index.ts +++ b/web/ee/tests/playwright/acceptance/members/index.ts @@ -33,20 +33,6 @@ const lightFastTags = buildAcceptanceTags({ const createInviteEmail = (scope: string) => `${scope}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}@agenta.test` -const waitForInviteResponse = async (page: any) => { - const response = await page.waitForResponse( - (res: any) => - res.request().method() === "POST" && - res.url().includes("/workspaces/") && - res.url().includes("/invite?"), - {timeout: 15000}, - ) - - if (!response.ok()) { - throw new Error(`Invite request failed (${response.status()}): ${await response.text()}`) - } -} - const waitForRemoveResponse = async (page: any) => { const response = await page.waitForResponse( (res: any) => @@ -64,6 +50,59 @@ const waitForRemoveResponse = async (page: any) => { } } +const openInviteMembersModal = async (page: any) => { + const inviteButton = page.getByRole("button", {name: "Invite Members"}).first() + await expect(inviteButton).toBeVisible({timeout: 20000}) + await expect(inviteButton).toBeEnabled() + + const inviteModal = page.getByRole("dialog", {name: "Invite Members"}) + + // Use a PAGE-LEVEL (unscoped) locator for the email input. + // + // Scoping through `inviteModal.getByPlaceholder(...)` is unreliable here because: + // 1. InviteUsersModal is a `dynamic()` import — the form mounts AFTER the modal + // wrapper becomes visible, so the dialog-scoped locator resolves to zero elements + // until the JS chunk fully evaluates. + // 2. rc-dialog briefly UNMOUNTS content while its `animatedVisible` useEffect + // settles (fires on the next frame after first render), making a dialog-scoped + // locator transiently stale. + // Searching the entire page avoids both issues while remaining unique in practice + // (only one invite form is ever present at a time). + const emailInput = page.getByPlaceholder("member@organization.com").first() + + for (let attempt = 0; attempt < 3; attempt++) { + // Ensure any previous dialog is closed before clicking again. + const alreadyOpen = await inviteModal.isVisible().catch(() => false) + if (!alreadyOpen) { + await inviteButton.click() + } + + // Wait for the email input to become visible. This is the most reliable + // signal that both the modal wrapper AND its dynamic content are ready. + const inputAppeared = await emailInput + .waitFor({state: "visible", timeout: 15000}) + .then(() => true) + .catch(() => false) + + if (inputAppeared) { + return {inviteModal, emailInput} + } + + // Form never appeared — dismiss any partial modal and retry. + await page.keyboard.press("Escape") + await page.waitForTimeout(500) + } + + // Final assertion: surfaces a clear error if the input never appeared. + await expect(emailInput).toBeVisible({timeout: 15000}) + return {inviteModal, emailInput} +} + +const submitInviteMembersModal = async (inviteModal: any) => { + await inviteModal.locator("form").evaluate((form: HTMLFormElement) => form.requestSubmit()) + await expect(inviteModal).not.toBeVisible({timeout: 30000}) +} + /** * Invite a member via the EE flow (email sent) and wait for their row to appear * in the members table with "Invitation Pending" status. @@ -76,23 +115,23 @@ const invitePendingMember = async (page: any, apiHelpers: any, uiHelpers: any): await page.goto(`${basePath}/settings`, {waitUntil: "domcontentloaded"}) await uiHelpers.expectPath("/settings") - const inviteButton = page.getByRole("button", {name: "Invite Members"}) - await expect(inviteButton).toBeVisible({timeout: 20000}) - await inviteButton.click() - - const inviteModal = page.getByRole("dialog", {name: "Invite Members"}) - const emailInput = inviteModal.getByPlaceholder("member@organization.com") + const {inviteModal, emailInput} = await openInviteMembersModal(page) // Wait for the email input rather than just the dialog — the InviteUsersModal // is a dynamic() import, so the form body can lag behind the modal wrapper. // Waiting for the input guarantees the chunk has fully rendered. - await expect(emailInput).toBeVisible({timeout: 20000}) + // Click before fill: rc-component/dialog briefly unmounts while animatedVisible + // catches up (useEffect fires after first render), which makes the locator + // stale. A click forces Playwright to wait for the element to be fully + // interactive before fill attempts to interact. + await emailInput.click() await emailInput.fill(testEmail) - await Promise.all([ - waitForInviteResponse(page), - inviteModal.getByRole("button", {name: "Invite"}).click(), - ]) - await expect(inviteModal).not.toBeVisible({timeout: 15000}) + // Submit the form and wait for the modal to close as the success signal. + // The InviteUsersModal only closes its onSuccess callback after the API + // returns successfully — so modal closure == invite accepted. + // Waiting for the network response by URL is fragile (URL-pattern drift, + // timing races between listener registration and the async form submit). + await submitInviteMembersModal(inviteModal) // Wait for the pending row to appear in the refreshed table await expect(page.getByText(testEmail)).toBeVisible({timeout: 15000}) @@ -106,7 +145,8 @@ const membersTests = () => { "should invite a member and verify pending state", {tag: lightFastTags}, async ({page, apiHelpers, uiHelpers}) => { - test.setTimeout(60000) + // 90 s: navigation + up to 3 modal-open attempts × 15 s + fill + submit + assertion + test.setTimeout(90000) const testEmail = createInviteEmail("test-member-invite") await scenarios.given("the user is authenticated", async () => { @@ -125,13 +165,14 @@ const membersTests = () => { await scenarios.when( "the user clicks Invite Members, fills in an email address and selects a role", async () => { - await page.getByRole("button", {name: "Invite Members"}).click() - - const inviteModal = page.getByRole("dialog", {name: "Invite Members"}) - const emailInput = inviteModal.getByPlaceholder("member@organization.com") + const {inviteModal, emailInput} = await openInviteMembersModal(page) // Wait for the input directly — the InviteUsersModal is a dynamic() // import so the form body can lag behind the modal wrapper appearing. - await expect(emailInput).toBeVisible({timeout: 20000}) + // Click before fill: rc-component/dialog briefly unmounts the panel + // while animatedVisible settles (useEffect fires after first render), + // making the locator transiently stale. Clicking first ensures the + // element is fully interactive before fill runs. + await emailInput.click() await emailInput.fill(testEmail) // EE renders a role selector; keep the default selection @@ -144,11 +185,10 @@ const membersTests = () => { await scenarios.and("the user submits the invitation", async () => { const inviteModal = page.getByRole("dialog", {name: "Invite Members"}) - await Promise.all([ - waitForInviteResponse(page), - inviteModal.getByRole("button", {name: "Invite"}).click(), - ]) - await expect(inviteModal).not.toBeVisible({timeout: 15000}) + // Submit the form and wait for the modal to dismiss as the success signal. + // The InviteUsersModal only closes after the API returns successfully, + // so modal closure is equivalent to a successful invite response. + await submitInviteMembersModal(inviteModal) }) await scenarios.then( diff --git a/web/ee/tests/playwright/acceptance/use-api/use-api.spec.ts b/web/ee/tests/playwright/acceptance/use-api/use-api.spec.ts new file mode 100644 index 0000000000..87005cd4b1 --- /dev/null +++ b/web/ee/tests/playwright/acceptance/use-api/use-api.spec.ts @@ -0,0 +1,4 @@ +import {test} from "@agenta/web-tests/tests/fixtures/base.fixture" +import useApiTests from "@agenta/oss/tests/playwright/10-use-api" + +test.describe("Registry: use API snippets", useApiTests) diff --git a/web/oss/tests/playwright/10-use-api.ts b/web/oss/tests/playwright/10-use-api.ts new file mode 100644 index 0000000000..31e38f6118 --- /dev/null +++ b/web/oss/tests/playwright/10-use-api.ts @@ -0,0 +1,3 @@ +import useApiTests from "./acceptance/use-api" + +export default useApiTests diff --git a/web/oss/tests/playwright/acceptance/app/test.ts b/web/oss/tests/playwright/acceptance/app/test.ts index 1d56c967e5..2a64cb9e47 100644 --- a/web/oss/tests/playwright/acceptance/app/test.ts +++ b/web/oss/tests/playwright/acceptance/app/test.ts @@ -43,6 +43,11 @@ export const openCreateAppDrawerForType = async ( .or(page.getByTestId(modalTypeTestId)) .first() + const drawer = page + .getByRole("dialog") + .filter({has: page.getByTestId("app-create-name-input")}) + .last() + for (let attempt = 0; attempt < 3; attempt += 1) { for (const entryPoint of createEntryPoints) { if (!(await entryPoint.isVisible().catch(() => false))) continue @@ -52,34 +57,43 @@ export const openCreateAppDrawerForType = async ( break } - const opened = await typeSelector - .waitFor({state: "visible", timeout: 3000}) + const typeSelectorVisible = await typeSelector + .waitFor({state: "visible", timeout: 4000}) .then(() => true) .catch(() => false) - if (opened) { - // The Popover re-renders when appTemplatesQueryAtom resolves, - // making the item briefly unstable. force:true dispatches the - // click immediately without waiting for Playwright's stability - // check, which otherwise retries until the 60 s test timeout. - await typeSelector.click({force: true}) - const drawer = page - .getByRole("dialog") - .filter({has: page.getByTestId("app-create-name-input")}) - .last() - await expect(drawer).toBeVisible({timeout: 15000}) + if (!typeSelectorVisible) { + await page.keyboard.press("Escape").catch(() => undefined) + continue + } + + // The Popover re-renders when appTemplatesQueryAtom resolves, making + // the item briefly unstable. dispatchEvent('click') fires a synthetic + // DOM event that bypasses both Playwright's stability check AND the + // viewport-position check (newer Playwright no longer allows force:true + // to click elements outside the viewport). The drawer check below + // catches the rare case where the click still missed. + await typeSelector.dispatchEvent("click") + + // Check whether the drawer opened. If the click landed on a stale + // element during re-render it won't appear — retry rather than throw. + const drawerOpened = await drawer + .waitFor({state: "visible", timeout: 8000}) + .then(() => true) + .catch(() => false) + + if (drawerOpened) { return drawer } + // Drawer didn't open — dismiss any leftover popover and try again. await page.keyboard.press("Escape").catch(() => undefined) + await page.waitForTimeout(200) } + // Final attempt: surfaces a clear failure if the drawer still won't open. await expect(typeSelector).toBeVisible({timeout: 15000}) - await typeSelector.click() - const drawer = page - .getByRole("dialog") - .filter({has: page.getByTestId("app-create-name-input")}) - .last() + await typeSelector.dispatchEvent("click") await expect(drawer).toBeVisible({timeout: 15000}) return drawer } diff --git a/web/oss/tests/playwright/acceptance/evaluators/tests.ts b/web/oss/tests/playwright/acceptance/evaluators/tests.ts index d04319b417..eaf653aa87 100644 --- a/web/oss/tests/playwright/acceptance/evaluators/tests.ts +++ b/web/oss/tests/playwright/acceptance/evaluators/tests.ts @@ -351,6 +351,9 @@ const createHumanEvaluatorFromDrawer = async ( await nameInput.fill(evaluatorName) await expect(nameInput).toHaveValue(evaluatorName) + const slugInput = drawer.locator('input[placeholder="Enter a unique slug"]').first() + await expect(slugInput).toHaveValue(evaluatorName, {timeout: 5000}) + // Fill in the feedback name (the first metric row) const feedbackNameInput = drawer .locator(`input[placeholder="${HUMAN_EVALUATOR_FEEDBACK_NAME_PLACEHOLDER}"]`) diff --git a/web/oss/tests/playwright/acceptance/features/use-api.feature b/web/oss/tests/playwright/acceptance/features/use-api.feature new file mode 100644 index 0000000000..0d734c716f --- /dev/null +++ b/web/oss/tests/playwright/acceptance/features/use-api.feature @@ -0,0 +1,40 @@ +# Tests: use-api/use-api.spec.ts -> use-api/index.ts +# RTM IDs: WEB-ACC-USEAPI-001, WEB-ACC-USEAPI-002 +# Tags: @scope:deployment @coverage:light @path:happy @speed:fast +# +# Implementation notes: +# - Variant mode: navigate to /apps/{id}/variants, click "Use API" (primary button) +# → DeploymentsDrawer opens in mode="variant" rendering VariantUseApiContent +# → Fetch Prompt/Config snippet uses application_variant_ref (variant-keyed endpoint) +# → Invoke LLM snippet uses axios.post to the variant invocation URL +# - Deployment mode: navigate to /apps/{id}/variants?tab=deployments&selectedEnvName=development +# → click "Use API" (primary button in the deployments tab header) +# → DeploymentsDrawer opens in mode="deployment" rendering UseApiContent +# → Fetch Prompt/Config snippet uses environment_ref (environment-keyed endpoint) +# → Invoke LLM snippet uses axios.post to the environment invocation URL +# - Both tests assert TypeScript tab only (Python and cURL are separate language concerns). + +Feature: Registry Use API — TypeScript snippets + As a user + I want to view TypeScript code snippets for calling my app via API + So that I can integrate Agenta into my TypeScript project + + Background: + Given the user is authenticated + And a completion app with at least one variant exists + + @light @happy @scope:deployment @speed:fast + Scenario: Variant TypeScript snippet shows application_variant_ref in Fetch section + Given the user is on the Variants registry page + When the user opens the Use API drawer + And the user selects the TypeScript tab + Then the Fetch Prompt/Config section displays the variant TypeScript snippet + And the Invoke LLM section displays a TypeScript axios snippet + + @light @happy @scope:deployment @speed:fast + Scenario: Deployment TypeScript snippet shows environment_ref in Fetch section + Given the user is on the Deployments registry page for the Development environment + When the user opens the Use API drawer + And the user selects the TypeScript tab + Then the Fetch Prompt/Config section displays the deployment TypeScript snippet + And the Invoke LLM section displays a TypeScript axios snippet diff --git a/web/oss/tests/playwright/acceptance/members/index.ts b/web/oss/tests/playwright/acceptance/members/index.ts index b503978a52..6882ab3d36 100644 --- a/web/oss/tests/playwright/acceptance/members/index.ts +++ b/web/oss/tests/playwright/acceptance/members/index.ts @@ -36,7 +36,8 @@ const membersTests = () => { "should invite a member and show the invite link modal", {tag: lightFastTags}, async ({page, apiHelpers, uiHelpers}) => { - test.setTimeout(60000) + // 90 s: settings page load (~5 s) + invite API call (~15 s) + modal assertions + test.setTimeout(90000) const testEmail = `test-member-invite-${Date.now()}@agenta-e2e.test` await scenarios.given("the user is authenticated", async () => { @@ -49,7 +50,7 @@ const membersTests = () => { await uiHelpers.expectPath("/settings") // The default tab is "workspace" which renders the Members section await expect(page.getByRole("button", {name: "Invite Members"})).toBeVisible({ - timeout: 15000, + timeout: 20000, }) }) @@ -69,26 +70,44 @@ const membersTests = () => { await scenarios.and("the user submits the invitation", async () => { const inviteModal = page.getByRole("dialog", {name: "Invite Members"}) + + // Set up the API response listener BEFORE clicking so we don't miss the + // response if the API is fast. + const inviteResponsePromise = page + .waitForResponse( + (r) => /\/invite/i.test(r.url()) && r.request().method() === "POST", + {timeout: 30000}, + ) + .catch(() => null) + await inviteModal.getByRole("button", {name: "Invite"}).click() - // Invite modal closes before link modal opens - await expect(inviteModal).not.toBeVisible({timeout: 15000}) + + // Wait for the API call to complete before asserting modal state. + await inviteResponsePromise + + // Invite modal should close once the server responds with the invite link. + await expect(inviteModal).not.toBeVisible({timeout: 20000}) }) await scenarios.then( "the invited user link modal appears with a shareable URL", async () => { const linkModal = page.getByRole("dialog", {name: "Invited user link"}) - await expect(linkModal).toBeVisible({timeout: 15000}) + await expect(linkModal).toBeVisible({timeout: 20000}) // Verify the modal shows the invited email - await expect(linkModal.getByText(testEmail)).toBeVisible({timeout: 5000}) + await expect(linkModal.getByText(testEmail)).toBeVisible({timeout: 10000}) // Verify the invite URL is present await expect(linkModal.getByText(/https?:\/\//)).toBeVisible({timeout: 5000}) // Close via the X button — "Copy & Close" calls navigator.clipboard which // throws in headless CI, preventing onCancel from being called. - await linkModal.locator('button[aria-label="Close"]').click() + const closeButton = linkModal + .locator('button[aria-label="Close"]') + .or(linkModal.getByRole("button", {name: "Close"})) + .first() + await closeButton.click() await expect(linkModal).not.toBeVisible({timeout: 10000}) }, ) diff --git a/web/oss/tests/playwright/acceptance/observability/index.ts b/web/oss/tests/playwright/acceptance/observability/index.ts index a1253549fc..7fb82bc652 100644 --- a/web/oss/tests/playwright/acceptance/observability/index.ts +++ b/web/oss/tests/playwright/acceptance/observability/index.ts @@ -58,11 +58,11 @@ const clickFirstTraceRow = async (page: any) => { * * Traces are indexed asynchronously. The first trace in an ephemeral project can * take up to ~150 s to appear. Setup (provider check + app creation + playground run) - * adds another 30-60 s on top. The function enables auto-refresh (15 s interval) + * adds another 60-90 s on top. The function enables auto-refresh (15 s interval) * so the page re-fetches automatically, and also performs periodic manual refreshes - * every 20 s for up to 200 s total while waiting for [data-tour="trace-row"]. + * every 15 s for up to 240 s total while waiting for [data-tour="trace-row"]. * - * Tests using this function must set test.setTimeout to at least 300000 (5 min). + * Tests using this function must set test.setTimeout to at least 420000 (7 min). */ const runPlaygroundAndGoToObservability = async ( page: any, @@ -86,10 +86,9 @@ const runPlaygroundAndGoToObservability = async ( // Select the mock test model await testProviderHelpers.selectTestModel() - // Fill in the completion input and run - const textbox = page - .locator('.agenta-shared-editor:has(div:text-is("Enter a value")) [role="textbox"]') - .first() + // Fill in the completion input and run. + // VariableCard renders antd TextArea (