Skip to content

Commit

Permalink
More shell tests (#653)
Browse files Browse the repository at this point in the history
  • Loading branch information
robgruen authored Feb 3, 2025
1 parent 313cf3e commit 28c43a0
Show file tree
Hide file tree
Showing 8 changed files with 259 additions and 214 deletions.
44 changes: 0 additions & 44 deletions .github/workflows/build-ts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,6 @@ jobs:
working-directory: ts
run: |
pnpm install --frozen-lockfile --strict-peer-dependencies
# - name: Install Playwright Browsers
# if: ${{ runner.os == 'windows' && matrix.version == '22' }}
# run: pnpm exec playwright install --with-deps
# working-directory: ts/packages/shell
- name: Build
if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }}
working-directory: ts
Expand All @@ -85,43 +81,3 @@ jobs:
working-directory: ts
run: |
npm run lint
# - name: Login to Azure
# if: ${{ github.event_name != 'merge_group' }}
# uses: azure/[email protected]
# with:
# client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }}
# tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }}
# subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }}
# - name: Get Keys
# if: ${{ github.event_name != 'merge_group' }}
# run: |
# node tools/scripts/getKeys.mjs --vault build-pipeline-kv
# working-directory: ts
# - name: Test CLI - smoke
# if: ${{ github.event_name != 'merge_group' }}
# run: |
# npm run start:dev 'prompt' 'why is the sky blue'
# working-directory: ts/packages/cli
# continue-on-error: true
# - name: Shell Tests - smoke (windows)
# if: ${{ github.event_name != 'merge_group' && runner.os == 'windows' && matrix.version == '22' }}
# timeout-minutes: 60
# run: |
# npx playwright test simple.spec.ts
# rm ../../.env
# - name: Shell Tests - smoke (linux)
# if: ${{ github.event_name != 'merge_group' && runner.os == 'Linux' && matrix.version == '22' }}
# timeout-minutes: 60
# run: |
# Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
# npx playwright test simple.spec.ts
# rm ../../.env
# working-directory: ts/packages/shell
# continue-on-error: true
# - name: Live Tests
# if: ${{ github.event_name != 'merge_group' && runner.os == 'linux' && matrix.version == '22' }}
# timeout-minutes: 60
# run: |
# npm run test:live
# working-directory: ts
# continue-on-error: true
12 changes: 7 additions & 5 deletions .github/workflows/shell-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: ["windows-latest"]
os: ["windows-latest", "ubuntu-latest"]
version: [20]

runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -98,19 +98,15 @@ jobs:
timeout-minutes: 60
run: |
npm run shell:test
rm ../../.env
working-directory: ts/packages/shell
continue-on-error: true

- name: Shell Tests (linux)
if: ${{ runner.os == 'Linux' }}
timeout-minutes: 60
# https://github.com/microsoft/playwright/issues/34251 - sysctl command
run: |
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
npm run shell:test
rm ../../.env
working-directory: ts/packages/shell
continue-on-error: true

Expand All @@ -129,3 +125,9 @@ jobs:
path: ts/packages/shell/playwright-report/
overwrite: true
retention-days: 30

- name: Clean up Keys
run: |
rm ./.env
working-directory: ts
if: always()
121 changes: 0 additions & 121 deletions .github/workflows/smoke-tests-pull_request_targets.yml.bak

This file was deleted.

7 changes: 3 additions & 4 deletions .github/workflows/smoke-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ on:
workflow_dispatch:
push:
branches: ["main"]
# pull_request:
# branches: ["main"]
pull_request_target:
branches: ["main"]
merge_group:
Expand All @@ -27,7 +25,9 @@ permissions:

env:
NODE_OPTIONS: --max_old_space_size=8192
DEBUG: pw:browser*
# DEBUG: pw:browser* # PlayWright debug messages
# ELECTRON_ENABLE_LOGGING: true # Electron debug messages
# DEBUG: typeagent:* # TypeAgent debug messages

jobs:
shell_and_cli:
Expand All @@ -37,7 +37,6 @@ jobs:
fail-fast: false
matrix:
os: ["ubuntu-latest", "windows-latest"]
#os: ["ubuntu-latest"]
version: [20]

runs-on: ${{ matrix.os }}
Expand Down
64 changes: 64 additions & 0 deletions ts/packages/shell/test/listAgent.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import test, {
ElectronApplication,
Page,
_electron,
_electron as electron,
expect,
} from "@playwright/test";
import {
exitApplication,
getAppPath,
sendUserRequestAndWaitForCompletion,
getLaunchArgs,
startShell,
testUserRequest,
} from "./testHelper";

// Annotate entire file as serial.
test.describe.configure({ mode: "serial" });

test.describe("List Agent Tests", () => {
test("create_update_clear_list", async ({}, testInfo) => {
console.log(`Running test '${testInfo.title}`);

await testUserRequest(
[
"create a shopping list",
"what's on the shopping list?",
"add eggs, milk, flour to the shopping list",
"what's on the shopping list?",
"remove milk from the shopping list",
"what's on the shopping list?",
"clear the shopping list",
"what's on the shopping list?",
],
[
"Created list: shopping",
"List 'shopping' is empty",
"Added items: eggs,milk,flour to list shopping",
"eggs\nmilk\nflour",
"Removed items: milk from list shopping",
"eggs\nflour",
"Cleared list: shopping",
"List 'shopping' is empty",
],
);
});

// test("delete_list", async ({}, testInfo) => {
// console.log(`Running test '${testInfo.title}`);

// await testUserRequest(
// [
// "delete the shopping list",
// "is there a shopping list?"
// ],
// [
// "Cleared list: shopping",
// "List 'shopping' is empty"
// ]);
// });
});
Loading

0 comments on commit 28c43a0

Please sign in to comment.