From 29899223e42761f8e48b46c9e10b9dd20c9c8d67 Mon Sep 17 00:00:00 2001 From: isopropilick Date: Tue, 15 Sep 2026 10:32:11 -0600 Subject: [PATCH] Enhance CI configuration and documentation; add new action palette adapter --- .github/workflows/ci.yml | 6 ++- CHANGELOG.md | 37 +++++++++++++++ README.md | 10 ++++ package-lock.json | 8 ++-- package.json | 5 +- scripts/run-profile.mjs | 42 ++++++++++++++++ test/step-definitions/steps.ts | 2 +- test/support/qkforce-actions.mjs | 45 ++++++++++++++++++ test/support/qkforce-generated.mjs | 51 ++++++++++++++++++++ test/unit/qkforce-generated.test.mjs | 71 ++++++++++++++++++++++++++++ test/unit/run-profile.test.mjs | 24 ++++++++++ wdio.conf.ts | 6 ++- 12 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 scripts/run-profile.mjs create mode 100644 test/support/qkforce-actions.mjs create mode 100644 test/support/qkforce-generated.mjs create mode 100644 test/unit/qkforce-generated.test.mjs create mode 100644 test/unit/run-profile.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcdee01..820ced8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,12 +5,14 @@ on: branches: [develop, main] pull_request: branches: [develop, main] + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read jobs: validate: + if: github.event_name == 'push' || github.event.pull_request.draft == false runs-on: ubuntu-latest steps: - name: Checkout @@ -31,9 +33,9 @@ jobs: - name: Execute fixed QKForce profile run: npm run qkforce:test - - name: Build portable report + - name: Verify portable report exists if: always() - run: npm run qkforce:report + run: test -s qreport-results/index.html - name: Upload QKTestAnalytics results if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5440301 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +All notable changes to QKForce Framework CWQ are recorded here. + +## Unreleased + +No unreleased changes. + +## 0.2.0 - 2026-09-11 + +### Review fixes + +- Regenerate the lockfile from the registry for QKTestAnalytics 0.4.1. +- Build portable HTML through the fixed test entrypoint after runner success or failure and preserve nonzero status. +- Add entrypoint failure/report sequencing tests and verify HTML presence in CI. + + +Initial release candidate for the fixed QKForce execution template. + +### Added + +- Fixed TypeScript, Cucumber, WebdriverIO and QKTestAnalytics execution profile. +- Generated-project manifest validation and deterministic `qkforce:test` entrypoint. +- Canonical `features/`, `steps/` and page-object layout shared with the FrontEnd generator. +- QKTestAnalytics adapter integration, result manifests and portable template contract documentation. +- Dependency-free structural tests for generated project paths and execution entrypoints. + +### Changed + +- QKForce-generated suites use `test/qkforce/manifest.json` as the runtime provenance boundary. +- The template consumes `@qacg/qk-test-analytics` 0.4.1 at the manifest level. + +### Known limitations + +- The Analytics 0.4.1 lockfile is regenerated and clean-install/contract checks pass; merge and exact-candidate CI evidence remain required (#10). +- A successful required terminal CI run on the release candidate is still needed (#6). +- Portable report orchestration is implemented; passing/failing real-scenario acceptance remains tracked in #8. diff --git a/README.md b/README.md index 746efa0..acba8f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # QKForce Framework CWQ +## Generated project execution + +Frontend-generated suites use `test/qkforce/manifest.json` (schema 1), exact feature paths, `steps.mjs` and `selectors.json`. The runtime loads only that selection, supplies explicit Cucumber imports and drains legacy palette actions in order. Packaged workspaces without generated inputs fail instead of running template examples. Regenerate projects after migrating to this template revision; publish the runtime to the actual DB-configured Git remote before deploying API #108/frontend #61. + +`npm run test:contract` covers config, immutable test data and generated-workspace validation. The separate HTML-report entrypoint gap remains #8. Real browser/client execution remains the release gate. + The QKForce browser-automation template is the fixed `TypeScript + Cucumber + WebdriverIO + QKTestAnalytics` profile. ## Contract status @@ -68,3 +74,7 @@ npm run check ``` Active development is performed on `develop`. + +## Fixed execution and reporting + +`npm run qkforce:test` archives the previous cycle, runs WDIO, then builds `qreport-results/index.html` even when WDIO fails. The runner exit code is preserved; report failure makes a successful run fail. No command is accepted from the execution manifest. `qkforce:report` remains available for rebuilding an existing report. Cancellation or a forcibly terminated client cannot guarantee report generation. diff --git a/package-lock.json b/package-lock.json index 594d0d3..23241c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "qkforce-framework-cwq", "version": "0.2.0", "devDependencies": { - "@qacg/qk-test-analytics": "0.4.0", + "@qacg/qk-test-analytics": "0.4.1", "@types/node": "22.15.21", "@wdio/cli": "9.31.1", "@wdio/cucumber-framework": "9.31.1", @@ -1718,9 +1718,9 @@ } }, "node_modules/@qacg/qk-test-analytics": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@qacg/qk-test-analytics/-/qk-test-analytics-0.4.0.tgz", - "integrity": "sha512-IqdfcU3YWOhBTC1cefsaO15Eu5hC5UoTlKJOnrntfTzRrA1NAenoxUesuh2DBt/LDUIZspaZkNFH11IGl/9Nzw==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@qacg/qk-test-analytics/-/qk-test-analytics-0.4.1.tgz", + "integrity": "sha512-QX8hgd6mHnJjs56G4GTi8xid09xVbwCRB1y8uVcUBX++30NxLsgHMJ0t1VZqZcIaMcaWCnXO/x+b0SCsLxIQPA==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/package.json b/package.json index f00db7d..da070a1 100644 --- a/package.json +++ b/package.json @@ -4,16 +4,17 @@ "private": true, "description": "Fixed QKForce WebdriverIO, Cucumber and QKTestAnalytics execution template.", "type": "module", + "qkforce": { "generatedWorkspaceSchema": 1 }, "engines": { "node": ">=20.19.0 <25" }, "scripts": { - "qkforce:test": "qkta cycle --new -- npm run test:wdio", + "qkforce:test": "node scripts/run-profile.mjs", "test": "npm run qkforce:test", "test:wdio": "wdio run ./wdio.conf.ts", "test:smoke": "qkta cycle --new -- wdio run ./wdio.conf.ts --cucumberOpts.tagExpression=@smoke", "qkforce:report": "qkta build", "qkforce:analytics": "qkta analyze --output qreport-results/analytics.json", "typecheck": "tsc --noEmit", - "test:contract": "node --test ./test/unit/qkforce-config.test.mjs ./test/unit/qkforce-test-data.test.mjs ./test/unit/wdio-qkta-adapter.test.mjs", + "test:contract": "node --test ./test/unit/*.test.mjs", "check": "npm run typecheck && npm run test:contract" }, "devDependencies": { diff --git a/scripts/run-profile.mjs b/scripts/run-profile.mjs new file mode 100644 index 0000000..b9b1e0b --- /dev/null +++ b/scripts/run-profile.mjs @@ -0,0 +1,42 @@ +import { spawn } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { ExecutionDataManager, buildReport } from '@qacg/qk-test-analytics'; + +export async function executeAndReport(run, report) { + let exitCode = 1; + try { + exitCode = await run(); + } catch (error) { + console.error('[QKForce] Runner failed:', error.message); + } + try { + await report(); + } catch (error) { + console.error('[QKForce] Report generation failed:', error.message); + return exitCode || 1; + } + return exitCode; +} + +function runWdio() { + return new Promise((resolve, reject) => { + if (!process.env.npm_execpath) { + reject(new Error('Start the fixed profile with npm run qkforce:test')); + return; + } + // The command is fixed locally. No package manifest field supplies a shell command. + const child = spawn(process.execPath, [process.env.npm_execpath, 'run', 'test:wdio'], { + stdio: 'inherit', shell: false + }); + child.once('error', reject); + child.once('close', code => resolve(code ?? 1)); + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + new ExecutionDataManager().archiveCurrentReport(); + process.exitCode = await executeAndReport(runWdio, () => buildReport({ + reportsDir: 'qreport-results/media-bucket/reports', + output: 'qreport-results/index.html' + })); +} diff --git a/test/step-definitions/steps.ts b/test/step-definitions/steps.ts index 6c4f4dd..a3efe42 100644 --- a/test/step-definitions/steps.ts +++ b/test/step-definitions/steps.ts @@ -6,7 +6,7 @@ Given('the user is on the login page', async () => { await browser.url('/login'); }); -When(/^the user logs in with (\\w+) and (.+)$/, async (username: string, password: string) => { +When(/^the user logs in with (\w+) and (.+)$/, async (username: string, password: string) => { await (await getElement('loginPage.inputUsername')).setValue(username); await (await getElement('loginPage.inputPassword')).setValue(password); await (await getElement('loginPage.submitButton')).click(); diff --git a/test/support/qkforce-actions.mjs b/test/support/qkforce-actions.mjs new file mode 100644 index 0000000..31e59ed --- /dev/null +++ b/test/support/qkforce-actions.mjs @@ -0,0 +1,45 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +/** Adapter for the action palette emitted by QKForce. One instance per WDIO worker. */ +export function createQkfActions(browser, root = process.cwd()) { + let tail = Promise.resolve(); + const enqueue = action => { + const result = tail.then(action); + tail = result; + // Legacy saved snippets did not await calls. flush() surfaces failures at the step boundary. + void result.catch(() => {}); + return result; + }; + const element = selector => { + if (typeof selector !== 'string' || !selector.trim()) throw new Error('QKForce action requires a selector string'); + return browser.$(selector); + }; + const call = (selector, method, ...args) => enqueue(async () => (await element(selector))[method](...args)); + return Object.freeze({ + click: selector => call(selector, 'click'), + doubleClick: selector => call(selector, 'doubleClick'), + rightClick: selector => call(selector, 'click', { button: 'right' }), + enterValue: (selector, value) => call(selector, 'setValue', value), + addValue: (selector, value) => call(selector, 'addValue', value), + clearValue: selector => call(selector, 'clearValue'), + isVisible: selector => call(selector, 'isDisplayed'), + exists: selector => call(selector, 'isExisting'), + isEnabled: selector => call(selector, 'isEnabled'), + isClickable: selector => call(selector, 'isClickable'), + isSelected: selector => call(selector, 'isSelected'), + waitForVisible: selector => call(selector, 'waitForDisplayed'), + waitForExistence: selector => call(selector, 'waitForExist'), + waitForClickable: selector => call(selector, 'waitForClickable'), + waitForEnabled: selector => call(selector, 'waitForEnabled'), + goto: url => enqueue(() => browser.url(url)), + takeScreenshot: name => enqueue(async () => { + const directory = path.resolve(root, 'qreport-results', 'screenshots'); + await fs.mkdir(directory, { recursive: true }); + const label = String(name ?? 'screenshot').replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 80); + return browser.saveScreenshot(path.join(directory, `${label}-${randomUUID()}.png`)); + }), + async flush() { try { await tail; } finally { tail = Promise.resolve(); } } + }); +} diff --git a/test/support/qkforce-generated.mjs b/test/support/qkforce-generated.mjs new file mode 100644 index 0000000..0c7dec4 --- /dev/null +++ b/test/support/qkforce-generated.mjs @@ -0,0 +1,51 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const GENERATED_ROOT = 'test/qkforce'; +export const GENERATED_MANIFEST = `${GENERATED_ROOT}/manifest.json`; +const fail = message => { throw new Error(`Invalid QKForce generated workspace: ${message}`); }; + +function localFile(root, relative) { + if (typeof relative !== 'string' || !relative.startsWith(`${GENERATED_ROOT}/`) + || relative.includes('\\') || relative.split('/').some(part => !part || part === '.' || part === '..')) { + fail('unsafe generated file path'); + } + let current = path.resolve(root); + for (const segment of relative.split('/')) { + current = path.join(current, segment); + let stat; + try { stat = fs.lstatSync(current); } catch { fail('declared generated file is missing'); } + if (stat.isSymbolicLink()) fail('generated paths must not contain symlinks'); + } + if (!fs.statSync(current).isFile()) fail('declared generated input must be a file'); + return current; +} + +export function loadGeneratedWorkspace(root = process.cwd()) { + const manifestPath = path.join(root, GENERATED_MANIFEST); + if (!fs.existsSync(manifestPath)) { + // A local template checkout may run the example. An API package never may. + if (fs.existsSync(path.join(root, 'qkforce-package-manifest.json'))) fail('regenerate the project before execution'); + return null; + } + let manifest; + try { manifest = JSON.parse(fs.readFileSync(localFile(root, GENERATED_MANIFEST), 'utf8')); } + catch { fail('generated manifest is not readable JSON'); } + if (manifest?.schemaVersion !== 1 || !Array.isArray(manifest.features) + || manifest.features.length === 0) fail('select at least one project test case'); + if (new Set(manifest.features).size !== manifest.features.length) fail('duplicate feature paths'); + for (const file of manifest.features) { + if (typeof file !== 'string' || !file.startsWith(`${GENERATED_ROOT}/features/`) || !file.endsWith('.feature')) fail('invalid feature path'); + localFile(root, file); + } + if (manifest.steps !== `${GENERATED_ROOT}/steps.mjs` || manifest.selectors !== `${GENERATED_ROOT}/selectors.json`) fail('unsupported generated module paths'); + localFile(root, manifest.steps); + localFile(root, manifest.selectors); + return Object.freeze({ specs: manifest.features.map(file => `./${file}`), imports: [`./${manifest.steps}`] }); +} + +export function loadGeneratedSelectors(root = process.cwd()) { + const value = JSON.parse(fs.readFileSync(localFile(root, `${GENERATED_ROOT}/selectors.json`), 'utf8')); + if (!value || typeof value !== 'object' || Array.isArray(value)) fail('selectors must be an object'); + return value; +} diff --git a/test/unit/qkforce-generated.test.mjs b/test/unit/qkforce-generated.test.mjs new file mode 100644 index 0000000..c2eda22 --- /dev/null +++ b/test/unit/qkforce-generated.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { loadGeneratedWorkspace, loadGeneratedSelectors } from '../support/qkforce-generated.mjs'; +import { createQkfActions } from '../support/qkforce-actions.mjs'; + +function fixture(t, features=['test/qkforce/features/project.feature']) { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'qkf-generated-')); + t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + fs.mkdirSync(path.join(root,'test/qkforce/features'),{recursive:true}); + fs.writeFileSync(path.join(root,'test/qkforce/features/project.feature'),'Feature: project'); + fs.writeFileSync(path.join(root,'test/qkforce/steps.mjs'),''); + fs.writeFileSync(path.join(root,'test/qkforce/selectors.json'),'{}'); + const manifest={schemaVersion:1,features,steps:'test/qkforce/steps.mjs',selectors:'test/qkforce/selectors.json'}; + fs.writeFileSync(path.join(root,'test/qkforce/manifest.json'),JSON.stringify(manifest)); + return root; +} + +test('runs exactly generated cases and never template or stale feature globs', t=>{ + const root=fixture(t); + fs.writeFileSync(path.join(root,'test/qkforce/features/stale.feature'),'Feature: stale'); + assert.deepEqual(loadGeneratedWorkspace(root),{specs:['./test/qkforce/features/project.feature'],imports:['./test/qkforce/steps.mjs']}); + assert.deepEqual(loadGeneratedSelectors(root),{}); +}); +test('an empty generated selection fails instead of running the template sample',t=>{ + assert.throws(()=>loadGeneratedWorkspace(fixture(t,[])),/at least one/); +}); +test('an API package without generated selection cannot fall back to samples',t=>{ + const root=fixture(t);fs.unlinkSync(path.join(root,'test/qkforce/manifest.json')); + assert.equal(loadGeneratedWorkspace(root),null); + fs.writeFileSync(path.join(root,'qkforce-package-manifest.json'),'{}'); + assert.throws(()=>loadGeneratedWorkspace(root),/regenerate/); +}); +test('unsafe, missing, duplicate and symlinked feature inputs are rejected',t=>{ + for(const features of [['test/qkforce/features/../../secret.feature'],['test/qkforce/features/missing.feature'],['test/qkforce/features/project.feature','test/qkforce/features/project.feature']]) { + assert.throws(()=>loadGeneratedWorkspace(fixture(t,features))); + } + const root=fixture(t); + const feature=path.join(root,'test/qkforce/features/project.feature'); + fs.unlinkSync(feature);fs.symlinkSync(path.join(root,'test/qkforce/steps.mjs'),feature); + assert.throws(()=>loadGeneratedWorkspace(root),/symlinks/); +}); +test('legacy unawaited actions execute sequentially and finish before flush returns',async()=>{ + const events=[]; + const browser={url:async url=>{await Promise.resolve();events.push(url);},$:async selector=>({click:async()=>events.push(selector)})}; + const qkf=createQkfActions(browser); + qkf.goto('/login');qkf.click('#submit'); + await qkf.flush();assert.deepEqual(events,['/login','#submit']); +}); +test('queued action failures fail the step and reset for the next scenario',async()=>{ + let fail=true; + const qkf=createQkfActions({url:async()=>{if(fail)throw new Error('navigation failed');}}); + qkf.goto('/login');await assert.rejects(qkf.flush(),/navigation failed/); + fail=false;qkf.goto('/ok');await qkf.flush(); +}); +test('all palette element actions map to real WDIO methods',async()=>{ + const calls=[]; + const element=new Proxy({}, {get:(_,method)=>method === 'then' ? undefined : (...args)=>{calls.push([method,...args]);return true;}}); + const qkf=createQkfActions({$:()=>element}); + for(const method of ['click','doubleClick','rightClick','enterValue','addValue','clearValue','isVisible','exists','isEnabled','isClickable','isSelected','waitForVisible','waitForExistence','waitForClickable','waitForEnabled']) qkf[method]('#x','value'); + await qkf.flush(); + assert.equal(calls.length,15);assert.deepEqual(calls[2],['click',{button:'right'}]);assert.deepEqual(calls[3],['setValue','value']); +}); +test('screenshots stay inside the results directory',async t=>{ + const root=fixture(t);let saved; + const qkf=createQkfActions({saveScreenshot:async file=>{saved=file;}},root); + qkf.takeScreenshot('../../outside');await qkf.flush(); + assert.equal(path.dirname(saved),path.join(root,'qreport-results/screenshots')); +}); diff --git a/test/unit/run-profile.test.mjs b/test/unit/run-profile.test.mjs new file mode 100644 index 0000000..a46671a --- /dev/null +++ b/test/unit/run-profile.test.mjs @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { executeAndReport } from '../../scripts/run-profile.mjs'; + +test('builds the report after both passing and failing runs, preserving runner status', async () => { + for (const code of [0, 1, 7]) { + const events = []; + const result = await executeAndReport(async () => { events.push('runner'); return code; }, async () => { events.push('report'); }); + assert.equal(result, code); + assert.deepEqual(events, ['runner', 'report']); + } +}); + +test('attempts reporting after runner startup failure', async () => { + let reported = false; + assert.equal(await executeAndReport(async () => { throw new Error('spawn failed'); }, () => { reported = true; }), 1); + assert.ok(reported); +}); + +test('a report failure cannot convert a failed runner to success or hide its exit code', async () => { + for (const code of [0, 7]) { + assert.equal(await executeAndReport(async () => code, () => { throw new Error('unwritable report'); }), code || 1); + } +}); diff --git a/wdio.conf.ts b/wdio.conf.ts index f55d7ed..e364e4b 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -1,9 +1,11 @@ import { createWdioCucumberAdapter } from '@qacg/qk-test-analytics/adapters/wdio-cucumber'; import { loadQkforceRunConfig } from './test/support/qkforce-config.mjs'; import { loadQkforceTestData } from './test/support/qkforce-test-data.mjs'; +import { loadGeneratedWorkspace } from './test/support/qkforce-generated.mjs'; const qkforce = loadQkforceRunConfig(); loadQkforceTestData(); +const generated = loadGeneratedWorkspace(); const analytics = createWdioCucumberAdapter({ capture: 'on-failure', @@ -13,7 +15,7 @@ const analytics = createWdioCucumberAdapter({ export const config: WebdriverIO.Config = { runner: 'local', tsConfigPath: './tsconfig.json', - specs: ['./test/features/**/*.feature'], + specs: generated?.specs ?? ['./test/features/**/*.feature'], maxInstances: 1, capabilities: [{ browserName: 'chrome', 'goog:chromeOptions': { args: ['--headless=new','--disable-gpu','--no-sandbox','--window-size=1440,900'] } }], logLevel: 'error', @@ -23,6 +25,6 @@ export const config: WebdriverIO.Config = { connectionRetryCount: 3, framework: 'cucumber', reporters: ['spec'], - cucumberOpts: { import: ['./test/step-definitions/**/*.ts'], language: 'en', timeout: 60_000, retry: 0, failFast: false, strict: true }, + cucumberOpts: { import: generated?.imports ?? ['./test/step-definitions/**/*.ts'], language: 'en', timeout: 60_000, retry: 0, failFast: false, strict: true }, ...analytics.hooks };