[@typespec/spector] - Enable Multiple Scenario Paths (#4836)

Currently we could execute the `serve`/`server-test`/... command such
as:

`pnpm .\cmd\cli.mjs serve ..\http-specs\specs`

i.e we could provide only one path of scenarios to execute. But, since
we separated the scenarios to azure and non-azure specific cases to 2
different packages, we need a way to execute commands such as:

`pnpm .\cmd\cli.mjs serve ..\http-specs\specs
..\..\..\typespec-azure\packages\azure-http-specs\specs\`

The same procedure must be followed in all other commands such as
`validate-scenarios`, `generate-scenarios-summary`, etc. Since we have
to do it for all the commands and we need to change in all the places,
it is much simpler to do it in one location where we copy the files to
one common temp location and execute from there.

Also, in this PR, I have removed the `private: true` option for all the
5 packages to start the rollout process.

Please review and approve the PR. Thanks

.chronus/changes/MultipleScenariosII-2024-9-24-16-24-37.md

@@ -0,0 +1,10 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/http-specs"
+  - "@typespec/spec-api"
+  - "@typespec/spec-coverage-sdk"
+  - "@typespec/spector"
+---
+
+Release cadl-ranch migrated packages to enable SDK and service testing

packages/http-specs/package.json

@@ -1,7 +1,6 @@
 {
   "name": "@typespec/http-specs",
-  "private": true,
-  "version": "0.1.0",
+  "version": "0.1.0-alpha.0",
   "description": "Spec scenarios and mock apis",
   "main": "dist/index.js",
   "type": "module",

packages/spec-api/package.json

@@ -1,10 +1,9 @@
 {
   "name": "@typespec/spec-api",
-  "version": "0.1.0",
+  "version": "0.1.0-alpha.0",
   "description": "Spec api to implement mock api",
   "main": "dist/index.js",
   "type": "module",
-  "private": true,
   "scripts": {
     "watch": "tsc -p ./tsconfig.build.json --watch",
     "build": "tsc -p ./tsconfig.build.json",

packages/spec-coverage-sdk/package.json

@@ -1,9 +1,8 @@
 {
   "name": "@typespec/spec-coverage-sdk",
-  "version": "0.1.0",
+  "version": "0.1.0-alpha.0",
   "description": "Spec utility to manage the reported coverage",
   "main": "dist/index.js",
-  "private": true,
   "type": "module",
   "scripts": {
     "watch": "tsc -p ./tsconfig.build.json --watch",

packages/spector/package.json

@@ -1,14 +1,13 @@
 {
   "name": "@typespec/spector",
-  "version": "0.1.0",
+  "version": "0.1.0-alpha.0",
   "description": "Typespec Core Tool to validate, run mock api, collect coverage.",
   "exports": {
     ".": {
       "import": "./dist/src/index.js",
       "typespec": "./lib/main.tsp"
     }
   },
-  "private": true,
   "type": "module",
   "bin": {
     "tsp-spector": "./cmd/cli.mjs"

packages/spector/src/actions/check-coverage.ts

@@ -11,9 +11,11 @@ export interface CheckCoverageConfig {
   coverageFiles: string[];
   mergedCoverageFile: string;
   ignoreNotImplemented?: boolean;
+  exitDueToPreviousError?: boolean;
+  hasMoreScenarios?: boolean;
 }
 
-export async function checkCoverage(config: CheckCoverageConfig) {
+export async function checkCoverage(config: CheckCoverageConfig): Promise<boolean> {
   const inputCoverageFiles = (
     await Promise.all(config.coverageFiles.map((x) => findFilesFromPattern(x)))
   ).flat();
@@ -81,7 +83,16 @@ export async function checkCoverage(config: CheckCoverageConfig) {
   const coverageReport = createCoverageReport(config.scenariosPath, results);
   await writeFile(config.mergedCoverageFile, JSON.stringify(coverageReport, null, 2));
 
-  if (diagnosticsReporter.diagnostics.length) {
-    process.exit(1);
+  if (diagnosticsReporter.diagnostics.length === 0) {
+    if (config.exitDueToPreviousError) {
+      process.exit(1);
+    }
+    return false;
+  } else {
+    if (config.hasMoreScenarios) {
+      return true;
+    } else {
+      process.exit(1);
+    }
   }
 }

packages/spector/src/actions/generate-scenario-summary.ts

@@ -1,4 +1,4 @@
-import { writeFile } from "fs/promises";
+import { readFile, writeFile } from "fs/promises";
 import pc from "picocolors";
 import prettier from "prettier";
 import type { Scenario, ScenarioEndpoint } from "../lib/decorators.js";
@@ -8,20 +8,28 @@ import { loadScenarios } from "../scenarios-resolver.js";
 export interface GenerateScenarioSummaryConfig {
   scenariosPath: string;
   outputFile: string;
+  overrideOutputFile?: boolean;
 }
 
 export async function generateScenarioSummary({
   scenariosPath,
   outputFile,
+  overrideOutputFile,
 }: GenerateScenarioSummaryConfig) {
   const [scenarios, diagnostics] = await loadScenarios(scenariosPath);
 
   if (diagnostics.length > 0) {
     process.exit(-1);
   }
 
-  const summary = await createScenarioSummary(scenarios);
-  await writeFile(outputFile, summary);
+  let summary = await createScenarioSummary(scenarios);
+  if (overrideOutputFile) {
+    const existingContent = await readFile(outputFile, "utf-8");
+    summary = summary.replace(/# Spec Project summary/, "");
+    await writeFile(outputFile, `${existingContent}\n${summary}`);
+  } else {
+    await writeFile(outputFile, summary);
+  }
   logger.info(`${pc.green("✓")} Scenario summary generated at ${outputFile}.`);
 }
 

packages/spector/src/actions/serve.ts

@@ -1,12 +1,13 @@
 import { spawn } from "child_process";
 import fetch from "node-fetch";
+import { resolve } from "path";
 import { MockApiApp } from "../app/app.js";
 import { AdminUrls } from "../constants.js";
 import { logger } from "../logger.js";
 import { ensureScenariosPathExists } from "../utils/index.js";
 
 export interface ServeConfig {
-  scenariosPath: string;
+  scenariosPath: string | string[];
   coverageFile: string;
   port: number;
 }
@@ -16,7 +17,14 @@ export interface StopConfig {
 }
 
 export async function serve(config: ServeConfig) {
-  await ensureScenariosPathExists(config.scenariosPath);
+  if (Array.isArray(config.scenariosPath)) {
+    for (let idx = 0; idx < config.scenariosPath.length; idx++) {
+      config.scenariosPath[idx] = resolve(process.cwd(), config.scenariosPath[idx]);
+      await ensureScenariosPathExists(config.scenariosPath[idx]);
+    }
+  } else {
+    await ensureScenariosPathExists(config.scenariosPath);
+  }
 
   const server = new MockApiApp({
     port: config.port,
@@ -30,12 +38,15 @@ export async function startInBackground(config: ServeConfig) {
   return new Promise<void>((resolve) => {
     const [nodeExe, entrypoint] = process.argv;
     logger.info(`Starting server in background at port ${config.port}`);
+    const scenariosPath = Array.isArray(config.scenariosPath)
+      ? config.scenariosPath.join(" ")
+      : config.scenariosPath;
     const cp = spawn(
       nodeExe,
       [
         entrypoint,
         "serve",
-        config.scenariosPath,
+        scenariosPath,
         "--port",
         config.port.toString(),
         "--coverageFile",

packages/spector/src/actions/validate-mock-apis.ts

@@ -6,9 +6,15 @@ import { createDiagnosticReporter } from "../utils/diagnostic-reporter.js";
 
 export interface ValidateMockApisConfig {
   scenariosPath: string;
+  exitDueToPreviousError?: boolean;
+  hasMoreScenarios?: boolean;
 }
 
-export async function validateMockApis({ scenariosPath }: ValidateMockApisConfig) {
+export async function validateMockApis({
+  scenariosPath,
+  exitDueToPreviousError,
+  hasMoreScenarios,
+}: ValidateMockApisConfig) {
   const mockApis = await loadScenarioMockApiFiles(scenariosPath);
   const scenarioFiles = await findScenarioSpecFiles(scenariosPath);
 
@@ -69,7 +75,17 @@ export async function validateMockApis({ scenariosPath }: ValidateMockApisConfig
     }
   }
 
-  if (diagnostics.diagnostics.length) {
-    process.exit(1);
+  if (diagnostics.diagnostics.length === 0) {
+    if (exitDueToPreviousError && !hasMoreScenarios) {
+      process.exit(1);
+    }
+    if (exitDueToPreviousError) return exitDueToPreviousError;
+    else return false;
+  } else {
+    if (hasMoreScenarios) {
+      return true;
+    } else {
+      process.exit(1);
+    }
   }
 }

packages/spector/src/actions/validate-scenarios.ts

@@ -4,14 +4,29 @@ import { loadScenarios } from "../scenarios-resolver.js";
 
 export interface ValidateScenarioConfig {
   scenariosPath: string;
+  exitDueToPreviousError?: boolean;
+  hasMoreScenarios?: boolean;
 }
 
-export async function validateScenarios({ scenariosPath }: ValidateScenarioConfig) {
+export async function validateScenarios({
+  scenariosPath,
+  exitDueToPreviousError,
+  hasMoreScenarios,
+}: ValidateScenarioConfig) {
   const [_, diagnostics] = await loadScenarios(scenariosPath);
 
-  if (diagnostics.length > 0) {
-    process.exit(-1);
-  } else {
+  if (diagnostics.length === 0) {
     logger.info(`${pc.green("✓")} All scenarios are valid.`);
+    if (exitDueToPreviousError && !hasMoreScenarios) {
+      process.exit(-1);
+    }
+    if (exitDueToPreviousError) return exitDueToPreviousError;
+    else return false;
+  } else {
+    if (hasMoreScenarios) {
+      return true;
+    } else {
+      process.exit(-1);
+    }
   }
 }

packages/spector/src/app/app.ts

@@ -1,4 +1,5 @@
 import { MockApiDefinition, MockRequest, RequestExt, ScenarioMockApi } from "@typespec/spec-api";
+import { ScenariosMetadata } from "@typespec/spec-coverage-sdk";
 import { Response, Router } from "express";
 import { getScenarioMetadata } from "../coverage/common.js";
 import { CoverageTracker } from "../coverage/coverage-tracker.js";
@@ -9,6 +10,11 @@ import { MockApiServer } from "../server/index.js";
 import { ApiMockAppConfig } from "./config.js";
 import { processRequest } from "./request-processor.js";
 
+export interface ScenariosAndScenariosMetadata {
+  scenarios: Record<string, ScenarioMockApi>;
+  scenariosMetadata: ScenariosMetadata;
+}
+
 export class MockApiApp {
   private router = Router();
   private server: MockApiServer;
@@ -22,14 +28,27 @@ export class MockApiApp {
   public async start(): Promise<void> {
     this.server.use("/", internalRouter);
 
-    const scenarios = await loadScenarioMockApis(this.config.scenarioPath);
-    this.coverageTracker.setScenarios(
-      await getScenarioMetadata(this.config.scenarioPath),
-      scenarios,
-    );
-    for (const [name, scenario] of Object.entries(scenarios)) {
-      this.registerScenario(name, scenario);
+    const ScenariosAndScenariosMetadata: ScenariosAndScenariosMetadata[] = [];
+    if (Array.isArray(this.config.scenarioPath)) {
+      for (let idx = 0; idx < this.config.scenarioPath.length; idx++) {
+        const scenarios = await loadScenarioMockApis(this.config.scenarioPath[idx]);
+        const scenariosMetadata = await getScenarioMetadata(this.config.scenarioPath[idx]);
+        ScenariosAndScenariosMetadata.push({ scenarios, scenariosMetadata });
+      }
+    } else {
+      const scenarios = await loadScenarioMockApis(this.config.scenarioPath);
+      const scenariosMetadata = await getScenarioMetadata(this.config.scenarioPath);
+      ScenariosAndScenariosMetadata.push({ scenarios, scenariosMetadata });
+    }
+
+    this.coverageTracker.setScenarios(ScenariosAndScenariosMetadata);
+
+    for (const { scenarios } of ScenariosAndScenariosMetadata) {
+      for (const [name, scenario] of Object.entries(scenarios)) {
+        this.registerScenario(name, scenario);
+      }
     }
+
     this.router.get("/.coverage", (req, res) => {
       res.json(this.coverageTracker.computeCoverage());
     });

packages/spector/src/app/config.ts

@@ -7,7 +7,7 @@ export interface ApiMockAppConfig {
   /**
    * Path where are the scenarios.
    */
-  scenarioPath: string;
+  scenarioPath: string | string[];
 
   /**
    * Coverage file Path.