Update test framework: fix run_tests.py to support all test files, add auto-import-check for test files

node_modules/mcp-evals/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

node_modules/mcp-evals/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

node_modules/mcp-evals/dist/cli.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+import { runAllEvals } from './index.js';
+import * as dotenv from 'dotenv';
+import { loadYamlEvalConfig } from './yaml-loader.js';
+import path from 'path';
+import { createRequire } from 'module';
+// Load .env file
+dotenv.config();
+// Register tsx as a loader
+const require = createRequire(import.meta.url);
+require('tsx');
+async function main() {
+    const userEvalsPath = process.argv[2];
+    const userServerPath = process.argv[3];
+    if (!userEvalsPath) {
+        console.error('Please provide a path to your evals file');
+        console.error('Usage: npx mcp-eval <evals-path> <server-path>');
+        console.error('       <evals-path> can be a .ts/.js or .yaml/.yml file');
+        process.exit(1);
+    }
+    if (!userServerPath) {
+        console.error('Please provide a path to your server file');
+        console.error('Usage: npx mcp-eval <evals-path> <server-path>');
+        console.error('       <evals-path> can be a .ts/.js or .yaml/.yml file');
+        process.exit(1);
+    }
+    const absoluteEvalsPath = path.resolve(process.cwd(), userEvalsPath);
+    const absoluteServerPath = path.resolve(process.cwd(), userServerPath);
+    console.log('Running evals file:', absoluteEvalsPath);
+    console.log('Using server file:', absoluteServerPath);
+    try {
+        let config;
+        // Check file extension to determine how to load the config
+        const fileExtension = path.extname(absoluteEvalsPath).toLowerCase();
+        if (fileExtension === '.yaml' || fileExtension === '.yml') {
+            // Load YAML configuration
+            config = loadYamlEvalConfig(absoluteEvalsPath, absoluteServerPath);
+        }
+        else {
+            // Import the TypeScript/JavaScript file 
+            const module = await import(absoluteEvalsPath);
+            config = module.default;
+            if (!config || !config.evals) {
+                console.error('Invalid config: must export a default config with evals array');
+                process.exit(1);
+            }
+        }
+        console.log('Running all evaluations...\n');
+        const results = await runAllEvals(config, absoluteServerPath);
+        console.log('\nEvaluation Results:');
+        for (const [name, result] of results.entries()) {
+            console.log(`\n${name}:`);
+            console.log(JSON.stringify(result, null, 2));
+        }
+        process.exit(0);
+    }
+    catch (error) {
+        console.error('Error running evaluations:', error);
+        process.exit(1);
+    }
+}
+main().catch(error => {
+    console.error('Error running evaluations:', error);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

node_modules/mcp-evals/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC,iBAAiB;AACjB,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,2BAA2B;AAC3B,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,OAAO,CAAC,KAAK,CAAC,CAAC;AAEf,KAAK,UAAU,IAAI;IACjB,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC1D,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAChE,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAChE,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC,CAAC;IACrE,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;IAEvE,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,iBAAiB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,kBAAkB,CAAC,CAAC;IAEtD,IAAI,CAAC;QACH,IAAI,MAAkB,CAAC;QAEvB,2DAA2D;QAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE,CAAC;QAEpE,IAAI,aAAa,KAAK,OAAO,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;YAC1D,0BAA0B;YAC1B,MAAM,GAAG,kBAAkB,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC/C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;YAExB,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC7B,OAAO,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAE9D,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;IACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

node_modules/mcp-evals/dist/index.d.ts

@@ -0,0 +1,9 @@
+import { type LanguageModel } from "ai";
+import { EvalConfig } from './types.js';
+export declare function runEvals(model: LanguageModel | undefined, prompt: string, serverPath: string): Promise<string>;
+export declare function grade(model: LanguageModel | undefined, prompt: string, serverPath?: string): Promise<string>;
+export declare function runAllEvals(config: EvalConfig, serverPath: string): Promise<Map<string, any>>;
+export * from './types.js';
+export { metrics } from './metrics.js';
+export type { MetricsConfig } from './metrics.js';
+//# sourceMappingURL=index.d.ts.map

node_modules/mcp-evals/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,aAAa,EACnB,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,wBAAsB,QAAQ,CAAC,KAAK,EAAE,aAAa,YAAa,EAAE,MAAM,EAAE,MAAM,EAAC,UAAU,EAAE,MAAM,mBAuClG;AAED,wBAAsB,KAAK,CAAC,KAAK,EAAE,aAAa,YAAa,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,mBAwCjG;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,6BAiCvE;AAGD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC"}

node_modules/mcp-evals/dist/index.js

@@ -0,0 +1,112 @@
+import { Experimental_StdioMCPTransport } from "ai/mcp-stdio";
+import { experimental_createMCPClient, streamText, } from "ai";
+import { openai } from "@ai-sdk/openai";
+const defaultModel = openai("gpt-4o");
+export async function runEvals(model = defaultModel, prompt, serverPath) {
+    const transport = new Experimental_StdioMCPTransport({
+        command: "tsx",
+        args: [serverPath],
+        env: Object.fromEntries(Object.entries(process.env).filter(([_, v]) => v !== undefined))
+    });
+    const client = await experimental_createMCPClient({
+        transport,
+    });
+    const tools = await client.tools();
+    try {
+        const result = streamText({
+            model,
+            tools,
+            system: "You are an assistant responsible for evaluating the results of calling various tools. Given the user's query, use the tools available to you to answer the question.",
+            prompt,
+            maxRetries: 1,
+            maxSteps: 10,
+            onError: ({ error }) => {
+                console.error(error);
+            },
+        });
+        let fullText = '';
+        for await (const chunk of result.fullStream) {
+            if (chunk.type === 'text-delta') {
+                fullText += chunk.textDelta;
+            }
+        }
+        return fullText;
+    }
+    catch (error) {
+        console.error('Error in runEvals:', error);
+        throw error;
+    }
+}
+export async function grade(model = defaultModel, prompt, serverPath) {
+    const finalServerPath = serverPath || process.argv[3]; // Use provided serverPath or CLI args
+    if (!finalServerPath) {
+        throw new Error('Server path not provided');
+    }
+    const result = await runEvals(model, prompt, finalServerPath);
+    const evalSystemPromt = `You are an expert evaluator assessing how well an LLM answers a given question. Review the provided answer and score it from 1 to 5 in each of the following categories:
+        Accuracy – Does the answer contain factual errors or hallucinations?
+        Completeness – Does the answer fully address all parts of the question?
+        Relevance – Is the information directly related to the question?
+        Clarity – Is the explanation easy to understand and well-structured?
+        Reasoning – Does the answer show logical thinking or provide evidence or rationale?
+        Return your evaluation as a JSON object in the format:
+        {
+            "accuracy": 1-5,
+            "completeness": 1-5,
+            "relevance": 1-5,
+            "clarity": 1-5,
+            "reasoning": 1-5,
+            "overall_comments": "A short paragraph summarizing the strengths and weaknesses of the answer."
+        }`;
+    const evalPromt = `Here is the user input: ${prompt}
+  Here is the LLM's answer: ${result}`;
+    const evalResult = streamText({
+        model,
+        maxRetries: 1,
+        maxSteps: 10,
+        system: evalSystemPromt,
+        prompt: evalPromt,
+        onError: ({ error }) => {
+            console.error(error);
+        },
+    });
+    for await (const _ of evalResult.fullStream) {
+    }
+    return await evalResult.text;
+}
+export async function runAllEvals(config, serverPath) {
+    const results = new Map();
+    let transport;
+    try {
+        transport = new Experimental_StdioMCPTransport({
+            command: "tsx",
+            args: [serverPath],
+            env: Object.fromEntries(Object.entries(process.env).filter(([_, v]) => v !== undefined))
+        });
+        const client = await experimental_createMCPClient({
+            transport,
+        });
+        for (const evaluation of config.evals) {
+            console.log(`Running ${evaluation.name}...`);
+            try {
+                const result = await evaluation.run(config.model);
+                results.set(evaluation.name, result);
+            }
+            catch (error) {
+                console.error(`Error running ${evaluation.name}:`, error);
+                results.set(evaluation.name, { error: error instanceof Error ? error.message : String(error) });
+            }
+        }
+        return results;
+    }
+    finally {
+        // Clean up the transport
+        if (transport) {
+            await transport.close?.();
+        }
+    }
+}
+// Export everything needed by consumers
+export * from './types.js';
+export { metrics } from './metrics.js';
+//# sourceMappingURL=index.js.map

node_modules/mcp-evals/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,8BAA8B,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EACL,4BAA4B,EAC5B,UAAU,GAEX,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAqB,YAAY,EAAE,MAAc,EAAC,UAAkB;IACjG,MAAM,SAAS,GAAG,IAAI,8BAA8B,CAAC;QACnD,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,UAAU,CAAC;QAClB,GAAG,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAA2B;KACnH,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;QAChD,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAEnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,CAAC;YACxB,KAAK;YACL,KAAK;YACL,MAAM,EACJ,sKAAsK;YACxK,MAAM;YACN,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,QAAqB,YAAY,EAAE,MAAc,EAAE,UAAmB;IAChG,MAAM,eAAe,GAAG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,sCAAsC;IAC7F,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG;;;;;;;;;;;;;;UAchB,CAAA;IACR,MAAM,SAAS,GAAG,2BAA2B,MAAM;8BACvB,MAAM,EAAE,CAAA;IACpC,MAAM,UAAU,GAAG,UAAU,CAAC;QACpB,KAAK;QACL,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,EAAE;QACZ,MAAM,EAAE,eAAe;QACvB,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;YACrB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;KACN,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;IAE9C,CAAC;IAED,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAkB,EAAE,UAAkB;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAe,CAAC;IACvC,IAAI,SAAS,CAAC;IAEd,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,8BAA8B,CAAC;YAC7C,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,GAAG,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAA2B;SACnH,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;YAChD,SAAS;SACV,CAAC,CAAC;QAEH,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iBAAiB,UAAU,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC1D,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClG,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,yBAAyB;QACzB,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;AACH,CAAC;AAED,wCAAwC;AACxC,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}

node_modules/mcp-evals/dist/metrics.d.ts

@@ -0,0 +1,23 @@
+export interface MetricsConfig {
+    metricsPort?: number;
+    otelExporterUrl?: string;
+    serviceName?: string;
+    debug?: boolean;
+    enableTracing?: boolean;
+    otelEndpoint?: string;
+}
+declare class Metrics {
+    private static instance;
+    private sdk;
+    private tracer;
+    private registry;
+    private metrics;
+    private metricsServer;
+    private constructor();
+    static initialize(metricsPort?: number, options?: {}): Metrics;
+}
+export declare const metrics: {
+    initialize: (metricsPort?: number, options?: {}) => Metrics;
+};
+export {};
+//# sourceMappingURL=metrics.d.ts.map

node_modules/mcp-evals/dist/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAYA,MAAM,WAAW,aAAa;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAEH,cAAM,OAAO;IACX,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAU;IACjC,OAAO,CAAC,GAAG,CAAU;IACrB,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,OAAO,CAIb;IACF,OAAO,CAAC,aAAa,CAAsB;IAE3C,OAAO;IAkEP,MAAM,CAAC,UAAU,CAAC,WAAW,SAAO,EAAE,OAAO,KAAK;CAkEnD;AAED,eAAO,MAAM,OAAO;;CAEnB,CAAC"}

node_modules/mcp-evals/dist/metrics.js

@@ -0,0 +1,142 @@
+// @ts-nocheck
+// metrics.ts
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { trace, SpanStatusCode } from '@opentelemetry/api';
+import { Registry, Counter, Histogram } from 'prom-client';
+import express from 'express';
+class Metrics {
+    static instance;
+    sdk;
+    tracer;
+    registry;
+    metrics;
+    metricsServer;
+    constructor(options = {}) {
+        // Initialize OpenTelemetry if tracing is enabled
+        if (options.enableTracing) {
+            // Default to local collector endpoint if not specified
+            const otelEndpoint = options.otelEndpoint || 'http://localhost:4318/v1/traces';
+            console.log(`Configuring OpenTelemetry with endpoint: ${otelEndpoint}`);
+            const exporterOptions = {
+                url: otelEndpoint,
+                headers: {}, // Additional headers if needed
+                concurrencyLimit: 10
+            };
+            this.sdk = new NodeSDK({
+                resource: resourceFromAttributes({
+                    [SemanticResourceAttributes.SERVICE_NAME]: options.serviceName || 'mcp-server',
+                }),
+                spanProcessor: new SimpleSpanProcessor(new OTLPTraceExporter(exporterOptions)),
+            });
+            // Start the SDK
+            try {
+                this.sdk.start();
+                console.log(`OpenTelemetry SDK started successfully, sending traces to ${otelEndpoint}`);
+            }
+            catch (error) {
+                console.error('Failed to start OpenTelemetry SDK:', error);
+            }
+        }
+        else {
+            console.log('OpenTelemetry tracing is disabled');
+        }
+        this.tracer = trace.getTracer('mcp-tracer');
+        // Initialize Prometheus metrics
+        this.registry = new Registry();
+        this.metrics = {
+            toolCalls: new Counter({
+                name: 'mcp_tool_calls_total',
+                help: 'Total number of tool calls',
+                labelNames: ['tool_name'],
+                registers: [this.registry]
+            }),
+            toolErrors: new Counter({
+                name: 'mcp_tool_errors_total',
+                help: 'Total number of tool errors',
+                labelNames: ['tool_name'],
+                registers: [this.registry]
+            }),
+            toolLatency: new Histogram({
+                name: 'mcp_tool_latency_seconds',
+                help: 'Tool call latency in seconds',
+                labelNames: ['tool_name'],
+                buckets: [0.1, 0.5, 1, 2, 5],
+                registers: [this.registry]
+            })
+        };
+        // Setup metrics server
+        this.metricsServer = express();
+        this.metricsServer.get('/metrics', async (req, res) => {
+            res.set('Content-Type', this.registry.contentType);
+            res.end(await this.registry.metrics());
+        });
+    }
+    static initialize(metricsPort = 9090, options = {}) {
+        if (!Metrics.instance) {
+            Metrics.instance = new Metrics(options);
+            // Patch the McpServer prototype to add instrumentation
+            const originalTool = McpServer.prototype.tool;
+            McpServer.prototype.tool = function (...args) {
+                // The tool method can be called with multiple signatures:
+                // (name, description, parameters, handler)
+                // (name, description, parameters, options, handler)
+                const name = args[0];
+                const description = args[1];
+                const parameters = args[2];
+                // Last arg is always the handler
+                const handler = args[args.length - 1];
+                // Check if options are present (second to last argument when length > 4)
+                const hasOptions = args.length > 4;
+                const options = hasOptions ? args[3] : undefined;
+                // Create new args array with wrapped handler
+                const newArgs = hasOptions
+                    ? [name, description, parameters, options]
+                    : [name, description, parameters];
+                // Add wrapped handler that includes instrumentation
+                newArgs.push(async (...handlerArgs) => {
+                    const span = Metrics.instance.tracer.startSpan(`tool.${name}`);
+                    const startTime = process.hrtime();
+                    try {
+                        // Increment tool calls counter
+                        Metrics.instance.metrics.toolCalls.inc({ tool_name: name });
+                        const result = await handler(...handlerArgs);
+                        span.setStatus({ code: SpanStatusCode.OK });
+                        return result;
+                    }
+                    catch (error) {
+                        // Increment error counter
+                        Metrics.instance.metrics.toolErrors.inc({ tool_name: name });
+                        span.setStatus({
+                            code: SpanStatusCode.ERROR,
+                            message: error instanceof Error ? error.message : String(error)
+                        });
+                        throw error;
+                    }
+                    finally {
+                        // Record latency
+                        const [seconds, nanoseconds] = process.hrtime(startTime);
+                        const duration = seconds + nanoseconds / 1e9;
+                        Metrics.instance.metrics.toolLatency.observe({ tool_name: name }, duration);
+                        span.end();
+                    }
+                });
+                // Call original tool method with new args
+                return originalTool.apply(this, newArgs);
+            };
+            // Start metrics server
+            Metrics.instance.metricsServer.listen(metricsPort, () => {
+                console.log(`Metrics server listening on port ${metricsPort}`);
+            });
+        }
+        return Metrics.instance;
+    }
+}
+export const metrics = {
+    initialize: (metricsPort = 9090, options = {}) => Metrics.initialize(metricsPort, options)
+};
+//# sourceMappingURL=metrics.js.map

node_modules/mcp-evals/dist/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,aAAa;AACb,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,0BAA0B,EAAE,MAAM,qCAAqC,CAAC;AACjF,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAC;AAC5E,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,OAAO,MAAM,SAAS,CAAC;AAW9B,MAAM,OAAO;IACH,MAAM,CAAC,QAAQ,CAAU;IACzB,GAAG,CAAU;IACb,MAAM,CAAM;IACZ,QAAQ,CAAW;IACnB,OAAO,CAIb;IACM,aAAa,CAAsB;IAE3C,YAAoB,OAAO,GAAG,EAAE;QAC9B,iDAAiD;QACjD,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,uDAAuD;YACvD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,iCAAiC,CAAC;YAC/E,OAAO,CAAC,GAAG,CAAC,4CAA4C,YAAY,EAAE,CAAC,CAAC;YAExE,MAAM,eAAe,GAAG;gBACpB,GAAG,EAAE,YAAY;gBACjB,OAAO,EAAE,EAAE,EAAE,+BAA+B;gBAC5C,gBAAgB,EAAE,EAAE;aACvB,CAAC;YAEF,IAAI,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC;gBACnB,QAAQ,EAAE,sBAAsB,CAAC;oBAC7B,CAAC,0BAA0B,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,YAAY;iBACjF,CAAC;gBACF,aAAa,EAAE,IAAI,mBAAmB,CAAC,IAAI,iBAAiB,CAAC,eAAe,CAAC,CAAC;aACjF,CAAC,CAAC;YAEH,gBAAgB;YAChB,IAAI,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,6DAA6D,YAAY,EAAE,CAAC,CAAC;YAC7F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC/D,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAE5C,gCAAgC;QAChC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAE/B,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,IAAI,OAAO,CAAC;gBACrB,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE,4BAA4B;gBAClC,UAAU,EAAE,CAAC,WAAW,CAAC;gBACzB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC3B,CAAC;YACF,UAAU,EAAE,IAAI,OAAO,CAAC;gBACtB,IAAI,EAAE,uBAAuB;gBAC7B,IAAI,EAAE,6BAA6B;gBACnC,UAAU,EAAE,CAAC,WAAW,CAAC;gBACzB,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC3B,CAAC;YACF,WAAW,EAAE,IAAI,SAAS,CAAC;gBACzB,IAAI,EAAE,0BAA0B;gBAChC,IAAI,EAAE,8BAA8B;gBACpC,UAAU,EAAE,CAAC,WAAW,CAAC;gBACzB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC5B,SAAS,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC3B,CAAC;SACH,CAAC;QAEF,uBAAuB;QACvB,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;YACpD,GAAG,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACnD,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,GAAG,EAAE;QAChD,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,CAAC,QAAQ,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,uDAAuD;YACvD,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9C,SAAS,CAAC,SAAS,CAAC,IAAI,GAAG,UAAS,GAAG,IAAI;gBACzC,0DAA0D;gBAC1D,2CAA2C;gBAC3C,oDAAoD;gBACpD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAE3B,iCAAiC;gBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAEtC,yEAAyE;gBACzE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAEjD,6CAA6C;gBAC7C,MAAM,OAAO,GAAG,UAAU;oBACtB,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC;oBAC1C,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;gBAEtC,oDAAoD;gBACpD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,WAAW,EAAE,EAAE;oBAClC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;oBAC/D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnC,IAAI,CAAC;wBACD,+BAA+B;wBAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC5D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,WAAW,CAAC,CAAC;wBAC7C,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC5C,OAAO,MAAM,CAAC;oBAClB,CAAC;oBACD,OAAO,KAAK,EAAE,CAAC;wBACX,0BAA0B;wBAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC7D,IAAI,CAAC,SAAS,CAAC;4BACX,IAAI,EAAE,cAAc,CAAC,KAAK;4BAC1B,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;yBAClE,CAAC,CAAC;wBACH,MAAM,KAAK,CAAC;oBAChB,CAAC;4BACO,CAAC;wBACL,iBAAiB;wBACjB,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;wBACzD,MAAM,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,GAAG,CAAC;wBAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;wBAC5E,IAAI,CAAC,GAAG,EAAE,CAAC;oBACf,CAAC;gBACL,CAAC,CAAC,CAAC;gBAEH,0CAA0C;gBAC1C,OAAO,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,CAAC,CAAC;YAEF,uBAAuB;YACvB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,GAAG,EAAE;gBACtD,OAAO,CAAC,GAAG,CAAC,oCAAoC,WAAW,EAAE,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,UAAU,EAAE,CAAC,WAAW,GAAG,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC;CAC3F,CAAC"}

node_modules/mcp-evals/dist/types.d.ts

@@ -0,0 +1,34 @@
+import { type LanguageModel } from 'ai';
+export interface EvalResult {
+    accuracy: number;
+    completeness: number;
+    relevance: number;
+    clarity: number;
+    reasoning: number;
+    overall_comments: string;
+}
+export interface EvalFunction {
+    name: string;
+    description: string;
+    run: (model: LanguageModel) => Promise<EvalResult>;
+}
+export interface EvalConfig {
+    model: LanguageModel;
+    evals: EvalFunction[];
+}
+export interface YamlModelConfig {
+    provider: 'openai' | 'anthropic';
+    name: string;
+    api_key?: string;
+}
+export interface YamlEval {
+    name: string;
+    description: string;
+    prompt: string;
+    expected_result?: string;
+}
+export interface YamlEvalConfig {
+    model?: YamlModelConfig;
+    evals: YamlEval[];
+}
+//# sourceMappingURL=types.d.ts.map

node_modules/mcp-evals/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB"}

node_modules/mcp-evals/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

node_modules/mcp-evals/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

node_modules/mcp-evals/dist/yaml-loader.d.ts

@@ -0,0 +1,14 @@
+import { YamlEvalConfig, EvalConfig } from './types.js';
+/**
+ * Load and parse a YAML configuration file
+ */
+export declare function loadYamlConfig(filePath: string): YamlEvalConfig;
+/**
+ * Convert a YAML configuration to an EvalConfig
+ */
+export declare function yamlConfigToEvalConfig(yamlConfig: YamlEvalConfig, serverPath: string): EvalConfig;
+/**
+ * Load a YAML config file and convert it to an EvalConfig
+ */
+export declare function loadYamlEvalConfig(filePath: string, serverPath: string): EvalConfig;
+//# sourceMappingURL=yaml-loader.d.ts.map

node_modules/mcp-evals/dist/yaml-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml-loader.d.ts","sourceRoot":"","sources":["../src/yaml-loader.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,cAAc,EAAY,UAAU,EAAgB,MAAM,YAAY,CAAC;AAGhF;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,CAwB/D;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU,CAkDjG;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU,CAGnF"}

node_modules/mcp-evals/dist/yaml-loader.js

@@ -0,0 +1,96 @@
+import * as yaml from 'js-yaml';
+import * as fs from 'fs';
+import * as path from 'path';
+import { openai } from "@ai-sdk/openai";
+import { anthropic } from "@ai-sdk/anthropic";
+import { grade } from './index.js';
+/**
+ * Load and parse a YAML configuration file
+ */
+export function loadYamlConfig(filePath) {
+    try {
+        const absolutePath = path.resolve(filePath);
+        const fileContents = fs.readFileSync(absolutePath, 'utf8');
+        const config = yaml.load(fileContents);
+        if (!config || !config.evals || !Array.isArray(config.evals)) {
+            throw new Error('Invalid YAML config: must have an "evals" array');
+        }
+        // Validate each eval
+        for (const evalItem of config.evals) {
+            if (!evalItem.name || !evalItem.description || !evalItem.prompt) {
+                throw new Error(`Invalid eval: each eval must have "name", "description", and "prompt" fields`);
+            }
+        }
+        return config;
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            throw new Error(`Failed to load YAML config: ${error.message}`);
+        }
+        throw new Error('Failed to load YAML config: Unknown error');
+    }
+}
+/**
+ * Convert a YAML configuration to an EvalConfig
+ */
+export function yamlConfigToEvalConfig(yamlConfig, serverPath) {
+    // Setup model
+    let model;
+    if (yamlConfig.model) {
+        if (yamlConfig.model.provider === 'openai') {
+            // Set the API key as an environment variable if provided
+            if (yamlConfig.model.api_key) {
+                process.env.OPENAI_API_KEY = yamlConfig.model.api_key;
+            }
+            model = openai(yamlConfig.model.name);
+        }
+        else if (yamlConfig.model.provider === 'anthropic') {
+            // Set the API key as an environment variable if provided
+            if (yamlConfig.model.api_key) {
+                process.env.ANTHROPIC_API_KEY = yamlConfig.model.api_key;
+            }
+            model = anthropic(yamlConfig.model.name);
+        }
+        else {
+            throw new Error(`Unsupported model provider: ${yamlConfig.model.provider}`);
+        }
+    }
+    else {
+        // Default to GPT-4
+        model = openai("gpt-4o");
+    }
+    // Convert YAML evals to EvalFunctions
+    const evalFunctions = yamlConfig.evals.map((yamlEval) => ({
+        name: yamlEval.name,
+        description: yamlEval.description,
+        run: async (evalModel) => {
+            try {
+                const result = await grade(evalModel, yamlEval.prompt, serverPath);
+                return JSON.parse(result);
+            }
+            catch (error) {
+                // If JSON parsing fails, return a default structure
+                return {
+                    accuracy: 0,
+                    completeness: 0,
+                    relevance: 0,
+                    clarity: 0,
+                    reasoning: 0,
+                    overall_comments: `Error running evaluation: ${error instanceof Error ? error.message : String(error)}`
+                };
+            }
+        }
+    }));
+    return {
+        model,
+        evals: evalFunctions
+    };
+}
+/**
+ * Load a YAML config file and convert it to an EvalConfig
+ */
+export function loadYamlEvalConfig(filePath, serverPath) {
+    const yamlConfig = loadYamlConfig(filePath);
+    return yamlConfigToEvalConfig(yamlConfig, serverPath);
+}
+//# sourceMappingURL=yaml-loader.js.map

node_modules/mcp-evals/dist/yaml-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml-loader.js","sourceRoot":"","sources":["../src/yaml-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAG9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAmB,CAAC;QAEzD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QAED,qBAAqB;QACrB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAChE,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;YAClG,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,UAA0B,EAAE,UAAkB;IACnF,cAAc;IACd,IAAI,KAAoB,CAAC;IACzB,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,UAAU,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3C,yDAAyD;YACzD,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC;YACxD,CAAC;YACD,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAW,CAAC,CAAC;QAC/C,CAAC;aAAM,IAAI,UAAU,CAAC,KAAK,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YACrD,yDAAyD;YACzD,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC;YAC3D,CAAC;YACD,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,IAAW,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,+BAA+B,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;SAAM,CAAC;QACN,mBAAmB;QACnB,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAED,sCAAsC;IACtC,MAAM,aAAa,GAAmB,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAkB,EAAE,EAAE,CAAC,CAAC;QAClF,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,GAAG,EAAE,KAAK,EAAE,SAAwB,EAAE,EAAE;YACtC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACnE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,oDAAoD;gBACpD,OAAO;oBACL,QAAQ,EAAE,CAAC;oBACX,YAAY,EAAE,CAAC;oBACf,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,CAAC;oBACZ,gBAAgB,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;iBACxG,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC,CAAC;IAEJ,OAAO;QACL,KAAK;QACL,KAAK,EAAE,aAAa;KACrB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,UAAkB;IACrE,MAAM,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,OAAO,sBAAsB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;AACxD,CAAC"}