#!/usr/bin/env node /** * Test harness for copilot-skill-factory observer.js * * Simulates hook payloads to verify: * 1. postToolUse logging works * 2. sessionEnd analysis detects patterns * 3. sessionStart injects context * 4. Cross-session pattern detection triggers skill generation */ const { execFileSync, execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); const OBSERVER = path.join(__dirname, 'bin', 'observer.js'); const DATA_DIR = path.join(__dirname, '.test-data'); // Override data directory for tests process.env.COPILOT_PLUGIN_DATA = DATA_DIR; // Also override HOME for skills dir process.env.HOME = DATA_DIR; let passed = 0; let failed = 0; function test(name, fn) { try { fn(); console.log(` āœ“ ${name}`); passed++; } catch (e) { console.log(` āœ— ${name}: ${e.message}`); failed++; } } function runObserver(event, payload) { return execFileSync('node', [OBSERVER, event], { input: JSON.stringify(payload), encoding: 'utf8', env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, timeout: 5000 }); } function cleanup() { try { fs.rmSync(DATA_DIR, { recursive: true, force: true }); } catch {} } console.log('\nšŸ­ Skill Factory — Test Suite\n'); // ── postToolUse tests ────────────────────────────────────────────────────── console.log('postToolUse:'); cleanup(); const SESSION_A = 'test-session-aaa'; const SESSION_B = 'test-session-bbb'; test('logs a tool call', () => { runObserver('postToolUse', { sessionId: SESSION_A, timestamp: Date.now(), cwd: '/test/project', toolName: 'bash', toolArgs: { command: 'git status', description: 'check git status' } }); const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`); if (!fs.existsSync(logPath)) throw new Error('Session log not created'); const content = fs.readFileSync(logPath, 'utf8').trim(); if (!content) throw new Error('Session log is empty'); const entry = JSON.parse(content.split('\n')[0]); if (entry.toolName !== 'bash') throw new Error(`Expected bash, got ${entry.toolName}`); if (!entry.fingerprint) throw new Error('Missing fingerprint'); if (entry.argsSummary.commandCategory !== 'git') throw new Error(`Expected git category, got ${entry.argsSummary.commandCategory}`); }); test('fingerprints are stable for same tool+args', () => { const fp1 = runObserverAndGetFingerprint('bash', { command: 'git diff' }); const fp2 = runObserverAndGetFingerprint('bash', { command: 'git diff --stat' }); // Different command lengths → different fingerprints if (fp1 === fp2) throw new Error('Different commands should have different fingerprints'); }); test('fingerprints match for similar file operations', () => { const fp1 = runObserverAndGetFingerprint('edit', { path: '/a/b/file1.py', old_string: 'abc' }); const fp2 = runObserverAndGetFingerprint('edit', { path: '/x/y/file2.py', old_string: 'xyz' }); // Same tool, file extension normalized, content normalized → same fingerprint if (fp1 !== fp2) throw new Error('Similar edits should have same fingerprint'); }); test('skips noisy tools (todo, ask)', () => { runObserver('postToolUse', { sessionId: SESSION_A, timestamp: Date.now(), cwd: '/test', toolName: 'todo_write', toolArgs: { todos: [] } }); const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`); const content = fs.readFileSync(logPath, 'utf8').trim(); const todoEntries = content.split('\n').filter(l => l.includes('todo_write')); if (todoEntries.length > 0) throw new Error('todo_write should be skipped'); }); function runObserverAndGetFingerprint(toolName, toolArgs) { const tmpSession = 'tmp-' + Math.random().toString(36).slice(2, 8); runObserver('postToolUse', { sessionId: tmpSession, timestamp: Date.now(), cwd: '/test', toolName, toolArgs }); const logPath = path.join(DATA_DIR, 'sessions', `${tmpSession}.jsonl`); const entry = JSON.parse(fs.readFileSync(logPath, 'utf8').trim()); return entry.fingerprint; } // ── sessionEnd tests ─────────────────────────────────────────────────────── console.log('\nsessionEnd:'); cleanup(); test('analyzes session and creates pattern DB', () => { // Simulate a full session with a repeated workflow const sessionId = 'test-session-1'; const workflow = [ { toolName: 'bash', toolArgs: { command: 'git pull' } }, { toolName: 'grep', toolArgs: { pattern: 'TODO' } }, { toolName: 'edit', toolArgs: { path: 'src/app.py', old_string: 'old', new_string: 'new' } }, { toolName: 'bash', toolArgs: { command: 'pytest' } }, ]; for (const call of workflow) { runObserver('postToolUse', { sessionId, timestamp: Date.now(), cwd: '/test/project', ...call }); } runObserver('sessionEnd', { sessionId, timestamp: Date.now(), cwd: '/test/project', reason: 'complete' }); // Check DB was created const dbPath = path.join(DATA_DIR, 'pattern-db.json'); if (!fs.existsSync(dbPath)) throw new Error('Pattern DB not created'); const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); if (Object.keys(db.patterns).length === 0) throw new Error('No patterns detected'); if (!db.sessions[sessionId]) throw new Error('Session not recorded in DB'); if (db.sessions[sessionId].toolCount !== 4) throw new Error(`Expected 4 tools, got ${db.sessions[sessionId].toolCount}`); }); test('detects cross-session patterns and generates skills', () => { cleanup(); // Session 1: Same workflow const workflow = [ { toolName: 'bash', toolArgs: { command: 'npm install' } }, { toolName: 'edit', toolArgs: { path: 'src/index.ts', old_string: 'a', new_string: 'b' } }, { toolName: 'bash', toolArgs: { command: 'npm test' } }, ]; // Run 2 sessions with the same workflow for (let s = 1; s <= 2; s++) { const sid = `cross-session-${s}`; for (const call of workflow) { runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test/project', ...call }); } runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test/project', reason: 'complete' }); } // After 2 sessions with same pattern, a skill should be proposed const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed'); if (!fs.existsSync(proposedDir)) throw new Error('Proposed directory not created'); const proposed = fs.readdirSync(proposedDir, { withFileTypes: true }) .filter(d => d.isDirectory()) .map(d => d.name); if (proposed.length === 0) throw new Error('No skills proposed after cross-session patterns'); // Verify SKILL.md was generated const skillMd = path.join(proposedDir, proposed[0], 'SKILL.md'); if (!fs.existsSync(skillMd)) throw new Error('SKILL.md not generated'); const content = fs.readFileSync(skillMd, 'utf8'); if (!content.includes('Auto-generated by Skill Factory')) throw new Error('SKILL.md missing auto-gen marker'); if (!content.includes('2 sessions')) throw new Error('SKILL.md should mention 2 sessions'); console.log(` → Generated skill: ${proposed[0]}`); }); test('does not generate skills for single-session patterns', () => { cleanup(); const sid = 'single-session'; const workflow = [ { toolName: 'bash', toolArgs: { command: 'echo hello' } }, { toolName: 'read_file', toolArgs: { path: 'test.txt' } }, { toolName: 'edit', toolArgs: { path: 'test.txt', old_string: 'a', new_string: 'b' } }, ]; for (const call of workflow) { runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', ...call }); } runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed'); const hasProposed = fs.existsSync(proposedDir) && fs.readdirSync(proposedDir).length > 0; if (hasProposed) throw new Error('Should not propose skills from a single session'); }); // ── sessionStart tests ───────────────────────────────────────────────────── console.log('\nsessionStart:'); cleanup(); test('is silent when no proposals exist', () => { const output = runObserver('sessionStart', { sessionId: 'start-test', timestamp: Date.now(), cwd: '/test', source: 'new' }); if (output.trim()) throw new Error('Should be silent with no proposals'); }); test('injects context when proposals exist', () => { cleanup(); // Create a fake proposed skill const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed', 'test-skill'); fs.mkdirSync(proposedDir, { recursive: true }); fs.writeFileSync(path.join(proposedDir, 'SKILL.md'), '# Test\n'); const output = runObserver('sessionStart', { sessionId: 'start-test-2', timestamp: Date.now(), cwd: '/test', source: 'new' }); if (!output.trim()) throw new Error('Should inject context when proposals exist'); const parsed = JSON.parse(output.trim()); if (!parsed.additionalContext) throw new Error('Missing additionalContext'); if (!parsed.additionalContext.includes('Skill Factory')) throw new Error('Context should mention Skill Factory'); if (!parsed.additionalContext.includes('/skill-factory review')) throw new Error('Context should suggest /skill-factory review'); }); // ── Edge cases ───────────────────────────────────────────────────────────── console.log('\nEdge cases:'); cleanup(); test('handles empty stdin gracefully', () => { try { execFileSync('node', [OBSERVER, 'postToolUse'], { input: '', encoding: 'utf8', env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, timeout: 3000 }); // Should exit 0 } catch (e) { if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`); } }); test('handles malformed JSON gracefully', () => { try { execFileSync('node', [OBSERVER, 'postToolUse'], { input: '{not valid json', encoding: 'utf8', env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, timeout: 3000 }); } catch (e) { // Should still exit 0 (hook failures are silent) if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`); } }); test('sessionEnd with fewer than 3 tools creates no patterns', () => { cleanup(); const sid = 'short-session'; runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: 'ls' } }); runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'read_file', toolArgs: { path: 'test.txt' } }); runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); const dbPath = path.join(DATA_DIR, 'pattern-db.json'); let patternCount = 0; if (fs.existsSync(dbPath)) { const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); patternCount = Object.keys(db.patterns).length; } if (patternCount > 0) throw new Error('Sessions with <3 tools should not create patterns'); }); test('session cleanup keeps only last 20 sessions', () => { cleanup(); // Create 25 sessions for (let i = 1; i <= 25; i++) { const sid = `bulk-session-${i}`; runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: `cmd-${i}` } }); runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'grep', toolArgs: { pattern: `pat-${i}` } }); runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'edit', toolArgs: { path: 'file.py', old_string: 'a', new_string: 'b' } }); runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); } const dbPath = path.join(DATA_DIR, 'pattern-db.json'); const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); const sessionCount = Object.keys(db.sessions).length; if (sessionCount > 20) throw new Error(`Session cleanup failed: ${sessionCount} > 20`); }); // ── Summary ──────────────────────────────────────────────────────────────── console.log(`\n${'─'.repeat(50)}`); console.log(`Results: ${passed} passed, ${failed} failed`); console.log(`${'─'.repeat(50)}\n`); // Cleanup cleanup(); process.exit(failed > 0 ? 1 : 0);