droideparanoico
94cfd80dd0
Meta-plugin for GitHub Copilot CLI that watches workflows and auto-generates reusable agent skills. - postToolUse hook: fingerprints and logs tool calls per session - sessionEnd hook: sliding-window pattern detection across sessions - sessionStart hook: injects context about proposed skills - /skill-factory slash command for reviewing/managing skills - 13 passing tests validating observer, analyzer, and edge cases Inspired by hermes-skill-factory (Romanescu11/hermes-skill-factory)
357 lines
13 KiB
JavaScript
357 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* Test harness for copilot-skill-factory observer.js
|
|
*
|
|
* Simulates hook payloads to verify:
|
|
* 1. postToolUse logging works
|
|
* 2. sessionEnd analysis detects patterns
|
|
* 3. sessionStart injects context
|
|
* 4. Cross-session pattern detection triggers skill generation
|
|
*/
|
|
|
|
const { execFileSync, execSync } = require('child_process');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const OBSERVER = path.join(__dirname, 'bin', 'observer.js');
|
|
const DATA_DIR = path.join(__dirname, '.test-data');
|
|
|
|
// Override data directory for tests
|
|
process.env.COPILOT_PLUGIN_DATA = DATA_DIR;
|
|
|
|
// Also override HOME for skills dir
|
|
process.env.HOME = DATA_DIR;
|
|
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
function test(name, fn) {
|
|
try {
|
|
fn();
|
|
console.log(` ✓ ${name}`);
|
|
passed++;
|
|
} catch (e) {
|
|
console.log(` ✗ ${name}: ${e.message}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
function runObserver(event, payload) {
|
|
return execFileSync('node', [OBSERVER, event], {
|
|
input: JSON.stringify(payload),
|
|
encoding: 'utf8',
|
|
env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR },
|
|
timeout: 5000
|
|
});
|
|
}
|
|
|
|
function cleanup() {
|
|
try { fs.rmSync(DATA_DIR, { recursive: true, force: true }); } catch {}
|
|
}
|
|
|
|
console.log('\n🏭 Skill Factory — Test Suite\n');
|
|
|
|
// ── postToolUse tests ──────────────────────────────────────────────────────
|
|
|
|
console.log('postToolUse:');
|
|
cleanup();
|
|
|
|
const SESSION_A = 'test-session-aaa';
|
|
const SESSION_B = 'test-session-bbb';
|
|
|
|
test('logs a tool call', () => {
|
|
runObserver('postToolUse', {
|
|
sessionId: SESSION_A,
|
|
timestamp: Date.now(),
|
|
cwd: '/test/project',
|
|
toolName: 'bash',
|
|
toolArgs: { command: 'git status', description: 'check git status' }
|
|
});
|
|
|
|
const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`);
|
|
if (!fs.existsSync(logPath)) throw new Error('Session log not created');
|
|
|
|
const content = fs.readFileSync(logPath, 'utf8').trim();
|
|
if (!content) throw new Error('Session log is empty');
|
|
|
|
const entry = JSON.parse(content.split('\n')[0]);
|
|
if (entry.toolName !== 'bash') throw new Error(`Expected bash, got ${entry.toolName}`);
|
|
if (!entry.fingerprint) throw new Error('Missing fingerprint');
|
|
if (entry.argsSummary.commandCategory !== 'git') throw new Error(`Expected git category, got ${entry.argsSummary.commandCategory}`);
|
|
});
|
|
|
|
test('fingerprints are stable for same tool+args', () => {
|
|
const fp1 = runObserverAndGetFingerprint('bash', { command: 'git diff' });
|
|
const fp2 = runObserverAndGetFingerprint('bash', { command: 'git diff --stat' });
|
|
// Different command lengths → different fingerprints
|
|
if (fp1 === fp2) throw new Error('Different commands should have different fingerprints');
|
|
});
|
|
|
|
test('fingerprints match for similar file operations', () => {
|
|
const fp1 = runObserverAndGetFingerprint('edit', { path: '/a/b/file1.py', old_string: 'abc' });
|
|
const fp2 = runObserverAndGetFingerprint('edit', { path: '/x/y/file2.py', old_string: 'xyz' });
|
|
// Same tool, file extension normalized, content normalized → same fingerprint
|
|
if (fp1 !== fp2) throw new Error('Similar edits should have same fingerprint');
|
|
});
|
|
|
|
test('skips noisy tools (todo, ask)', () => {
|
|
runObserver('postToolUse', {
|
|
sessionId: SESSION_A,
|
|
timestamp: Date.now(),
|
|
cwd: '/test',
|
|
toolName: 'todo_write',
|
|
toolArgs: { todos: [] }
|
|
});
|
|
|
|
const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`);
|
|
const content = fs.readFileSync(logPath, 'utf8').trim();
|
|
const todoEntries = content.split('\n').filter(l => l.includes('todo_write'));
|
|
if (todoEntries.length > 0) throw new Error('todo_write should be skipped');
|
|
});
|
|
|
|
function runObserverAndGetFingerprint(toolName, toolArgs) {
|
|
const tmpSession = 'tmp-' + Math.random().toString(36).slice(2, 8);
|
|
runObserver('postToolUse', {
|
|
sessionId: tmpSession,
|
|
timestamp: Date.now(),
|
|
cwd: '/test',
|
|
toolName,
|
|
toolArgs
|
|
});
|
|
|
|
const logPath = path.join(DATA_DIR, 'sessions', `${tmpSession}.jsonl`);
|
|
const entry = JSON.parse(fs.readFileSync(logPath, 'utf8').trim());
|
|
return entry.fingerprint;
|
|
}
|
|
|
|
// ── sessionEnd tests ───────────────────────────────────────────────────────
|
|
|
|
console.log('\nsessionEnd:');
|
|
cleanup();
|
|
|
|
test('analyzes session and creates pattern DB', () => {
|
|
// Simulate a full session with a repeated workflow
|
|
const sessionId = 'test-session-1';
|
|
const workflow = [
|
|
{ toolName: 'bash', toolArgs: { command: 'git pull' } },
|
|
{ toolName: 'grep', toolArgs: { pattern: 'TODO' } },
|
|
{ toolName: 'edit', toolArgs: { path: 'src/app.py', old_string: 'old', new_string: 'new' } },
|
|
{ toolName: 'bash', toolArgs: { command: 'pytest' } },
|
|
];
|
|
|
|
for (const call of workflow) {
|
|
runObserver('postToolUse', {
|
|
sessionId,
|
|
timestamp: Date.now(),
|
|
cwd: '/test/project',
|
|
...call
|
|
});
|
|
}
|
|
|
|
runObserver('sessionEnd', {
|
|
sessionId,
|
|
timestamp: Date.now(),
|
|
cwd: '/test/project',
|
|
reason: 'complete'
|
|
});
|
|
|
|
// Check DB was created
|
|
const dbPath = path.join(DATA_DIR, 'pattern-db.json');
|
|
if (!fs.existsSync(dbPath)) throw new Error('Pattern DB not created');
|
|
|
|
const db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
|
|
if (Object.keys(db.patterns).length === 0) throw new Error('No patterns detected');
|
|
|
|
if (!db.sessions[sessionId]) throw new Error('Session not recorded in DB');
|
|
if (db.sessions[sessionId].toolCount !== 4) throw new Error(`Expected 4 tools, got ${db.sessions[sessionId].toolCount}`);
|
|
});
|
|
|
|
test('detects cross-session patterns and generates skills', () => {
|
|
cleanup();
|
|
|
|
// Session 1: Same workflow
|
|
const workflow = [
|
|
{ toolName: 'bash', toolArgs: { command: 'npm install' } },
|
|
{ toolName: 'edit', toolArgs: { path: 'src/index.ts', old_string: 'a', new_string: 'b' } },
|
|
{ toolName: 'bash', toolArgs: { command: 'npm test' } },
|
|
];
|
|
|
|
// Run 2 sessions with the same workflow
|
|
for (let s = 1; s <= 2; s++) {
|
|
const sid = `cross-session-${s}`;
|
|
for (const call of workflow) {
|
|
runObserver('postToolUse', {
|
|
sessionId: sid,
|
|
timestamp: Date.now(),
|
|
cwd: '/test/project',
|
|
...call
|
|
});
|
|
}
|
|
runObserver('sessionEnd', {
|
|
sessionId: sid,
|
|
timestamp: Date.now(),
|
|
cwd: '/test/project',
|
|
reason: 'complete'
|
|
});
|
|
}
|
|
|
|
// After 2 sessions with same pattern, a skill should be proposed
|
|
const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed');
|
|
if (!fs.existsSync(proposedDir)) throw new Error('Proposed directory not created');
|
|
|
|
const proposed = fs.readdirSync(proposedDir, { withFileTypes: true })
|
|
.filter(d => d.isDirectory())
|
|
.map(d => d.name);
|
|
|
|
if (proposed.length === 0) throw new Error('No skills proposed after cross-session patterns');
|
|
|
|
// Verify SKILL.md was generated
|
|
const skillMd = path.join(proposedDir, proposed[0], 'SKILL.md');
|
|
if (!fs.existsSync(skillMd)) throw new Error('SKILL.md not generated');
|
|
|
|
const content = fs.readFileSync(skillMd, 'utf8');
|
|
if (!content.includes('Auto-generated by Skill Factory')) throw new Error('SKILL.md missing auto-gen marker');
|
|
if (!content.includes('2 sessions')) throw new Error('SKILL.md should mention 2 sessions');
|
|
|
|
console.log(` → Generated skill: ${proposed[0]}`);
|
|
});
|
|
|
|
test('does not generate skills for single-session patterns', () => {
|
|
cleanup();
|
|
|
|
const sid = 'single-session';
|
|
const workflow = [
|
|
{ toolName: 'bash', toolArgs: { command: 'echo hello' } },
|
|
{ toolName: 'read_file', toolArgs: { path: 'test.txt' } },
|
|
{ toolName: 'edit', toolArgs: { path: 'test.txt', old_string: 'a', new_string: 'b' } },
|
|
];
|
|
|
|
for (const call of workflow) {
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', ...call });
|
|
}
|
|
runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' });
|
|
|
|
const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed');
|
|
const hasProposed = fs.existsSync(proposedDir) && fs.readdirSync(proposedDir).length > 0;
|
|
if (hasProposed) throw new Error('Should not propose skills from a single session');
|
|
});
|
|
|
|
// ── sessionStart tests ─────────────────────────────────────────────────────
|
|
|
|
console.log('\nsessionStart:');
|
|
cleanup();
|
|
|
|
test('is silent when no proposals exist', () => {
|
|
const output = runObserver('sessionStart', {
|
|
sessionId: 'start-test',
|
|
timestamp: Date.now(),
|
|
cwd: '/test',
|
|
source: 'new'
|
|
});
|
|
|
|
if (output.trim()) throw new Error('Should be silent with no proposals');
|
|
});
|
|
|
|
test('injects context when proposals exist', () => {
|
|
cleanup();
|
|
|
|
// Create a fake proposed skill
|
|
const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed', 'test-skill');
|
|
fs.mkdirSync(proposedDir, { recursive: true });
|
|
fs.writeFileSync(path.join(proposedDir, 'SKILL.md'), '# Test\n');
|
|
|
|
const output = runObserver('sessionStart', {
|
|
sessionId: 'start-test-2',
|
|
timestamp: Date.now(),
|
|
cwd: '/test',
|
|
source: 'new'
|
|
});
|
|
|
|
if (!output.trim()) throw new Error('Should inject context when proposals exist');
|
|
const parsed = JSON.parse(output.trim());
|
|
if (!parsed.additionalContext) throw new Error('Missing additionalContext');
|
|
if (!parsed.additionalContext.includes('Skill Factory')) throw new Error('Context should mention Skill Factory');
|
|
if (!parsed.additionalContext.includes('/skill-factory review')) throw new Error('Context should suggest /skill-factory review');
|
|
});
|
|
|
|
// ── Edge cases ─────────────────────────────────────────────────────────────
|
|
|
|
console.log('\nEdge cases:');
|
|
cleanup();
|
|
|
|
test('handles empty stdin gracefully', () => {
|
|
try {
|
|
execFileSync('node', [OBSERVER, 'postToolUse'], {
|
|
input: '',
|
|
encoding: 'utf8',
|
|
env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR },
|
|
timeout: 3000
|
|
});
|
|
// Should exit 0
|
|
} catch (e) {
|
|
if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`);
|
|
}
|
|
});
|
|
|
|
test('handles malformed JSON gracefully', () => {
|
|
try {
|
|
execFileSync('node', [OBSERVER, 'postToolUse'], {
|
|
input: '{not valid json',
|
|
encoding: 'utf8',
|
|
env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR },
|
|
timeout: 3000
|
|
});
|
|
} catch (e) {
|
|
// Should still exit 0 (hook failures are silent)
|
|
if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`);
|
|
}
|
|
});
|
|
|
|
test('sessionEnd with fewer than 3 tools creates no patterns', () => {
|
|
cleanup();
|
|
const sid = 'short-session';
|
|
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: 'ls' } });
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'read_file', toolArgs: { path: 'test.txt' } });
|
|
runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' });
|
|
|
|
const dbPath = path.join(DATA_DIR, 'pattern-db.json');
|
|
let patternCount = 0;
|
|
if (fs.existsSync(dbPath)) {
|
|
const db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
|
|
patternCount = Object.keys(db.patterns).length;
|
|
}
|
|
if (patternCount > 0) throw new Error('Sessions with <3 tools should not create patterns');
|
|
});
|
|
|
|
test('session cleanup keeps only last 20 sessions', () => {
|
|
cleanup();
|
|
|
|
// Create 25 sessions
|
|
for (let i = 1; i <= 25; i++) {
|
|
const sid = `bulk-session-${i}`;
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: `cmd-${i}` } });
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'grep', toolArgs: { pattern: `pat-${i}` } });
|
|
runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'edit', toolArgs: { path: 'file.py', old_string: 'a', new_string: 'b' } });
|
|
runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' });
|
|
}
|
|
|
|
const dbPath = path.join(DATA_DIR, 'pattern-db.json');
|
|
const db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
|
|
const sessionCount = Object.keys(db.sessions).length;
|
|
|
|
if (sessionCount > 20) throw new Error(`Session cleanup failed: ${sessionCount} > 20`);
|
|
});
|
|
|
|
// ── Summary ────────────────────────────────────────────────────────────────
|
|
|
|
console.log(`\n${'─'.repeat(50)}`);
|
|
console.log(`Results: ${passed} passed, ${failed} failed`);
|
|
console.log(`${'─'.repeat(50)}\n`);
|
|
|
|
// Cleanup
|
|
cleanup();
|
|
|
|
process.exit(failed > 0 ? 1 : 0);
|