droideparanoico
94cfd80dd0
Meta-plugin for GitHub Copilot CLI that watches workflows and auto-generates reusable agent skills. - postToolUse hook: fingerprints and logs tool calls per session - sessionEnd hook: sliding-window pattern detection across sessions - sessionStart hook: injects context about proposed skills - /skill-factory slash command for reviewing/managing skills - 13 passing tests validating observer, analyzer, and edge cases Inspired by hermes-skill-factory (Romanescu11/hermes-skill-factory)
527 lines
16 KiB
JavaScript
Executable File
527 lines
16 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
/**
|
|
* copilot-skill-factory — Observer
|
|
*
|
|
* Handles three hook events dispatched by Copilot CLI:
|
|
* postToolUse — logs every successful tool call to a per-session JSONL file
|
|
* sessionEnd — analyzes the session log, updates the pattern database
|
|
* sessionStart — injects additionalContext about recently detected patterns
|
|
*
|
|
* Input: JSON payload on stdin (camelCase format from hooks.json)
|
|
* Output: JSON to stdout for hooks that support output (sessionStart → additionalContext)
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
|
|
// ── helpers ────────────────────────────────────────────────────────────────
|
|
|
|
function readStdin() {
|
|
try {
|
|
const raw = fs.readFileSync(0, 'utf8');
|
|
if (!raw.trim()) return null;
|
|
return JSON.parse(raw);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function ensureDir(dir) {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
function dataDir() {
|
|
const fromEnv = process.env.COPILOT_PLUGIN_DATA || process.env.CLAUDE_PLUGIN_DATA;
|
|
if (fromEnv && fromEnv.trim() && !fromEnv.trim().startsWith('${')) return fromEnv;
|
|
// Fallback: use plugin root directory
|
|
const root = process.env.PLUGIN_ROOT || process.env.COPILOT_PLUGIN_ROOT || process.env.CLAUDE_PLUGIN_ROOT || __dirname;
|
|
return path.join(root, '.data');
|
|
}
|
|
|
|
function skillsDir() {
|
|
const home = process.env.HOME || process.env.USERPROFILE || '/tmp';
|
|
return path.join(home, '.copilot', 'skills', 'skill-factory');
|
|
}
|
|
|
|
function dbPath() {
|
|
return path.join(dataDir(), 'pattern-db.json');
|
|
}
|
|
|
|
function sessionLogPath(sessionId) {
|
|
return path.join(dataDir(), 'sessions', `${sessionId}.jsonl`);
|
|
}
|
|
|
|
function loadDb() {
|
|
try {
|
|
return JSON.parse(fs.readFileSync(dbPath(), 'utf8'));
|
|
} catch {
|
|
return { patterns: {}, sessions: {}, meta: { version: 1, createdAt: new Date().toISOString() } };
|
|
}
|
|
}
|
|
|
|
function saveDb(db) {
|
|
ensureDir(dataDir());
|
|
fs.writeFileSync(dbPath(), JSON.stringify(db, null, 2));
|
|
}
|
|
|
|
// ── tool-call fingerprinting ───────────────────────────────────────────────
|
|
|
|
/**
|
|
* Creates a fingerprint for a tool call — captures the "what" without the
|
|
* session-specific parameters (file paths, timestamps, exact content).
|
|
* This lets us match similar tool calls across sessions.
|
|
*/
|
|
function fingerprint(toolName, toolArgs) {
|
|
const normalized = normalizeArgs(toolName, toolArgs);
|
|
return `${toolName}:${hash(normalized)}`;
|
|
}
|
|
|
|
function normalizeArgs(toolName, toolArgs) {
|
|
if (!toolArgs || typeof toolArgs !== 'object') return '{}';
|
|
|
|
const args = { ...toolArgs };
|
|
|
|
// Strip file paths (replace with placeholder based on extension)
|
|
const fileFields = ['path', 'filePath', 'file_path', 'file', 'output', 'target'];
|
|
for (const f of fileFields) {
|
|
if (args[f] && typeof args[f] === 'string') {
|
|
const ext = path.extname(args[f]);
|
|
args[f] = `<file${ext}>`;
|
|
}
|
|
}
|
|
|
|
// Strip command content (replace with command length)
|
|
if (args.command && typeof args.command === 'string') {
|
|
args.command = `<cmd:${args.command.length}>`;
|
|
}
|
|
|
|
// Strip content/patch bodies (replace with length)
|
|
const contentFields = ['content', 'new_string', 'old_string', 'new_str', 'old_str',
|
|
'patch', 'text', 'body', 'message', 'code', 'script'];
|
|
for (const f of contentFields) {
|
|
if (args[f] && typeof args[f] === 'string') {
|
|
args[f] = `<content:${args[f].length}>`;
|
|
}
|
|
}
|
|
|
|
// Simplify arrays
|
|
for (const [k, v] of Object.entries(args)) {
|
|
if (Array.isArray(v)) args[k] = `<array:${v.length}>`;
|
|
}
|
|
|
|
return JSON.stringify(args, Object.keys(args).sort());
|
|
}
|
|
|
|
function hash(str) {
|
|
return crypto.createHash('md5').update(str).digest('hex').slice(0, 12);
|
|
}
|
|
|
|
// ── postToolUse handler ────────────────────────────────────────────────────
|
|
|
|
function handlePostToolUse(input) {
|
|
const { sessionId, toolName, toolArgs } = input;
|
|
|
|
// Skip noisy/redundant tools
|
|
const skipTools = ['todo_write', 'update_todo', 'TodoWrite', 'ask_user', 'AskUserQuestion'];
|
|
if (skipTools.includes(toolName)) return;
|
|
|
|
const fp = fingerprint(toolName, toolArgs);
|
|
const entry = {
|
|
ts: new Date().toISOString(),
|
|
sessionId,
|
|
toolName,
|
|
fingerprint: fp,
|
|
// Store minimal args for context (no full content)
|
|
argsSummary: summarizeArgs(toolName, toolArgs)
|
|
};
|
|
|
|
ensureDir(path.dirname(sessionLogPath(sessionId)));
|
|
fs.appendFileSync(sessionLogPath(sessionId), JSON.stringify(entry) + '\n');
|
|
}
|
|
|
|
function summarizeArgs(toolName, toolArgs) {
|
|
if (!toolArgs || typeof toolArgs !== 'object') return {};
|
|
|
|
const summary = {};
|
|
|
|
// Capture file pattern hints
|
|
const fileFields = ['path', 'filePath', 'file_path', 'file'];
|
|
for (const f of fileFields) {
|
|
if (toolArgs[f] && typeof toolArgs[f] === 'string') {
|
|
summary.fileExt = path.extname(toolArgs[f]) || undefined;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Capture command category hints
|
|
if (toolArgs.command && typeof toolArgs.command === 'string') {
|
|
const cmd = toolArgs.command.trim();
|
|
const firstWord = cmd.split(/\s+/)[0];
|
|
summary.commandCategory = firstWord;
|
|
}
|
|
|
|
// Tool-specific hints
|
|
switch (toolName) {
|
|
case 'bash':
|
|
case 'Bash':
|
|
case 'terminal':
|
|
summary.commandCategory = summary.commandCategory || 'shell';
|
|
break;
|
|
case 'edit':
|
|
case 'Edit':
|
|
case 'write':
|
|
case 'Write':
|
|
case 'create':
|
|
case 'Create':
|
|
summary.operation = 'write';
|
|
break;
|
|
case 'read_file':
|
|
case 'Read':
|
|
case 'view':
|
|
case 'View':
|
|
summary.operation = 'read';
|
|
break;
|
|
case 'grep':
|
|
case 'Grep':
|
|
case 'search':
|
|
summary.operation = 'search';
|
|
break;
|
|
case 'web_fetch':
|
|
case 'WebFetch':
|
|
case 'web_search':
|
|
case 'WebSearch':
|
|
summary.operation = 'web';
|
|
break;
|
|
}
|
|
|
|
return summary;
|
|
}
|
|
|
|
// ── sessionEnd handler (pattern analysis) ──────────────────────────────────
|
|
|
|
function handleSessionEnd(input) {
|
|
const { sessionId, reason } = input;
|
|
|
|
// Skip failed/aborted sessions
|
|
if (reason === 'error' || reason === 'abort') return;
|
|
|
|
const logPath = sessionLogPath(sessionId);
|
|
if (!fs.existsSync(logPath)) return;
|
|
|
|
const entries = [];
|
|
try {
|
|
const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n');
|
|
for (const line of lines) {
|
|
if (line.trim()) entries.push(JSON.parse(line));
|
|
}
|
|
} catch {
|
|
return;
|
|
}
|
|
|
|
if (entries.length < 3) return; // Too short to be a pattern
|
|
|
|
const db = loadDb();
|
|
|
|
// Extract sequences (sliding window of 3-8 consecutive tool calls)
|
|
for (let windowSize = 3; windowSize <= Math.min(8, entries.length); windowSize++) {
|
|
for (let i = 0; i <= entries.length - windowSize; i++) {
|
|
const window = entries.slice(i, i + windowSize);
|
|
const seqKey = window.map(e => e.fingerprint).join('→');
|
|
|
|
if (!db.patterns[seqKey]) {
|
|
db.patterns[seqKey] = {
|
|
sequence: window.map(e => ({
|
|
toolName: e.toolName,
|
|
fingerprint: e.fingerprint,
|
|
argsSummary: e.argsSummary
|
|
})),
|
|
sessions: [],
|
|
totalOccurrences: 0,
|
|
firstSeen: new Date().toISOString(),
|
|
lastSeen: new Date().toISOString(),
|
|
proposedSkill: null
|
|
};
|
|
}
|
|
|
|
const pattern = db.patterns[seqKey];
|
|
if (!pattern.sessions.includes(sessionId)) {
|
|
pattern.sessions.push(sessionId);
|
|
}
|
|
pattern.totalOccurrences++;
|
|
pattern.lastSeen = new Date().toISOString();
|
|
}
|
|
}
|
|
|
|
// Mark session as analyzed
|
|
db.sessions[sessionId] = {
|
|
analyzedAt: new Date().toISOString(),
|
|
toolCount: entries.length,
|
|
reason
|
|
};
|
|
|
|
saveDb(db);
|
|
|
|
// Generate proposed skills for patterns seen in 2+ sessions
|
|
generateProposedSkills(db);
|
|
|
|
// Clean up session log (keep last 10 for future reference)
|
|
cleanupOldSessions(db);
|
|
}
|
|
|
|
// ── skill generation ───────────────────────────────────────────────────────
|
|
|
|
function generateProposedSkills(db) {
|
|
const candidates = [];
|
|
|
|
for (const [key, pattern] of Object.entries(db.patterns)) {
|
|
// Only propose patterns seen in 2+ different sessions
|
|
if (pattern.sessions.length >= 2 && !pattern.proposedSkill) {
|
|
candidates.push({ key, pattern });
|
|
}
|
|
}
|
|
|
|
if (candidates.length === 0) return;
|
|
|
|
// Sort by session count (most repeated first), take top 3
|
|
candidates.sort((a, b) => b.pattern.sessions.length - a.pattern.sessions.length);
|
|
const topCandidates = candidates.slice(0, 3);
|
|
|
|
for (const { key, pattern } of topCandidates) {
|
|
const skillName = generateSkillName(pattern);
|
|
const skillMd = generateSkillMd(skillName, pattern);
|
|
|
|
// Write proposed SKILL.md
|
|
const skillDir = path.join(skillsDir(), 'proposed', skillName);
|
|
ensureDir(skillDir);
|
|
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), skillMd);
|
|
|
|
// Write proposal metadata
|
|
fs.writeFileSync(path.join(skillDir, '.proposal.json'), JSON.stringify({
|
|
patternKey: key,
|
|
detectedAt: new Date().toISOString(),
|
|
sessionCount: pattern.sessions.length,
|
|
totalOccurrences: pattern.totalOccurrences,
|
|
status: 'proposed'
|
|
}, null, 2));
|
|
|
|
pattern.proposedSkill = skillName;
|
|
}
|
|
|
|
saveDb(db);
|
|
}
|
|
|
|
function generateSkillName(pattern) {
|
|
// Derive a name from the dominant tool types in the sequence
|
|
const toolCounts = {};
|
|
for (const step of pattern.sequence) {
|
|
toolCounts[step.toolName] = (toolCounts[step.toolName] || 0) + 1;
|
|
}
|
|
|
|
const dominant = Object.entries(toolCounts)
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 2)
|
|
.map(([name]) => simplifyToolName(name));
|
|
|
|
const name = dominant.join('-') || 'generic-workflow';
|
|
return name.slice(0, 40).replace(/[^a-z0-9-]/g, '-').replace(/-+/g, '-');
|
|
}
|
|
|
|
function simplifyToolName(toolName) {
|
|
const map = {
|
|
'bash': 'shell', 'Bash': 'shell', 'terminal': 'shell',
|
|
'edit': 'edit', 'Edit': 'edit', 'write': 'write', 'Write': 'write',
|
|
'create': 'create', 'Create': 'create',
|
|
'read_file': 'read', 'Read': 'read', 'view': 'view',
|
|
'grep': 'search', 'Grep': 'search',
|
|
'web_search': 'web-search', 'WebSearch': 'web-search',
|
|
'web_fetch': 'web-fetch', 'WebFetch': 'web-fetch',
|
|
'patch': 'patch', 'apply_patch': 'patch',
|
|
'git': 'git', 'task': 'task', 'Task': 'task',
|
|
'glob': 'find', 'Glob': 'find',
|
|
'ask_user': 'ask', 'AskUserQuestion': 'ask',
|
|
'notebookedit': 'notebook', 'NotebookEdit': 'notebook'
|
|
};
|
|
return map[toolName] || toolName.toLowerCase().replace(/_/g, '-');
|
|
}
|
|
|
|
function generateSkillMd(skillName, pattern) {
|
|
const tools = pattern.sequence.map(s => `\`${s.toolName}\``);
|
|
const uniqueTools = [...new Set(tools)];
|
|
|
|
const steps = pattern.sequence.map((s, i) => {
|
|
const context = s.argsSummary.fileExt
|
|
? ` (typically with ${s.argsSummary.fileExt} files)`
|
|
: s.argsSummary.commandCategory
|
|
? ` (typically ${s.argsSummary.commandCategory} commands)`
|
|
: '';
|
|
return `${i + 1}. **\`${s.toolName}\`**${context}`;
|
|
});
|
|
|
|
return `---
|
|
name: ${skillName}
|
|
description: >
|
|
Auto-detected workflow: ${uniqueTools.join(', ')}.
|
|
Detected across ${pattern.sessions.length} sessions (${pattern.totalOccurrences} occurrences).
|
|
Triggers when you follow a similar pattern of tool calls.
|
|
argument-hint: 'Describe what you want to do with this workflow'
|
|
---
|
|
|
|
# ${skillName}
|
|
|
|
> 🤖 **Auto-generated by Skill Factory**
|
|
> Detected across **${pattern.sessions.length} sessions** with **${pattern.totalOccurrences} occurrences**.
|
|
> First seen: ${pattern.firstSeen} | Last seen: ${pattern.lastSeen}
|
|
|
|
## Detected Pattern
|
|
|
|
This skill was automatically detected from your repeated tool-call sequences:
|
|
|
|
${steps.join('\n')}
|
|
|
|
## When to Activate
|
|
|
|
Use this skill when you're about to perform a workflow involving ${uniqueTools.join(', ')}.
|
|
|
|
## Workflow
|
|
|
|
${pattern.sequence.map((s, i) => `${i + 1}. Run \`${s.toolName}\` to ${describeToolAction(s)}`).join('\n')}
|
|
|
|
## Notes
|
|
|
|
- This is a **proposed** skill — review and customize it before relying on it.
|
|
- Edit this file to add specific instructions, pitfalls, and examples.
|
|
- Once reviewed, move it from \`proposed/\` to the main skills directory to make it active.
|
|
|
|
---
|
|
*Generated by [copilot-skill-factory](https://github.com/droideparanoico/copilot-skill-factory)*
|
|
`;
|
|
}
|
|
|
|
function describeToolAction(step) {
|
|
const { toolName, argsSummary } = step;
|
|
const ext = argsSummary.fileExt ? ` (${argsSummary.fileExt} files)` : '';
|
|
const cmd = argsSummary.commandCategory ? ` (${argsSummary.commandCategory})` : '';
|
|
|
|
switch (toolName) {
|
|
case 'bash': case 'Bash': case 'terminal':
|
|
return `execute shell commands${cmd}${ext}`;
|
|
case 'edit': case 'Edit':
|
|
return `edit files${ext}`;
|
|
case 'write': case 'Write': case 'create': case 'Create':
|
|
return `create/write files${ext}`;
|
|
case 'read_file': case 'Read': case 'view': case 'View':
|
|
return `read files${ext}`;
|
|
case 'grep': case 'Grep':
|
|
return `search code${ext}`;
|
|
case 'web_search': case 'WebSearch':
|
|
return `search the web`;
|
|
case 'web_fetch': case 'WebFetch':
|
|
return `fetch web content`;
|
|
case 'patch': case 'apply_patch':
|
|
return `apply patches${ext}`;
|
|
case 'glob': case 'Glob':
|
|
return `find files`;
|
|
case 'task': case 'Task':
|
|
return `delegate to subagent`;
|
|
default:
|
|
return `perform operation`;
|
|
}
|
|
}
|
|
|
|
// ── sessionStart handler (inject context about detected skills) ─────────────
|
|
|
|
function handleSessionStart(input) {
|
|
const db = loadDb();
|
|
|
|
// Find proposed skills the user hasn't reviewed yet
|
|
let proposedSkills = [];
|
|
try {
|
|
const proposedDir = path.join(skillsDir(), 'proposed');
|
|
if (fs.existsSync(proposedDir)) {
|
|
proposedSkills = fs.readdirSync(proposedDir, { withFileTypes: true })
|
|
.filter(d => d.isDirectory())
|
|
.map(d => d.name);
|
|
}
|
|
} catch {}
|
|
|
|
// Find patterns ripe for proposal (seen in 2+ sessions, no skill yet)
|
|
const ripePatterns = Object.entries(db.patterns)
|
|
.filter(([, p]) => p.sessions.length >= 2 && !p.proposedSkill)
|
|
.map(([, p]) => p);
|
|
|
|
const totalProposed = proposedSkills.length;
|
|
const totalRipe = ripePatterns.length;
|
|
|
|
if (totalProposed === 0 && totalRipe === 0) {
|
|
// Nothing to report — silent
|
|
return;
|
|
}
|
|
|
|
// Emit additionalContext so the agent knows about skill factory
|
|
let context = '';
|
|
|
|
if (totalProposed > 0) {
|
|
context += `🏭 **Skill Factory** has ${totalProposed} proposed skill(s) ready for review.\n`;
|
|
context += `Use \`/skill-factory review\` to review them.\n`;
|
|
}
|
|
|
|
if (totalRipe > 0) {
|
|
context += `📊 **Skill Factory** detected ${totalRipe} repeatable workflow pattern(s) across your sessions.\n`;
|
|
context += `Use \`/skill-factory propose\` to see what can be turned into a skill.\n`;
|
|
}
|
|
|
|
// sessionStart command hooks can inject additionalContext via stdout
|
|
process.stdout.write(JSON.stringify({ additionalContext: context }));
|
|
}
|
|
|
|
// ── cleanup ────────────────────────────────────────────────────────────────
|
|
|
|
function cleanupOldSessions(db) {
|
|
const maxSessions = 20;
|
|
const sessions = Object.keys(db.sessions);
|
|
if (sessions.length <= maxSessions) return;
|
|
|
|
// Keep newest, delete oldest session logs
|
|
const toDelete = sessions
|
|
.sort((a, b) => (db.sessions[b].analyzedAt || '').localeCompare(db.sessions[a].analyzedAt || ''))
|
|
.slice(maxSessions);
|
|
|
|
for (const sid of toDelete) {
|
|
const logPath = sessionLogPath(sid);
|
|
try { fs.unlinkSync(logPath); } catch {}
|
|
delete db.sessions[sid];
|
|
}
|
|
saveDb(db);
|
|
}
|
|
|
|
// ── main ───────────────────────────────────────────────────────────────────
|
|
|
|
const eventType = process.argv[2];
|
|
if (!eventType) {
|
|
console.error('Usage: observer.js <postToolUse|sessionEnd|sessionStart>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const input = readStdin();
|
|
if (!input) {
|
|
// No input — exit silently
|
|
process.exit(0);
|
|
}
|
|
|
|
switch (eventType) {
|
|
case 'postToolUse':
|
|
handlePostToolUse(input);
|
|
break;
|
|
case 'sessionEnd':
|
|
handleSessionEnd(input);
|
|
break;
|
|
case 'sessionStart':
|
|
handleSessionStart(input);
|
|
break;
|
|
default:
|
|
console.error(`Unknown event: ${eventType}`);
|
|
process.exit(1);
|
|
}
|