commit 94cfd80dd0a8fba02c9f41c266d8adde0d8ad3a4 Author: droideparanoico Date: Fri Jun 19 08:22:11 2026 +0000 Initial commit: copilot-skill-factory v0.1.0 Meta-plugin for GitHub Copilot CLI that watches workflows and auto-generates reusable agent skills. - postToolUse hook: fingerprints and logs tool calls per session - sessionEnd hook: sliding-window pattern detection across sessions - sessionStart hook: injects context about proposed skills - /skill-factory slash command for reviewing/managing skills - 13 passing tests validating observer, analyzer, and edge cases Inspired by hermes-skill-factory (Romanescu11/hermes-skill-factory) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b31944a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.data/ +*.log +.DS_Store diff --git a/README.md b/README.md new file mode 100755 index 0000000..8aaa638 --- /dev/null +++ b/README.md @@ -0,0 +1,194 @@ +# šŸ­ Copilot Skill Factory + +> **The meta-plugin that creates skills.** +> Watches your GitHub Copilot CLI workflows and automatically turns them into reusable agent skills. + +Built for [GitHub Copilot CLI](https://docs.github.com/copilot/concepts/agents/about-copilot-cli) (v1.0.52+). + +Inspired by [hermes-skill-factory](https://github.com/Romanescu11/hermes-skill-factory) for [Hermes Agent](https://github.com/NousResearch/hermes-agent). + +--- + +## What It Does + +Every time you solve a problem with Copilot CLI, you're performing a workflow worth repeating. Skill Factory watches silently via hooks, detects patterns, and proposes reusable skills. + +``` +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +šŸ­ SKILL FACTORY — New Pattern Detected +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +You've repeated this workflow in 3 sessions: + + 1. bash (git commands) + 2. grep (search code) + 3. edit (apply changes) + 4. bash (run tests) + +Proposed Skill: git-edit-test +Status: Ready for review + +Use /skill-factory review to inspect it. +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +## How It Works + +### Architecture + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +│ Copilot CLI Session │ +│ │ +│ postToolUse ──┐ sessionEnd ──┐ │ +│ ā–¼ ā–¼ │ +│ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ +│ │ observer.js │ │ analyzer │ │ +│ │ (logs tools)│ │ (detects │ │ +│ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ patterns) │ │ +│ │ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ +│ ā–¼ ā–¼ │ +│ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ +│ │ pattern-db.json │ │ +│ │ (cross-session patterns) │ │ +│ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ +│ ā–¼ │ +│ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” │ +│ │ proposed//SKILL.md │ │ +│ │ (auto-generated skills) │ │ +│ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ │ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + +sessionStart ──► Injects notification about new skills + +/skill-factory ──► User reviews & manages skills +``` + +### Plugin Components + +| Component | File | Purpose | +|---|---|---| +| **Manifest** | `plugin.json` | Plugin metadata (name, version, hooks path) | +| **Hooks config** | `hooks/hooks.json` | Defines `postToolUse`, `sessionEnd`, `sessionStart` hooks | +| **Observer** | `bin/observer.js` | Main hook handler — logs, analyzes, notifies | +| **Skill** | `skills/skill-factory/SKILL.md` | `/skill-factory` slash command for user interaction | + +### Detection Algorithm + +1. **postToolUse hook** — Every successful tool call is fingerprinted (tool name + normalized args) and logged to a per-session JSONL file +2. **sessionEnd hook** — Analyzes the session using sliding windows (3-8 consecutive tool calls). Matching sequences across 2+ sessions are flagged as "patterns" +3. **Auto-generation** — Patterns seen in 2+ different sessions get a proposed `SKILL.md` generated automatically +4. **sessionStart hook** — Next session, the agent is notified about pending proposals via `additionalContext` +5. **User review** — The `/skill-factory` skill lets the user review, accept, edit, or dismiss proposals + +## Installation + +### Prerequisites + +- [GitHub Copilot CLI](https://docs.github.com/copilot/concepts/agents/about-copilot-cli) v1.0.52+ installed and authenticated +- Node.js 18+ (for the hook scripts) + +### Via Copilot CLI plugin install (recommended) + +```bash +copilot plugin install droideparanoico/copilot-skill-factory +``` + +Then install the companion skill: + +```bash +mkdir -p ~/.copilot/skills/skill-factory +cp skills/skill-factory/SKILL.md ~/.copilot/skills/skill-factory/ +``` + +### Manual installation + +```bash +# Clone the repo +git clone https://github.com/droideparanoico/copilot-skill-factory.git +cd copilot-skill-factory + +# Install plugin +mkdir -p ~/.copilot/plugins/skill-factory +cp plugin.json ~/.copilot/plugins/skill-factory/ +cp -r hooks/ ~/.copilot/plugins/skill-factory/ +cp -r bin/ ~/.copilot/plugins/skill-factory/ +chmod +x ~/.copilot/plugins/skill-factory/bin/observer.js + +# Install companion skill +mkdir -p ~/.copilot/skills/skill-factory +cp skills/skill-factory/SKILL.md ~/.copilot/skills/skill-factory/ +``` + +Then restart Copilot CLI. + +## Usage + +Once installed, Skill Factory runs silently in the background. Use the slash command to interact: + +| Command | What it does | +|---|---| +| `/skill-factory` | Show available commands | +| `/skill-factory review` | Review proposed skills (accept/edit/dismiss) | +| `/skill-factory propose` | Force-analyze current session patterns | +| `/skill-factory list` | List all generated skills | +| `/skill-factory status` | Show detection statistics | +| `/skill-factory save ` | Save with custom name | +| `/skill-factory dismiss ` | Dismiss a proposal | +| `/skill-factory clear` | Reset all data | + +You can also tell Copilot naturally: +- *"Review my skill factory proposals"* +- *"Save this workflow as a skill"* +- *"Turn this into a reusable skill"* + +## Configuration + +Skill Factory stores data in: + +``` +~/.copilot/skills/skill-factory/.data/ +ā”œā”€ā”€ pattern-db.json # Cross-session pattern database +└── sessions/ + └── .jsonl # Per-session tool call logs (kept for last 20 sessions) +``` + +Proposed skills are stored in: + +``` +~/.copilot/skills/skill-factory/proposed/ +└── / + ā”œā”€ā”€ SKILL.md # Generated skill definition + └── .proposal.json # Detection metadata +``` + +## Limitations + +- **Not real-time**: Patterns are analyzed at session end, not during the session +- **Heuristic-based**: Detection uses fingerprint similarity — similar but not identical workflows may not match +- **No cross-device sync**: Pattern database is local to each machine +- **Windows**: PowerShell hooks are not yet implemented (contributions welcome!) + +## Roadmap + +- [ ] Real-time pattern suggestion during sessions +- [ ] Cross-device sync via GitHub Gist +- [ ] Pattern confidence scoring +- [ ] Automatic skill refinement (merge similar patterns) +- [ ] Web dashboard for pattern visualization +- [ ] Export/import pattern database + +## Comparison with hermes-skill-factory + +| Feature | hermes-skill-factory | copilot-skill-factory | +|---|---|---| +| Platform | Hermes Agent | GitHub Copilot CLI | +| Observation | In-process (Python plugin) | Hooks (external Node.js) | +| Detection | AI-assisted pattern analysis | Heuristic fingerprint matching | +| Skill format | SKILL.md + plugin.py | SKILL.md (Copilot agent skill) | +| User interaction | Inline prompt during session | Slash command + sessionStart context | +| Slash commands | `/skill-factory propose`, etc. | `/skill-factory review`, etc. | + +## License + +MIT diff --git a/bin/observer.js b/bin/observer.js new file mode 100755 index 0000000..db099a8 --- /dev/null +++ b/bin/observer.js @@ -0,0 +1,526 @@ +#!/usr/bin/env node + +/** + * copilot-skill-factory — Observer + * + * Handles three hook events dispatched by Copilot CLI: + * postToolUse — logs every successful tool call to a per-session JSONL file + * sessionEnd — analyzes the session log, updates the pattern database + * sessionStart — injects additionalContext about recently detected patterns + * + * Input: JSON payload on stdin (camelCase format from hooks.json) + * Output: JSON to stdout for hooks that support output (sessionStart → additionalContext) + */ + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +// ── helpers ──────────────────────────────────────────────────────────────── + +function readStdin() { + try { + const raw = fs.readFileSync(0, 'utf8'); + if (!raw.trim()) return null; + return JSON.parse(raw); + } catch { + return null; + } +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function dataDir() { + const fromEnv = process.env.COPILOT_PLUGIN_DATA || process.env.CLAUDE_PLUGIN_DATA; + if (fromEnv && fromEnv.trim() && !fromEnv.trim().startsWith('${')) return fromEnv; + // Fallback: use plugin root directory + const root = process.env.PLUGIN_ROOT || process.env.COPILOT_PLUGIN_ROOT || process.env.CLAUDE_PLUGIN_ROOT || __dirname; + return path.join(root, '.data'); +} + +function skillsDir() { + const home = process.env.HOME || process.env.USERPROFILE || '/tmp'; + return path.join(home, '.copilot', 'skills', 'skill-factory'); +} + +function dbPath() { + return path.join(dataDir(), 'pattern-db.json'); +} + +function sessionLogPath(sessionId) { + return path.join(dataDir(), 'sessions', `${sessionId}.jsonl`); +} + +function loadDb() { + try { + return JSON.parse(fs.readFileSync(dbPath(), 'utf8')); + } catch { + return { patterns: {}, sessions: {}, meta: { version: 1, createdAt: new Date().toISOString() } }; + } +} + +function saveDb(db) { + ensureDir(dataDir()); + fs.writeFileSync(dbPath(), JSON.stringify(db, null, 2)); +} + +// ── tool-call fingerprinting ─────────────────────────────────────────────── + +/** + * Creates a fingerprint for a tool call — captures the "what" without the + * session-specific parameters (file paths, timestamps, exact content). + * This lets us match similar tool calls across sessions. + */ +function fingerprint(toolName, toolArgs) { + const normalized = normalizeArgs(toolName, toolArgs); + return `${toolName}:${hash(normalized)}`; +} + +function normalizeArgs(toolName, toolArgs) { + if (!toolArgs || typeof toolArgs !== 'object') return '{}'; + + const args = { ...toolArgs }; + + // Strip file paths (replace with placeholder based on extension) + const fileFields = ['path', 'filePath', 'file_path', 'file', 'output', 'target']; + for (const f of fileFields) { + if (args[f] && typeof args[f] === 'string') { + const ext = path.extname(args[f]); + args[f] = ``; + } + } + + // Strip command content (replace with command length) + if (args.command && typeof args.command === 'string') { + args.command = ``; + } + + // Strip content/patch bodies (replace with length) + const contentFields = ['content', 'new_string', 'old_string', 'new_str', 'old_str', + 'patch', 'text', 'body', 'message', 'code', 'script']; + for (const f of contentFields) { + if (args[f] && typeof args[f] === 'string') { + args[f] = ``; + } + } + + // Simplify arrays + for (const [k, v] of Object.entries(args)) { + if (Array.isArray(v)) args[k] = ``; + } + + return JSON.stringify(args, Object.keys(args).sort()); +} + +function hash(str) { + return crypto.createHash('md5').update(str).digest('hex').slice(0, 12); +} + +// ── postToolUse handler ──────────────────────────────────────────────────── + +function handlePostToolUse(input) { + const { sessionId, toolName, toolArgs } = input; + + // Skip noisy/redundant tools + const skipTools = ['todo_write', 'update_todo', 'TodoWrite', 'ask_user', 'AskUserQuestion']; + if (skipTools.includes(toolName)) return; + + const fp = fingerprint(toolName, toolArgs); + const entry = { + ts: new Date().toISOString(), + sessionId, + toolName, + fingerprint: fp, + // Store minimal args for context (no full content) + argsSummary: summarizeArgs(toolName, toolArgs) + }; + + ensureDir(path.dirname(sessionLogPath(sessionId))); + fs.appendFileSync(sessionLogPath(sessionId), JSON.stringify(entry) + '\n'); +} + +function summarizeArgs(toolName, toolArgs) { + if (!toolArgs || typeof toolArgs !== 'object') return {}; + + const summary = {}; + + // Capture file pattern hints + const fileFields = ['path', 'filePath', 'file_path', 'file']; + for (const f of fileFields) { + if (toolArgs[f] && typeof toolArgs[f] === 'string') { + summary.fileExt = path.extname(toolArgs[f]) || undefined; + break; + } + } + + // Capture command category hints + if (toolArgs.command && typeof toolArgs.command === 'string') { + const cmd = toolArgs.command.trim(); + const firstWord = cmd.split(/\s+/)[0]; + summary.commandCategory = firstWord; + } + + // Tool-specific hints + switch (toolName) { + case 'bash': + case 'Bash': + case 'terminal': + summary.commandCategory = summary.commandCategory || 'shell'; + break; + case 'edit': + case 'Edit': + case 'write': + case 'Write': + case 'create': + case 'Create': + summary.operation = 'write'; + break; + case 'read_file': + case 'Read': + case 'view': + case 'View': + summary.operation = 'read'; + break; + case 'grep': + case 'Grep': + case 'search': + summary.operation = 'search'; + break; + case 'web_fetch': + case 'WebFetch': + case 'web_search': + case 'WebSearch': + summary.operation = 'web'; + break; + } + + return summary; +} + +// ── sessionEnd handler (pattern analysis) ────────────────────────────────── + +function handleSessionEnd(input) { + const { sessionId, reason } = input; + + // Skip failed/aborted sessions + if (reason === 'error' || reason === 'abort') return; + + const logPath = sessionLogPath(sessionId); + if (!fs.existsSync(logPath)) return; + + const entries = []; + try { + const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n'); + for (const line of lines) { + if (line.trim()) entries.push(JSON.parse(line)); + } + } catch { + return; + } + + if (entries.length < 3) return; // Too short to be a pattern + + const db = loadDb(); + + // Extract sequences (sliding window of 3-8 consecutive tool calls) + for (let windowSize = 3; windowSize <= Math.min(8, entries.length); windowSize++) { + for (let i = 0; i <= entries.length - windowSize; i++) { + const window = entries.slice(i, i + windowSize); + const seqKey = window.map(e => e.fingerprint).join('→'); + + if (!db.patterns[seqKey]) { + db.patterns[seqKey] = { + sequence: window.map(e => ({ + toolName: e.toolName, + fingerprint: e.fingerprint, + argsSummary: e.argsSummary + })), + sessions: [], + totalOccurrences: 0, + firstSeen: new Date().toISOString(), + lastSeen: new Date().toISOString(), + proposedSkill: null + }; + } + + const pattern = db.patterns[seqKey]; + if (!pattern.sessions.includes(sessionId)) { + pattern.sessions.push(sessionId); + } + pattern.totalOccurrences++; + pattern.lastSeen = new Date().toISOString(); + } + } + + // Mark session as analyzed + db.sessions[sessionId] = { + analyzedAt: new Date().toISOString(), + toolCount: entries.length, + reason + }; + + saveDb(db); + + // Generate proposed skills for patterns seen in 2+ sessions + generateProposedSkills(db); + + // Clean up session log (keep last 10 for future reference) + cleanupOldSessions(db); +} + +// ── skill generation ─────────────────────────────────────────────────────── + +function generateProposedSkills(db) { + const candidates = []; + + for (const [key, pattern] of Object.entries(db.patterns)) { + // Only propose patterns seen in 2+ different sessions + if (pattern.sessions.length >= 2 && !pattern.proposedSkill) { + candidates.push({ key, pattern }); + } + } + + if (candidates.length === 0) return; + + // Sort by session count (most repeated first), take top 3 + candidates.sort((a, b) => b.pattern.sessions.length - a.pattern.sessions.length); + const topCandidates = candidates.slice(0, 3); + + for (const { key, pattern } of topCandidates) { + const skillName = generateSkillName(pattern); + const skillMd = generateSkillMd(skillName, pattern); + + // Write proposed SKILL.md + const skillDir = path.join(skillsDir(), 'proposed', skillName); + ensureDir(skillDir); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), skillMd); + + // Write proposal metadata + fs.writeFileSync(path.join(skillDir, '.proposal.json'), JSON.stringify({ + patternKey: key, + detectedAt: new Date().toISOString(), + sessionCount: pattern.sessions.length, + totalOccurrences: pattern.totalOccurrences, + status: 'proposed' + }, null, 2)); + + pattern.proposedSkill = skillName; + } + + saveDb(db); +} + +function generateSkillName(pattern) { + // Derive a name from the dominant tool types in the sequence + const toolCounts = {}; + for (const step of pattern.sequence) { + toolCounts[step.toolName] = (toolCounts[step.toolName] || 0) + 1; + } + + const dominant = Object.entries(toolCounts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 2) + .map(([name]) => simplifyToolName(name)); + + const name = dominant.join('-') || 'generic-workflow'; + return name.slice(0, 40).replace(/[^a-z0-9-]/g, '-').replace(/-+/g, '-'); +} + +function simplifyToolName(toolName) { + const map = { + 'bash': 'shell', 'Bash': 'shell', 'terminal': 'shell', + 'edit': 'edit', 'Edit': 'edit', 'write': 'write', 'Write': 'write', + 'create': 'create', 'Create': 'create', + 'read_file': 'read', 'Read': 'read', 'view': 'view', + 'grep': 'search', 'Grep': 'search', + 'web_search': 'web-search', 'WebSearch': 'web-search', + 'web_fetch': 'web-fetch', 'WebFetch': 'web-fetch', + 'patch': 'patch', 'apply_patch': 'patch', + 'git': 'git', 'task': 'task', 'Task': 'task', + 'glob': 'find', 'Glob': 'find', + 'ask_user': 'ask', 'AskUserQuestion': 'ask', + 'notebookedit': 'notebook', 'NotebookEdit': 'notebook' + }; + return map[toolName] || toolName.toLowerCase().replace(/_/g, '-'); +} + +function generateSkillMd(skillName, pattern) { + const tools = pattern.sequence.map(s => `\`${s.toolName}\``); + const uniqueTools = [...new Set(tools)]; + + const steps = pattern.sequence.map((s, i) => { + const context = s.argsSummary.fileExt + ? ` (typically with ${s.argsSummary.fileExt} files)` + : s.argsSummary.commandCategory + ? ` (typically ${s.argsSummary.commandCategory} commands)` + : ''; + return `${i + 1}. **\`${s.toolName}\`**${context}`; + }); + + return `--- +name: ${skillName} +description: > + Auto-detected workflow: ${uniqueTools.join(', ')}. + Detected across ${pattern.sessions.length} sessions (${pattern.totalOccurrences} occurrences). + Triggers when you follow a similar pattern of tool calls. +argument-hint: 'Describe what you want to do with this workflow' +--- + +# ${skillName} + +> šŸ¤– **Auto-generated by Skill Factory** +> Detected across **${pattern.sessions.length} sessions** with **${pattern.totalOccurrences} occurrences**. +> First seen: ${pattern.firstSeen} | Last seen: ${pattern.lastSeen} + +## Detected Pattern + +This skill was automatically detected from your repeated tool-call sequences: + +${steps.join('\n')} + +## When to Activate + +Use this skill when you're about to perform a workflow involving ${uniqueTools.join(', ')}. + +## Workflow + +${pattern.sequence.map((s, i) => `${i + 1}. Run \`${s.toolName}\` to ${describeToolAction(s)}`).join('\n')} + +## Notes + +- This is a **proposed** skill — review and customize it before relying on it. +- Edit this file to add specific instructions, pitfalls, and examples. +- Once reviewed, move it from \`proposed/\` to the main skills directory to make it active. + +--- +*Generated by [copilot-skill-factory](https://github.com/droideparanoico/copilot-skill-factory)* +`; +} + +function describeToolAction(step) { + const { toolName, argsSummary } = step; + const ext = argsSummary.fileExt ? ` (${argsSummary.fileExt} files)` : ''; + const cmd = argsSummary.commandCategory ? ` (${argsSummary.commandCategory})` : ''; + + switch (toolName) { + case 'bash': case 'Bash': case 'terminal': + return `execute shell commands${cmd}${ext}`; + case 'edit': case 'Edit': + return `edit files${ext}`; + case 'write': case 'Write': case 'create': case 'Create': + return `create/write files${ext}`; + case 'read_file': case 'Read': case 'view': case 'View': + return `read files${ext}`; + case 'grep': case 'Grep': + return `search code${ext}`; + case 'web_search': case 'WebSearch': + return `search the web`; + case 'web_fetch': case 'WebFetch': + return `fetch web content`; + case 'patch': case 'apply_patch': + return `apply patches${ext}`; + case 'glob': case 'Glob': + return `find files`; + case 'task': case 'Task': + return `delegate to subagent`; + default: + return `perform operation`; + } +} + +// ── sessionStart handler (inject context about detected skills) ───────────── + +function handleSessionStart(input) { + const db = loadDb(); + + // Find proposed skills the user hasn't reviewed yet + let proposedSkills = []; + try { + const proposedDir = path.join(skillsDir(), 'proposed'); + if (fs.existsSync(proposedDir)) { + proposedSkills = fs.readdirSync(proposedDir, { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => d.name); + } + } catch {} + + // Find patterns ripe for proposal (seen in 2+ sessions, no skill yet) + const ripePatterns = Object.entries(db.patterns) + .filter(([, p]) => p.sessions.length >= 2 && !p.proposedSkill) + .map(([, p]) => p); + + const totalProposed = proposedSkills.length; + const totalRipe = ripePatterns.length; + + if (totalProposed === 0 && totalRipe === 0) { + // Nothing to report — silent + return; + } + + // Emit additionalContext so the agent knows about skill factory + let context = ''; + + if (totalProposed > 0) { + context += `šŸ­ **Skill Factory** has ${totalProposed} proposed skill(s) ready for review.\n`; + context += `Use \`/skill-factory review\` to review them.\n`; + } + + if (totalRipe > 0) { + context += `šŸ“Š **Skill Factory** detected ${totalRipe} repeatable workflow pattern(s) across your sessions.\n`; + context += `Use \`/skill-factory propose\` to see what can be turned into a skill.\n`; + } + + // sessionStart command hooks can inject additionalContext via stdout + process.stdout.write(JSON.stringify({ additionalContext: context })); +} + +// ── cleanup ──────────────────────────────────────────────────────────────── + +function cleanupOldSessions(db) { + const maxSessions = 20; + const sessions = Object.keys(db.sessions); + if (sessions.length <= maxSessions) return; + + // Keep newest, delete oldest session logs + const toDelete = sessions + .sort((a, b) => (db.sessions[b].analyzedAt || '').localeCompare(db.sessions[a].analyzedAt || '')) + .slice(maxSessions); + + for (const sid of toDelete) { + const logPath = sessionLogPath(sid); + try { fs.unlinkSync(logPath); } catch {} + delete db.sessions[sid]; + } + saveDb(db); +} + +// ── main ─────────────────────────────────────────────────────────────────── + +const eventType = process.argv[2]; +if (!eventType) { + console.error('Usage: observer.js '); + process.exit(1); +} + +const input = readStdin(); +if (!input) { + // No input — exit silently + process.exit(0); +} + +switch (eventType) { + case 'postToolUse': + handlePostToolUse(input); + break; + case 'sessionEnd': + handleSessionEnd(input); + break; + case 'sessionStart': + handleSessionStart(input); + break; + default: + console.error(`Unknown event: ${eventType}`); + process.exit(1); +} diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..a1f2dfe --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "hooks": { + "postToolUse": [ + { + "type": "command", + "matcher": "", + "bash": "root=\"${PLUGIN_ROOT:-${COPILOT_PLUGIN_ROOT:-.}}\"; node \"$root/bin/observer.js\" postToolUse", + "timeoutSec": 5 + } + ], + "sessionEnd": [ + { + "type": "command", + "bash": "root=\"${PLUGIN_ROOT:-${COPILOT_PLUGIN_ROOT:-.}}\"; node \"$root/bin/observer.js\" sessionEnd", + "timeoutSec": 30 + } + ], + "sessionStart": [ + { + "type": "command", + "bash": "root=\"${PLUGIN_ROOT:-${COPILOT_PLUGIN_ROOT:-.}}\"; node \"$root/bin/observer.js\" sessionStart", + "timeoutSec": 10 + } + ] + } +} diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..9c5364b --- /dev/null +++ b/install.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +PLUGIN_NAME="copilot-skill-factory" +PLUGIN_DIR="${HOME}/.copilot/plugins/${PLUGIN_NAME}" +SKILL_DIR="${HOME}/.copilot/skills/skill-factory" + +echo "šŸ­ Installing ${PLUGIN_NAME}..." + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Install plugin +echo " → Installing plugin to ${PLUGIN_DIR}" +mkdir -p "${PLUGIN_DIR}" +cp "${SCRIPT_DIR}/plugin.json" "${PLUGIN_DIR}/" +cp -r "${SCRIPT_DIR}/hooks" "${PLUGIN_DIR}/" +cp -r "${SCRIPT_DIR}/bin" "${PLUGIN_DIR}/" +chmod +x "${PLUGIN_DIR}/bin/observer.js" + +# Install companion skill +echo " → Installing skill to ${SKILL_DIR}" +mkdir -p "${SKILL_DIR}" +cp "${SCRIPT_DIR}/skills/skill-factory/SKILL.md" "${SKILL_DIR}/" + +# Verify Node.js is available +if command -v node &>/dev/null; then + echo " āœ“ Node.js $(node --version) detected" +else + echo " ⚠ Node.js not found — hooks require Node.js 18+" +fi + +echo "" +echo "āœ… ${PLUGIN_NAME} installed successfully!" +echo "" +echo "Next steps:" +echo " 1. Restart Copilot CLI" +echo " 2. Work normally — Skill Factory watches silently" +echo " 3. After a few sessions, run /skill-factory review" +echo "" +echo "Data stored at: ${SKILL_DIR}/.data/" diff --git a/plugin.json b/plugin.json new file mode 100644 index 0000000..bd52cbc --- /dev/null +++ b/plugin.json @@ -0,0 +1,26 @@ +{ + "name": "copilot-skill-factory", + "description": "Meta-plugin that watches your Copilot CLI workflows and automatically proposes reusable agent skills. Detects repeatable tool-call patterns across sessions and generates SKILL.md files — just like hermes-skill-factory for Hermes Agent.", + "version": "0.1.0", + "author": { + "name": "Skill Factory" + }, + "homepage": "https://github.com/droideparanoico/copilot-skill-factory", + "repository": "https://github.com/droideparanoico/copilot-skill-factory", + "license": "MIT", + "keywords": [ + "skill-factory", + "meta-skill", + "skill-generator", + "automation", + "productivity", + "workflow" + ], + "category": "productivity", + "tags": [ + "skill-generation", + "automation", + "meta" + ], + "hooks": "hooks/hooks.json" +} diff --git a/skills/skill-factory/SKILL.md b/skills/skill-factory/SKILL.md new file mode 100755 index 0000000..4aba50c --- /dev/null +++ b/skills/skill-factory/SKILL.md @@ -0,0 +1,91 @@ +--- +name: skill-factory +description: > + Meta-skill for managing automatically detected workflows. Review proposed skills, + inspect detected patterns, and turn your repeated tool-call sequences into + reusable Copilot CLI agent skills. Use /skill-factory review to see what's ready. +argument-hint: '[review|propose|list|status|save |dismiss |clear]' +--- + +# šŸ­ Skill Factory + +> **The meta-skill that creates skills.** +> Automatically watches your workflows and turns them into reusable agent skills. + +## Commands + +When the user types `/skill-factory` (with or without arguments), follow this guide: + +### `/skill-factory review` +Show all proposed skills ready for review. For each one: +1. Read the `SKILL.md` in `~/.copilot/skills/skill-factory/proposed//SKILL.md` +2. Read the `.proposal.json` for metadata (session count, occurrences) +3. Present a summary to the user and ask if they want to: + - **Accept** — move to `~/.copilot/skills//` (makes it active) + - **Edit** — open the SKILL.md for editing first + - **Dismiss** — archive to `~/.copilot/skills/skill-factory/dismissed/` + - **Skip** — leave as-is for later + +### `/skill-factory propose` +Force analysis of current session patterns. Read the pattern database: +- `~/.copilot/skills/skill-factory/.data/pattern-db.json` (or in the plugin's data directory) +- Show the top 5 most frequent patterns (by session count) +- For each, describe the tool-call sequence and ask if the user wants to generate a skill + +### `/skill-factory list` +List all skills generated by Skill Factory (both proposed and accepted): +- `~/.copilot/skills/skill-factory/proposed/` — proposed +- `~/.copilot/skills//` — accepted (look for `.factory.json` marker) + +### `/skill-factory status` +Show Skill Factory statistics: +- Number of sessions tracked +- Number of patterns detected +- Number of proposed/accepted/dismissed skills +- Read from `~/.copilot/skills/skill-factory/.data/pattern-db.json` + +### `/skill-factory save ` +Save the last proposal with a custom name. Used when the user wants to rename a proposed skill. + +### `/skill-factory dismiss ` +Dismiss a proposed skill. Moves it to `~/.copilot/skills/skill-factory/dismissed/`. + +### `/skill-factory clear` +Clear the pattern database and all proposed skills. Asks for confirmation first. + +## Data Locations + +| Data | Path | +|---|---| +| Pattern database | `~/.copilot/skills/skill-factory/.data/pattern-db.json` | +| Proposed skills | `~/.copilot/skills/skill-factory/proposed//` | +| Accepted skills | `~/.copilot/skills//` (with `.factory.json` marker) | +| Dismissed skills | `~/.copilot/skills/skill-factory/dismissed//` | +| Session logs | `~/.copilot/skills/skill-factory/.data/sessions/.jsonl` | + +## Accepting a Skill + +When the user accepts a proposed skill: +1. Read the `SKILL.md` from `proposed//` +2. Create `~/.copilot/skills//SKILL.md` with the content +3. Create `~/.copilot/skills//.factory.json` with: + ```json + { "generatedBy": "skill-factory", "generatedAt": "", "patternKey": "" } + ``` +4. Remove from `proposed//` +5. Tell the user the skill is now active and can be invoked with `/` + +## Dismissing a Skill + +When the user dismisses a proposed skill: +1. Move `proposed//` to `dismissed//` +2. Update the pattern in the DB to mark it as dismissed + +## Important Notes + +- The plugin (`copilot-skill-factory`) handles automatic pattern detection via hooks. +- This skill handles user interaction — reviewing, accepting, editing, dismissing. +- Always confirm before deleting or moving files. +- If the data directory doesn't exist yet, tell the user the plugin needs to run for at least one session first. +- Proposed skills are **suggestions** — they may need editing before they're useful. +- Encourage the user to customize accepted skills with specific instructions, pitfalls, and examples. diff --git a/test.js b/test.js new file mode 100644 index 0000000..045b25e --- /dev/null +++ b/test.js @@ -0,0 +1,356 @@ +#!/usr/bin/env node + +/** + * Test harness for copilot-skill-factory observer.js + * + * Simulates hook payloads to verify: + * 1. postToolUse logging works + * 2. sessionEnd analysis detects patterns + * 3. sessionStart injects context + * 4. Cross-session pattern detection triggers skill generation + */ + +const { execFileSync, execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const OBSERVER = path.join(__dirname, 'bin', 'observer.js'); +const DATA_DIR = path.join(__dirname, '.test-data'); + +// Override data directory for tests +process.env.COPILOT_PLUGIN_DATA = DATA_DIR; + +// Also override HOME for skills dir +process.env.HOME = DATA_DIR; + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + console.log(` āœ“ ${name}`); + passed++; + } catch (e) { + console.log(` āœ— ${name}: ${e.message}`); + failed++; + } +} + +function runObserver(event, payload) { + return execFileSync('node', [OBSERVER, event], { + input: JSON.stringify(payload), + encoding: 'utf8', + env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, + timeout: 5000 + }); +} + +function cleanup() { + try { fs.rmSync(DATA_DIR, { recursive: true, force: true }); } catch {} +} + +console.log('\nšŸ­ Skill Factory — Test Suite\n'); + +// ── postToolUse tests ────────────────────────────────────────────────────── + +console.log('postToolUse:'); +cleanup(); + +const SESSION_A = 'test-session-aaa'; +const SESSION_B = 'test-session-bbb'; + +test('logs a tool call', () => { + runObserver('postToolUse', { + sessionId: SESSION_A, + timestamp: Date.now(), + cwd: '/test/project', + toolName: 'bash', + toolArgs: { command: 'git status', description: 'check git status' } + }); + + const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`); + if (!fs.existsSync(logPath)) throw new Error('Session log not created'); + + const content = fs.readFileSync(logPath, 'utf8').trim(); + if (!content) throw new Error('Session log is empty'); + + const entry = JSON.parse(content.split('\n')[0]); + if (entry.toolName !== 'bash') throw new Error(`Expected bash, got ${entry.toolName}`); + if (!entry.fingerprint) throw new Error('Missing fingerprint'); + if (entry.argsSummary.commandCategory !== 'git') throw new Error(`Expected git category, got ${entry.argsSummary.commandCategory}`); +}); + +test('fingerprints are stable for same tool+args', () => { + const fp1 = runObserverAndGetFingerprint('bash', { command: 'git diff' }); + const fp2 = runObserverAndGetFingerprint('bash', { command: 'git diff --stat' }); + // Different command lengths → different fingerprints + if (fp1 === fp2) throw new Error('Different commands should have different fingerprints'); +}); + +test('fingerprints match for similar file operations', () => { + const fp1 = runObserverAndGetFingerprint('edit', { path: '/a/b/file1.py', old_string: 'abc' }); + const fp2 = runObserverAndGetFingerprint('edit', { path: '/x/y/file2.py', old_string: 'xyz' }); + // Same tool, file extension normalized, content normalized → same fingerprint + if (fp1 !== fp2) throw new Error('Similar edits should have same fingerprint'); +}); + +test('skips noisy tools (todo, ask)', () => { + runObserver('postToolUse', { + sessionId: SESSION_A, + timestamp: Date.now(), + cwd: '/test', + toolName: 'todo_write', + toolArgs: { todos: [] } + }); + + const logPath = path.join(DATA_DIR, 'sessions', `${SESSION_A}.jsonl`); + const content = fs.readFileSync(logPath, 'utf8').trim(); + const todoEntries = content.split('\n').filter(l => l.includes('todo_write')); + if (todoEntries.length > 0) throw new Error('todo_write should be skipped'); +}); + +function runObserverAndGetFingerprint(toolName, toolArgs) { + const tmpSession = 'tmp-' + Math.random().toString(36).slice(2, 8); + runObserver('postToolUse', { + sessionId: tmpSession, + timestamp: Date.now(), + cwd: '/test', + toolName, + toolArgs + }); + + const logPath = path.join(DATA_DIR, 'sessions', `${tmpSession}.jsonl`); + const entry = JSON.parse(fs.readFileSync(logPath, 'utf8').trim()); + return entry.fingerprint; +} + +// ── sessionEnd tests ─────────────────────────────────────────────────────── + +console.log('\nsessionEnd:'); +cleanup(); + +test('analyzes session and creates pattern DB', () => { + // Simulate a full session with a repeated workflow + const sessionId = 'test-session-1'; + const workflow = [ + { toolName: 'bash', toolArgs: { command: 'git pull' } }, + { toolName: 'grep', toolArgs: { pattern: 'TODO' } }, + { toolName: 'edit', toolArgs: { path: 'src/app.py', old_string: 'old', new_string: 'new' } }, + { toolName: 'bash', toolArgs: { command: 'pytest' } }, + ]; + + for (const call of workflow) { + runObserver('postToolUse', { + sessionId, + timestamp: Date.now(), + cwd: '/test/project', + ...call + }); + } + + runObserver('sessionEnd', { + sessionId, + timestamp: Date.now(), + cwd: '/test/project', + reason: 'complete' + }); + + // Check DB was created + const dbPath = path.join(DATA_DIR, 'pattern-db.json'); + if (!fs.existsSync(dbPath)) throw new Error('Pattern DB not created'); + + const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); + if (Object.keys(db.patterns).length === 0) throw new Error('No patterns detected'); + + if (!db.sessions[sessionId]) throw new Error('Session not recorded in DB'); + if (db.sessions[sessionId].toolCount !== 4) throw new Error(`Expected 4 tools, got ${db.sessions[sessionId].toolCount}`); +}); + +test('detects cross-session patterns and generates skills', () => { + cleanup(); + + // Session 1: Same workflow + const workflow = [ + { toolName: 'bash', toolArgs: { command: 'npm install' } }, + { toolName: 'edit', toolArgs: { path: 'src/index.ts', old_string: 'a', new_string: 'b' } }, + { toolName: 'bash', toolArgs: { command: 'npm test' } }, + ]; + + // Run 2 sessions with the same workflow + for (let s = 1; s <= 2; s++) { + const sid = `cross-session-${s}`; + for (const call of workflow) { + runObserver('postToolUse', { + sessionId: sid, + timestamp: Date.now(), + cwd: '/test/project', + ...call + }); + } + runObserver('sessionEnd', { + sessionId: sid, + timestamp: Date.now(), + cwd: '/test/project', + reason: 'complete' + }); + } + + // After 2 sessions with same pattern, a skill should be proposed + const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed'); + if (!fs.existsSync(proposedDir)) throw new Error('Proposed directory not created'); + + const proposed = fs.readdirSync(proposedDir, { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => d.name); + + if (proposed.length === 0) throw new Error('No skills proposed after cross-session patterns'); + + // Verify SKILL.md was generated + const skillMd = path.join(proposedDir, proposed[0], 'SKILL.md'); + if (!fs.existsSync(skillMd)) throw new Error('SKILL.md not generated'); + + const content = fs.readFileSync(skillMd, 'utf8'); + if (!content.includes('Auto-generated by Skill Factory')) throw new Error('SKILL.md missing auto-gen marker'); + if (!content.includes('2 sessions')) throw new Error('SKILL.md should mention 2 sessions'); + + console.log(` → Generated skill: ${proposed[0]}`); +}); + +test('does not generate skills for single-session patterns', () => { + cleanup(); + + const sid = 'single-session'; + const workflow = [ + { toolName: 'bash', toolArgs: { command: 'echo hello' } }, + { toolName: 'read_file', toolArgs: { path: 'test.txt' } }, + { toolName: 'edit', toolArgs: { path: 'test.txt', old_string: 'a', new_string: 'b' } }, + ]; + + for (const call of workflow) { + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', ...call }); + } + runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); + + const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed'); + const hasProposed = fs.existsSync(proposedDir) && fs.readdirSync(proposedDir).length > 0; + if (hasProposed) throw new Error('Should not propose skills from a single session'); +}); + +// ── sessionStart tests ───────────────────────────────────────────────────── + +console.log('\nsessionStart:'); +cleanup(); + +test('is silent when no proposals exist', () => { + const output = runObserver('sessionStart', { + sessionId: 'start-test', + timestamp: Date.now(), + cwd: '/test', + source: 'new' + }); + + if (output.trim()) throw new Error('Should be silent with no proposals'); +}); + +test('injects context when proposals exist', () => { + cleanup(); + + // Create a fake proposed skill + const proposedDir = path.join(DATA_DIR, '.copilot', 'skills', 'skill-factory', 'proposed', 'test-skill'); + fs.mkdirSync(proposedDir, { recursive: true }); + fs.writeFileSync(path.join(proposedDir, 'SKILL.md'), '# Test\n'); + + const output = runObserver('sessionStart', { + sessionId: 'start-test-2', + timestamp: Date.now(), + cwd: '/test', + source: 'new' + }); + + if (!output.trim()) throw new Error('Should inject context when proposals exist'); + const parsed = JSON.parse(output.trim()); + if (!parsed.additionalContext) throw new Error('Missing additionalContext'); + if (!parsed.additionalContext.includes('Skill Factory')) throw new Error('Context should mention Skill Factory'); + if (!parsed.additionalContext.includes('/skill-factory review')) throw new Error('Context should suggest /skill-factory review'); +}); + +// ── Edge cases ───────────────────────────────────────────────────────────── + +console.log('\nEdge cases:'); +cleanup(); + +test('handles empty stdin gracefully', () => { + try { + execFileSync('node', [OBSERVER, 'postToolUse'], { + input: '', + encoding: 'utf8', + env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, + timeout: 3000 + }); + // Should exit 0 + } catch (e) { + if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`); + } +}); + +test('handles malformed JSON gracefully', () => { + try { + execFileSync('node', [OBSERVER, 'postToolUse'], { + input: '{not valid json', + encoding: 'utf8', + env: { ...process.env, COPILOT_PLUGIN_DATA: DATA_DIR, HOME: DATA_DIR }, + timeout: 3000 + }); + } catch (e) { + // Should still exit 0 (hook failures are silent) + if (e.status !== 0) throw new Error(`Expected exit 0, got ${e.status}`); + } +}); + +test('sessionEnd with fewer than 3 tools creates no patterns', () => { + cleanup(); + const sid = 'short-session'; + + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: 'ls' } }); + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'read_file', toolArgs: { path: 'test.txt' } }); + runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); + + const dbPath = path.join(DATA_DIR, 'pattern-db.json'); + let patternCount = 0; + if (fs.existsSync(dbPath)) { + const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); + patternCount = Object.keys(db.patterns).length; + } + if (patternCount > 0) throw new Error('Sessions with <3 tools should not create patterns'); +}); + +test('session cleanup keeps only last 20 sessions', () => { + cleanup(); + + // Create 25 sessions + for (let i = 1; i <= 25; i++) { + const sid = `bulk-session-${i}`; + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'bash', toolArgs: { command: `cmd-${i}` } }); + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'grep', toolArgs: { pattern: `pat-${i}` } }); + runObserver('postToolUse', { sessionId: sid, timestamp: Date.now(), cwd: '/test', toolName: 'edit', toolArgs: { path: 'file.py', old_string: 'a', new_string: 'b' } }); + runObserver('sessionEnd', { sessionId: sid, timestamp: Date.now(), cwd: '/test', reason: 'complete' }); + } + + const dbPath = path.join(DATA_DIR, 'pattern-db.json'); + const db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); + const sessionCount = Object.keys(db.sessions).length; + + if (sessionCount > 20) throw new Error(`Session cleanup failed: ${sessionCount} > 20`); +}); + +// ── Summary ──────────────────────────────────────────────────────────────── + +console.log(`\n${'─'.repeat(50)}`); +console.log(`Results: ${passed} passed, ${failed} failed`); +console.log(`${'─'.repeat(50)}\n`); + +// Cleanup +cleanup(); + +process.exit(failed > 0 ? 1 : 0);