diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..e908336 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug Report +about: Create a report to help us improve +title: '[BUG] ' +labels: bug +assignees: '' + +--- + +**Describe the Bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected Behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment (please complete the following information):** +- OS: [e.g. Ubuntu 22.04] +- Python Version: [e.g. 3.11] +- Docker Version: [e.g. 24.0] + +**Additional Context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..5d98db2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature Request +about: Suggest an idea for this project +title: '[FEATURE] ' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the Solution You'd Like** +A clear and concise description of what you want to happen. + +**Describe Alternatives You've Considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional Context** +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6ec7a44 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,23 @@ +## Description + +Please include a summary of the change and which issue is fixed. + +Fixes # (issue) + +## Type of Change + +- [ ] Bug fix (non-breaking change) +- [ ] New feature (non-breaking change) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Security fix + +## Checklist + +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5ace460 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..788f0d5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,21 @@ +name: Build + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + +jobs: + build: + name: Build and Test + runs-on: macos-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build + run: swift build + + - name: Test + run: swift test diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..f8a634c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,33 @@ +name: CodeQL +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + schedule: + - cron: '0 0 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: macos-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..256974a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,48 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the project team at j1admin@onebyjorah.com. All complaints will +be reviewed and investigated and will result in a response that is deemed +necessary and appropriate to the circumstances. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +[homepage]: https://www.contributor-covenant.org diff --git a/README.md b/README.md index ab09f02..a4826da 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ Ghost OS MCP Server (Swift) └── AXorcist ────── macOS accessibility engine ``` -~7,000 lines of Swift + Python vision sidecar. Built on [AXorcist](https://github.com/steipete/AXorcist) by [@steipete](https://github.com/steipete). +~8,950 lines of Swift + Python vision sidecar. Built on [AXorcist](https://github.com/steipete/AXorcist) by [@steipete](https://github.com/steipete). ## Contributing @@ -291,3 +291,9 @@ Thanks to everyone who has contributed to Ghost OS. ## License MIT + +--- + +[![Built by](https://img.shields.io/badge/built%20by-JorahOne%20LLC-FFB300?style=for-the-badge&labelColor=0d0d0c)](https://github.com/OneByJorah) + +Fork maintained by JorahOne, LLC as part of the JorahOne infrastructure ecosystem. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..be4cad0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +## Supported Versions + +We release patches for security vulnerabilities. Which versions are eligible +for receiving patches depends on the CVSS v3.0 rating: + +| Version | Supported | +| ------- | ------------------ | +| Latest | ✅ | +| < Latest| ❌ | + +## Reporting a Vulnerability + +Please report security vulnerabilities to **j1admin@onebyjorah.com**. Do NOT +report security vulnerabilities through public GitHub issues. + +You should receive a response within 48 hours. If for some reason you do not, +please follow up via email to ensure we received your original message. + +Please include the following information: + +- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) +- Full paths of source file(s) related to the manifestation of the issue +- The location of the affected source code (tag/branch/commit or direct URL) +- Any special configuration required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof-of-concept or exploit code (if possible) +- Impact of the issue, including how an attacker might exploit it + +We prefer to receive reports via email. We will acknowledge receipt within +48 hours and send a more detailed response within 72 hours. + +This project follows a 90-day disclosure timeline. diff --git a/docs/progress.md b/docs/progress.md index 1522dcd..d1259d4 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -1,33 +1,34 @@ # Ghost OS v2 Progress -## Current State: Phases 0-5 COMPLETE. Phase 6-7 next. +## Current State: Phases 0-7 COMPLETE. All 29 tools functional. ## What Works (tested end-to-end) -- **20/20 MCP tools** functional +- **29/29 MCP tools** functional (7 Perception, 1 Annotate, 10 Actions, 1 Wait, 5 Recipes, 2 Vision, 3 Learning) - **gmail-send recipe**: 5/5 success via ghost_run - **arxiv-download recipe**: 3/3 success (agent-created, 8 steps) - **Screenshots**: stable on multi-monitor, inline display in Claude Code - **Cmd+L navigation**: works consistently (no flicker) - **Scroll**: works on second monitor via element-based scroll - **All perception tools**: context, state, find, read, inspect, element_at -- **All action tools**: click, type (into field + at focus), press, hotkey, scroll +- **All action tools**: click, type (into field + at focus), press, hotkey, scroll, hover, long_press, drag - **All wait conditions**: urlContains, elementExists, elementGone, titleContains - **Recipe CRUD**: list, show, save (with decode error details), delete +- **Annotate**: ghost_annotate with Set-of-Marks labels +- **Vision**: ghost_ground with ShowUI-2B VLM grounding +- **Learning**: ghost_learn_start/stop/status with CGEvent tap recording ## Key Numbers -- 15 Swift files, ~3,800 lines (vs v1's 6,200) -- ~20 commits on main +- 26 Swift files, ~8,950 lines +- ~30+ commits on main - GitHub: ghostwright/ghost-os ## What's Next -### Phase 6: Setup & Polish -- `ghost setup` wizard: permissions, MCP config, recipe install, self-test -- Polish GHOST-MCP.md with learnings (use "To recipients" not "To", etc.) -- Bundled recipes: install from repo recipes/ dir on first run - -### Phase 7: Testing & Hardening -- Test against 10 apps (Slack, Finder, System Settings, Messages, etc.) +### Phase 8: Testing & Hardening +- Expand unit test coverage beyond LocatorBuilder +- Add CI test workflow to verify builds on push/PR +- Add integration tests for MCP server +- Add Python tests for vision sidecar - Recipe reliability tests (run each 5x, track success rate) - Stress test (MCP server up 24 hours) diff --git a/vision-sidecar/server.py b/vision-sidecar/server.py index c0e67a9..534a3bd 100644 --- a/vision-sidecar/server.py +++ b/vision-sidecar/server.py @@ -40,7 +40,7 @@ import tempfile import time import traceback -from http.server import HTTPServer, BaseHTTPRequestHandler +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from threading import Lock, Timer @@ -142,7 +142,7 @@ def _load_vlm(): # requires PyTorch tensors which MLX doesn't provide. from transformers import Qwen2VLImageProcessor _vlm_processor.image_processor = Qwen2VLImageProcessor.from_pretrained( - MODEL_PATH, use_fast=False + MODEL_PATH, use_fast=False, ) from transformers import AutoTokenizer @@ -205,10 +205,10 @@ def _vlm_ground(image_path: str, description: str, screen_w: float, screen_h: fl # Build chat template chat = [{"role": "user", "content": [ {"type": "image", "image": resized_path}, - {"type": "text", "text": prompt} + {"type": "text", "text": prompt}, ]}] formatted = _vlm_tokenizer.apply_chat_template( - chat, tokenize=False, add_generation_prompt=True + chat, tokenize=False, add_generation_prompt=True, ) # Run inference @@ -218,9 +218,9 @@ def _vlm_ground(image_path: str, description: str, screen_w: float, screen_h: fl _vlm_model, _vlm_processor, formatted, image=resized_path, max_tokens=128, - temp=0.0 + temp=0.0, ): - full_text += result.text if hasattr(result, 'text') else str(result) + full_text += result.text if hasattr(result, "text") else str(result) elapsed = time.time() - t0 log(f"VLM '{description}' -> '{full_text.strip()}' ({elapsed:.1f}s)") @@ -232,7 +232,7 @@ def _vlm_ground(image_path: str, description: str, screen_w: float, screen_h: fl pass # Parse [x, y] or (x, y) coordinates from model output - match = re.search(r'[\(\[]\s*([\d.]+)\s*,\s*([\d.]+)\s*[\)\]]', full_text) + match = re.search(r"[\(\[]\s*([\d.]+)\s*,\s*([\d.]+)\s*[\)\]]", full_text) if match: nx, ny = float(match.group(1)), float(match.group(2)) if nx <= 1.0 and ny <= 1.0: