diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0f3ccb8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install flake8 + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics \ No newline at end of file diff --git a/.gitignore b/.gitignore index 83972fa..66a90cd 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,7 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# Custom for this project +hand_landmarker.task +screenshot_*.png diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..150994c --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,79 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic +status, nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political + attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action +in response to any behavior that they deem inappropriate, threatening, +offensive, or harmful. + +Community leaders have the right to remove, edit, or reject comments, +commits, code, wiki edits, issues, and other contributions that are not +aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies +when an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email +address, posting via an official social media account, or acting as an +appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[tbd@yourproject.com] or via the project's issue tracker. All complaints +will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of +the reporter of any incident. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d18c066 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +# Contributing to Gesture Camera Effects + +Thank you for your interest in contributing! Please follow these guidelines to help make the project better. + +## How to Contribute + +1. Fork the repository +2. Create a new branch for your feature or bugfix (`git checkout -b feature/amazing-feature`) +3. Make your changes +4. Ensure your code follows the project's style (we use [PEP 8](https://www.python.org/dev/peps/pep-0008/)) +5. Test your changes thoroughly +6. Commit your changes (`git commit -m 'Add some amazing feature'`) +7. Push to the branch (`git push origin feature/amazing-feature`) +8. Open a Pull Request + +## Reporting Issues + +Please use the GitHub Issues tracker to report bugs or request features. Include: + +- A clear and descriptive title +- Steps to reproduce the issue (if applicable) +- Expected vs actual behavior +- Screenshots or logs if helpful +- Your environment (OS, Python version, etc.) + +## Code of Conduct + +Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for details. + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..dfb102e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index c7488b2..9031c6b 100644 --- a/README.md +++ b/README.md @@ -1 +1,206 @@ -# gesture-camera-effects \ No newline at end of file +# Interactive Gesture Rectangle Camera with Multi Visual Effects + +## Description + +Membangun aplikasi Computer Vision berbasis Python + OpenCV + MediaPipe Hands yang memungkinkan pengguna membuat area efek kamera secara interaktif menggunakan gesture dua tangan. + +Pengguna membuat sebuah rectangle dengan melakukan gesture pinch pada kedua tangan. Rectangle mengikuti pergerakan tangan selama pinch masih dilakukan. Ketika pinch dilepas, rectangle akan terkunci (locked) dan langsung menerapkan efek visual unik hanya pada area rectangle. + +Aplikasi mendukung beberapa rectangle aktif secara bersamaan, sehingga setiap area dapat memiliki efek yang berbeda. + +## Alur Penggunaan + +1. Deteksi Tangan + +Program mendeteksi: + +- Tangan kiri +- Tangan kanan + +Menggunakan MediaPipe Hands. + +2. Gesture Membuat Rectangle + +Syarat: + +- Kedua tangan harus melakukan pinch (ujung ibu jari dan ujung telunjuk bersentuhan). + +- Ketika kondisi ini terpenuhi: + + - Rectangle dibuat. + - Sudut diagonal rectangle menggunakan: + - Ujung telunjuk tangan kiri + - Ujung telunjuk tangan kanan + + - Rectangle akan terus mengikuti pergerakan kedua telunjuk selama pinch dipertahankan. + +3. Lock Rectangle + +Saat salah satu atau kedua tangan melepaskan pinch: + + - Rectangle berhenti bergerak. + - Posisi rectangle disimpan secara permanen. + - Efek visual langsung diterapkan pada area rectangle. + + Rectangle yang sudah terkunci tidak dapat dipindahkan lagi. + +## Efek Kamera + +Efek hanya diterapkan di dalam area rectangle, sedangkan area di luar rectangle tetap menampilkan tampilan kamera normal. + +Empat efek yang digunakan: + +| Urutan | Efek | Deskripsi | +|--------|----------------|----------------------------------------------------------------------| +| 1 | 🌫️ Gaussian Blur | Area menjadi buram dengan transisi halus. | +| 2 | 🟦 Pixelation | Area berubah menjadi tampilan kotak-kotak (mosaic). | +| 3 | 🎨 Cartoon | Area berubah menjadi gaya kartun menggunakan kombinasi smoothing dan edge detection. | +| 4 | ⚡ Edge Detection | Area hanya menampilkan garis tepi objek sehingga terlihat futuristik. | + +Setiap rectangle baru akan mendapatkan efek secara berurutan: + +Rectangle 1 → Gaussian Blur +Rectangle 2 → Pixelation +Rectangle 3 → Cartoon +Rectangle 4 → Edge Detection +Rectangle 5 → Gaussian Blur +... + +## Multi Rectangle + +Pengguna dapat membuat beberapa rectangle. + +Contoh: + +Rectangle 1 → Gaussian Blur +Rectangle 2 → Pixelation +Rectangle 3 → Cartoon +Rectangle 4 → Edge Detection + +Seluruh rectangle tetap aktif secara bersamaan. + +## Gesture Hapus + +Gesture: + +- Closed Fist (tangan mengepal) + +Fungsi: + +- Menghapus rectangle terakhir yang dibuat (Undo). +- Rectangle lainnya tetap aktif beserta efeknya. + +## Arsitektur Program + +Webcam + │ + ▼ +OpenCV + │ + ▼ +MediaPipe Hands + │ + ▼ +Gesture Detector + │ + ▼ +Rectangle Manager + │ + ▼ +Effect Manager + │ + ▼ +Renderer + │ + ▼ +Camera Display + +## Struktur Projekt + +gesture-camera-effects/ +│ +├── main.py +├── camera.py +├── hand_tracker.py +├── gesture_detector.py +├── rectangle.py +├── rectangle_manager.py +├── effects.py +├── effect_manager.py +├── renderer.py +├── utils.py +│ +├── assets/ +│ +└── requirements.txt + +## State Machine + +Idle + │ + ▼ +Both Hands Pinch + │ + ▼ +Drawing Rectangle + │ + Release Pinch + │ + ▼ +Rectangle Locked + │ + ▼ +Effect Applied + │ + ▼ +Idle + +Gesture Closed Fist: + +Closed Fist + │ + ▼ +Delete Last Rectangle + │ + ▼ +Render Ulang + +## Fitur Tambahan + +- Border rectangle memiliki warna berbeda sesuai efek. +- Nama efek ditampilkan di sudut rectangle (Blur, Pixel, Cartoon, Edge). +- Maksimal 10 rectangle aktif untuk menjaga performa. +- Tombol keyboard: + - R → Hapus semua rectangle. + - S → Simpan screenshot. + - Q → Keluar aplikasi. +- Animasi singkat saat rectangle berhasil terkunci agar interaksi terasa lebih hidup. + +## Instalasi + +1. Clone repository ini +2. Install dependensi: + ``` + pip install -r requirements.txt + ``` +3. Jalankan aplikasi: + ``` + python main.py + ``` + +## Penggunaan + +- Pastikan webcam terhubung dan bekerja. +- Jadikan kedua tangan menghadap kamera. +- Lakukan pinch (ujung ibu jari dan jeli telunjuk menyentuh) dengan kedua tangan sekaligus untuk mulai membuat rectangle. +- Gerakkan tangan untuk mengubah ukuran rectangle. +- Lepaskan pinch untuk mengunci rectangle dan menerapkan efek. +- Untuk menghapus rectangle terakhir, buat kepalan tangan (closed fist) dengan salah satu tangan. +- Tekan 'R' untuk menghapus semua rectangle. +- Tekan 'S' untuk menyimpan screenshot. +- Tekan 'Q' untuk keluar. + +## Catatan + +- Aplikasi ini menggunakan MediaPipe Hands untuk deteksi tangan, sehingga kinerja mungkin tergantung pada hardware. +- Untuk hasil terbaik, pastikan pencahayaan adekuat dan background tidak terlalu ramai. diff --git a/camera.py b/camera.py new file mode 100644 index 0000000..84d6b6d --- /dev/null +++ b/camera.py @@ -0,0 +1,32 @@ +import cv2 + + +class Camera: + def __init__(self, width=640, height=480): + self.width = width + self.height = height + self.cap = None + self.is_opened = False + + def open(self): + self.cap = cv2.VideoCapture(0) + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.width) + self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height) + self.is_opened = self.cap.isOpened() + if not self.is_opened: + print("Error: Could not open webcam") + return self.is_opened + + def read(self): + if not self.is_opened: + return None + ret, frame = self.cap.read() + if not ret: + print("Warning: Empty frame received") + return None + return frame + + def release(self): + if self.cap is not None: + self.cap.release() + self.is_opened = False \ No newline at end of file diff --git a/effect_manager.py b/effect_manager.py new file mode 100644 index 0000000..a3e450d --- /dev/null +++ b/effect_manager.py @@ -0,0 +1,68 @@ +import cv2 +import numpy as np + + +class EffectManager: + def __init__(self): + # Effect types: 0=Grayscale, 1=Edge Detection, 2=Blur, 3=Color Inversion + pass + + def apply_effects(self, frame, locked_rectangles): + """ + Apply effects to the frame for all locked rectangles. + Later rectangles (higher index) will appear on top of earlier ones. + :param frame: the input frame (BGR) + :param locked_rectangles: list of Rectangle objects that are locked + :return: the frame with effects applied + """ + if not locked_rectangles: + return frame + + output = frame.copy() + height, width = frame.shape[:2] + + for rect in locked_rectangles: + if not rect.is_locked: + continue + + # Get the rectangle coordinates in pixels + x1, y1, x2, y2 = rect.get_points(width, height) + # Ensure coordinates are within the frame + x1 = max(0, x1) + y1 = max(0, y1) + x2 = min(width, x2) + y2 = min(height, y2) + + if x1 >= x2 or y1 >= y2: + continue # Invalid rectangle + + # Extract the region of interest + roi = output[y1:y2, x1:x2] + # Apply the effect + effected_roi = self._apply_effect(roi, rect.effect_type) + # Put the effected region back + output[y1:y2, x1:x2] = effected_roi + + return output + + def _apply_effect(self, roi, effect_type): + """ + Apply the specified effect to the region of interest. + :param roi: the region of interest (BGR image) + :param effect_type: integer 0-3 + :return: the effected region (BGR image) + """ + if effect_type == 0: # Grayscale + gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) + return cv2.cvtColor(gray, cv2.COLOR_GRAY2BGR) + elif effect_type == 1: # Edge Detection + gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) + edges = cv2.Canny(gray, 50, 150) + return cv2.cvtColor(edges, cv2.COLOR_GRAY2BGR) + elif effect_type == 2: # Blur + return cv2.GaussianBlur(roi, (15, 15), 0) + elif effect_type == 3: # Color Inversion + return cv2.bitwise_not(roi) + else: + # Default: return original + return roi \ No newline at end of file diff --git a/effects.py b/effects.py new file mode 100644 index 0000000..399994c --- /dev/null +++ b/effects.py @@ -0,0 +1,2 @@ +# This file is intentionally left empty. +# The effect functionality is implemented in effect_manager.py. \ No newline at end of file diff --git a/gesture_detector.py b/gesture_detector.py new file mode 100644 index 0000000..8767236 --- /dev/null +++ b/gesture_detector.py @@ -0,0 +1,150 @@ +import numpy as np + + +class GestureDetector: + def __init__(self, pinch_threshold=0.05, fist_threshold=0.07): + self.pinch_threshold = pinch_threshold # Normalized distance threshold for pinch + self.fist_threshold = fist_threshold # Normalized distance threshold for fist + self.pinch_state = { + 'left': False, + 'right': False + } + self.fist_state = { + 'left': False, + 'right': False + } + self.previous_pinch_state = { + 'left': False, + 'right': False + } + self.previous_fist_state = { + 'left': False, + 'right': False + } + + def detect(self, hand_landmarks): + """ + Detect gestures for both hands. + Returns a dictionary with gesture states for each hand and overall gestures. + """ + # Initialize default states + gestures = { + 'left': { + 'pinch': False, + 'fist': False, + 'index_tip': None, + 'thumb_tip': None + }, + 'right': { + 'pinch': False, + 'fist': False, + 'index_tip': None, + 'thumb_tip': None + } + } + + if not hand_landmarks or not hand_landmarks.multi_hand_landmarks: + self._update_previous_pinch_state(False, False) + self._update_previous_fist_state(False, False) + return gestures + + # Process each detected hand + for hand_idx, hand_landmark in enumerate(hand_landmarks.multi_hand_landmarks): + # Get handedness (left or right) + handedness_obj = hand_landmarks.multi_handedness[hand_idx][0] + handedness = handedness_obj.label.lower() + if handedness not in ['left', 'right']: + # Skip if not left or right (shouldn't happen) + continue + + # Get thumb tip (landmark 4) and index finger tip (landmark 8) + thumb_tip = hand_landmark.landmark[4] + index_tip = hand_landmark.landmark[8] + pinky_tip = hand_landmark.landmark[20] + + # Store tips for rectangle drawing + gestures[handedness]['index_tip'] = index_tip + gestures[handedness]['thumb_tip'] = thumb_tip + + # Calculate normalized distance between thumb and index finger (for pinch) + pinch_distance = np.sqrt( + (thumb_tip.x - index_tip.x) ** 2 + + (thumb_tip.y - index_tip.y) ** 2 + + (thumb_tip.z - index_tip.z) ** 2 + ) + + # Calculate normalized distance between thumb and pinky (for fist) + fist_distance = np.sqrt( + (thumb_tip.x - pinky_tip.x) ** 2 + + (thumb_tip.y - pinky_tip.y) ** 2 + + (thumb_tip.z - pinky_tip.z) ** 2 + ) + + # Determine pinch state + is_pinching = pinch_distance < self.pinch_threshold + gestures[handedness]['pinch'] = is_pinching + + # Determine fist state: both thumb-index and thumb-pinky close + is_fist = (pinch_distance < self.fist_threshold) and (fist_distance < self.fist_threshold) + gestures[handedness]['fist'] = is_fist + + # Update state trackers + self.pinch_state[handedness] = is_pinching + self.fist_state[handedness] = is_fist + + # Update pinch state for edge detection (release detection) + left_pinching = gestures['left']['pinch'] if hand_landmarks.multi_hand_landmarks else False + right_pinching = gestures['right']['pinch'] if hand_landmarks.multi_hand_landmarks else False + + # Update fist state for edge detection + left_fist = gestures['left']['fist'] if hand_landmarks.multi_hand_landmarks else False + right_fist = gestures['right']['fist'] if hand_landmarks.multi_hand_landmarks else False + + # Only consider pinch state if both hands are detected + both_hands_detected = len(hand_landmarks.multi_hand_landmarks) == 2 + + # Update previous states + self._update_previous_pinch_state(left_pinching, right_pinching) + self._update_previous_fist_state(left_fist, right_fist) + + # Add overall gesture states + gestures['both_hands_pinch'] = self.pinch_state['left'] and self.pinch_state['right'] + gestures['left_fist'] = self.fist_state['left'] + gestures['right_fist'] = self.fist_state['right'] + gestures['any_fist'] = self.fist_state['left'] or self.fist_state['right'] + + return gestures + + def _update_previous_pinch_state(self, left_pinching, right_pinching): + self.previous_pinch_state['left'] = left_pinching + self.previous_pinch_state['right'] = right_pinching + + def _update_previous_fist_state(self, left_fist, right_fist): + self.previous_fist_state['left'] = left_fist + self.previous_fist_state['right'] = right_fist + + def is_pinch_released(self, hand): + """Return True if pinch was just released for the given hand.""" + return (self.previous_pinch_state[hand] == True and + self.pinch_state[hand] == False) + + def is_pinch_started(self, hand): + """Return True if pinch was just started for the given hand.""" + return (self.previous_pinch_state[hand] == False and + self.pinch_state[hand] == True) + + def is_fist_made(self, hand): + """Return True if fist was just made for the given hand.""" + return (self.previous_fist_state[hand] == False and + self.fist_state[hand] == True) + + def is_fist_released(self, hand): + """Return True if fist was just released for the given hand.""" + return (self.previous_fist_state[hand] == True and + self.fist_state[hand] == False) + + def reset(self): + self.pinch_state = {'left': False, 'right': False} + self.fist_state = {'left': False, 'right': False} + self.previous_pinch_state = {'left': False, 'right': False} + self.previous_fist_state = {'left': False, 'right': False} \ No newline at end of file diff --git a/hand_tracker.py b/hand_tracker.py new file mode 100644 index 0000000..029e529 --- /dev/null +++ b/hand_tracker.py @@ -0,0 +1,103 @@ +import mediapipe as mp +from mediapipe.tasks import python +from mediapipe.tasks.python import vision +import time +import numpy as np +import cv2 + + +class HandTracker: + def __init__(self, max_num_hands=2, min_detection_confidence=0.7, min_tracking_confidence=0.5): + # Initialize the hand landmarker + base_options = python.BaseOptions(model_asset_path='hand_landmarker.task') + options = vision.HandLandmarkerOptions( + base_options=base_options, + running_mode=vision.RunningMode.VIDEO, + num_hands=max_num_hands, + min_hand_detection_confidence=min_detection_confidence, + min_hand_presence_confidence=min_tracking_confidence, + min_tracking_confidence=min_tracking_confidence + ) + self.detector = vision.HandLandmarker.create_from_options(options) + self.timestamp = 0 + + # We'll define helper classes to mimic the old Mediapipe solution's output + class NormalizedLandmark: + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + class NormalizedLandmarkList: + def __init__(self): + self.landmark = [] + + class Classification: + def __init__(self, label, score): + self.label = label + self.score = score + + self.NormalizedLandmark = NormalizedLandmark + self.NormalizedLandmarkList = NormalizedLandmarkList + self.Classification = Classification + + def process(self, frame): + # Convert the frame to RGB as required by MediaPipe + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_frame) + + # Update timestamp + self.timestamp = int(time.time() * 1000) + + # Detect hand landmarks + detection_result = self.detector.detect_for_video(mp_image, self.timestamp) + + # Convert detection_result to the format expected by the rest of the code + # We'll create an object that mimics the old mp.solutions.hands.Hands output + results = type('results', (), {})() + results.multi_hand_landmarks = [] + results.multi_handedness = [] + + if detection_result.hand_landmarks: + for hand_landmarks in detection_result.hand_landmarks: + # Create a NormalizedLandmarkList for this hand + landmark_list = self.NormalizedLandmarkList() + for landmark in hand_landmarks: + landmark_list.landmark.append( + self.NormalizedLandmark(landmark.x, landmark.y, landmark.z) + ) + results.multi_hand_landmarks.append(landmark_list) + + # Handle handedness + handedness_list = [] + if detection_result.handedness and len(detection_result.handedness) > len(results.multi_handedness): + handedness_info = detection_result.handedness[len(results.multi_handedness)] + if handedness_info: + # Take the first category (should be only one) + category = handedness_info[0] + handedness_obj = self.Classification(category.category_name, category.score) + handedness_list.append(handedness_obj) + results.multi_handedness.append(handedness_list) + + return results + + def draw_landmarks(self, frame, hand_landmarks): + if hand_landmarks: + for hand_lms in hand_landmarks.multi_hand_landmarks: + for idx, landmark in enumerate(hand_lms.landmark): + h, w, _ = frame.shape + cx, cy = int(landmark.x * w), int(landmark.y * h) + cv2.circle(frame, (cx, cy), 5, (0, 255, 0), -1) + for connection in mp.solutions.hands.HAND_CONNECTIONS: + start_idx, end_idx = connection + start_point = hand_landmarks.landmark[start_idx] + end_point = hand_landmarks.landmark[end_idx] + x1 = int(start_point.x * w) + y1 = int(start_point.y * h) + x2 = int(end_point.x * w) + y2 = int(end_point.y * h) + cv2.line(frame, (x1, y1), (x2, y2), (255, 0, 0), 2) + return frame + + def release(self): + self.detector.close() \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..d0b2e4f --- /dev/null +++ b/main.py @@ -0,0 +1,99 @@ +import sys +import time +import cv2 +from camera import Camera +from hand_tracker import HandTracker +from gesture_detector import GestureDetector +from rectangle_manager import RectangleManager +from effect_manager import EffectManager +from renderer import Renderer +from utils import FPS, draw_fps + + +def main(): + print("MAIN STARTED") + sys.stdout.flush() + # Initialize components + camera = Camera() + hand_tracker = HandTracker() + gesture_detector = GestureDetector() + rectangle_manager = RectangleManager() + effect_manager = EffectManager() + renderer = Renderer() + fps_counter = FPS() + + # Open camera + if not camera.open(): + print("Failed to open camera") + return + + print("Starting Interactive Gesture Rectangle Camera...") + sys.stdout.flush() + print("Controls:") + print(" - Pinch with both hands to create/resize rectangle") + print(" - Release pinch to lock rectangle and apply effect") + print(" - Closed fist to delete last rectangle (undo)") + print(" - 'R' to clear all rectangles") + print(" - 'S' to save screenshot") + print(" - 'Q' to quit") + sys.stdout.flush() + + while True: + # Capture frame + frame = camera.read() + if frame is None: + print("Failed to capture frame") + break + + # Flip frame horizontally for natural interaction + frame = cv2.flip(frame, 1) + + # Process hand tracking + hand_landmarks = hand_tracker.process(frame) + + # Detect gestures + gestures = gesture_detector.detect(hand_landmarks) + + # Update rectangle manager based on gestures + rectangle_manager.update(gestures) + + # Get locked rectangles + locked_rectangles = rectangle_manager.get_locked_rectangles() + + # Apply effects to the frame (based on locked rectangles) + effected_frame = effect_manager.apply_effects(frame, locked_rectangles) + + # Render the frame (with effects applied) and draw UI + output_frame = renderer.render( + effected_frame, + hand_landmarks, + rectangle_manager.get_temp_rectangle(), + locked_rectangles + ) + + # Draw FPS + fps = fps_counter.update() + draw_fps(output_frame, fps) + + # Display the frame + cv2.imshow('Gesture Camera Effects', output_frame) + + # Handle key presses + key = cv2.waitKey(1) & 0xFF + if key == ord('q') or key == ord('Q'): + break + elif key == ord('r') or key == ord('R'): + rectangle_manager.clear_all() + elif key == ord('s') or key == ord('S'): + screenshot_path = f"screenshot_{fps_counter.get_timestamp()}.png" + cv2.imwrite(screenshot_path, output_frame) + print(f"Screenshot saved: {screenshot_path}") + + # Cleanup + camera.release() + hand_tracker.release() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/rectangle.py b/rectangle.py new file mode 100644 index 0000000..0e5794a --- /dev/null +++ b/rectangle.py @@ -0,0 +1,112 @@ +import numpy as np +import cv2 + + +class Rectangle: + def __init__(self, start_point, end_point, effect_type, effect_name, color): + """ + Initialize a rectangle. + :param start_point: (x, y) in normalized coordinates (0-1) + :param end_point: (x, y) in normalized coordinates (0-1) + :param effect_type: integer 0-3 + :param effect_name: string name of the effect + :param color: BGR color tuple for the rectangle border + """ + self.start_point = start_point # Normalized (x, y) + self.end_point = end_point # Normalized (x, y) + self.effect_type = effect_type + self.effect_name = effect_name + self.color = color + self.is_locked = False + self.is_temp = False # True while being drawn + + def set_temp(self, is_temp): + """Set whether this rectangle is temporary (being drawn).""" + self.is_temp = is_temp + + def lock(self): + """Lock the rectangle (stop drawing).""" + self.is_locked = True + self.is_temp = False + + def get_points(self, width, height): + """ + Get the pixel coordinates of the rectangle. + Returns (x1, y1, x2, y2) where (x1, y1) is top-left and (x2, y2) is bottom-right. + """ + x1 = int(self.start_point[0] * width) + y1 = int(self.start_point[1] * height) + x2 = int(self.end_point[0] * width) + y2 = int(self.end_point[1] * height) + + # Ensure top-left and bottom-right + x1, x2 = min(x1, x2), max(x1, x2) + y1, y2 = min(y1, y2), max(y1, y2) + + return x1, y1, x2, y2 + + def contains_point(self, point, width, height): + """ + Check if a point (in normalized coordinates) is inside the rectangle. + """ + x, y = point + x1, y1, x2, y2 = self.get_points(width, height) + px = int(x * width) + py = int(y * height) + return x1 <= px <= x2 and y1 <= py <= y2 + + def update_end_point(self, end_point): + """Update the end point while drawing.""" + self.end_point = end_point + + def draw_temp(self, frame, width, height): + """Draw the temporary rectangle (while drawing).""" + if not self.is_temp: + return + + x1, y1, x2, y2 = self.get_points(width, height) + # Draw a dashed line effect for the temporary rectangle + self._draw_dashed_rectangle(frame, (x1, y1), (x2, y2), self.color, 2) + # Draw effect name + label = self.effect_name + label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] + # Background for text + cv2.rectangle(frame, (x1, y1 - label_size[1] - 10), + (x1 + label_size[0], y1), self.color, -1) + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) + + def draw_locked(self, frame, width, height): + """Draw the locked rectangle.""" + if not self.is_locked: + return + + x1, y1, x2, y2 = self.get_points(width, height) + # Draw solid rectangle + cv2.rectangle(frame, (x1, y1), (x2, y2), self.color, 2) + # Draw effect name + label = self.effect_name + label_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] + # Background for text + cv2.rectangle(frame, (x1, y1 - label_size[1] - 10), + (x1 + label_size[0], y1), self.color, -1) + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) + + def _draw_dashed_rectangle(self, img, pt1, pt2, color, thickness, dash_length=10): + """Draw a dashed rectangle.""" + x1, y1 = pt1 + x2, y2 = pt2 + + # Top edge + for x in range(x1, x2, dash_length * 2): + cv2.line(img, (x, y1), (min(x + dash_length, x2), y1), color, thickness) + # Bottom edge + for x in range(x1, x2, dash_length * 2): + cv2.line(img, (x, y2), (min(x + dash_length, x2), y2), color, thickness) + # Left edge + for y in range(y1, y2, dash_length * 2): + cv2.line(img, (x1, y), (x1, min(y + dash_length, y2)), color, thickness) + # Right edge + for y in range(y1, y2, dash_length * 2): + cv2.line(img, (x2, y), (x2, min(y + dash_length, y2)), color, thickness) \ No newline at end of file diff --git a/rectangle_manager.py b/rectangle_manager.py new file mode 100644 index 0000000..50b7e65 --- /dev/null +++ b/rectangle_manager.py @@ -0,0 +1,178 @@ +import time +from rectangle import Rectangle +from effect_manager import EffectManager + + +class RectangleManager: + def __init__(self, max_rectangles=10): + self.max_rectangles = max_rectangles + self.rectangles = [] # List of Rectangle objects + self.effect_manager = EffectManager() + self.drawing = False + self.start_point = None + self.current_point = None + self.lock_time = None + self.animation_duration = 0.5 # seconds for lock animation + self.effect_sequence = [ + ("Grayscale", (0, 255, 0)), # Green + ("Edge Detection", (0, 0, 255)), # Red + ("Blur", (255, 0, 0)), # Blue + ("Color Inversion", (0, 255, 255)) # Yellow + ] + # For fist detection edge triggering + self.previous_fist_state = {'left': False, 'right': False} + + def update(self, gestures): + """ + Update rectangle state based on gestures. + :param gestures: dict from GestureDetector.detect() + """ + # Get fist states + left_fist = gestures['left']['fist'] + right_fist = gestures['right']['fist'] + + # Detect fist made (edge trigger) + left_fist_made = (not self.previous_fist_state['left']) and left_fist + right_fist_made = (not self.previous_fist_state['right']) and right_fist + any_fist_made = left_fist_made or right_fist_made + + # Update previous fist state for next frame + self.previous_fist_state['left'] = left_fist + self.previous_fist_state['right'] = right_fist + + # Get pinch states + both_hands_pinch = gestures.get('both_hands_pinch', False) + + # State machine logic + if not self.drawing and both_hands_pinch: + # Start drawing a new rectangle + self._start_drawing(gestures) + elif self.drawing and both_hands_pinch: + # Update the rectangle while drawing + self._update_drawing(gestures) + elif self.drawing and not both_hands_pinch: + # Pinch released - lock the rectangle + self._lock_rectangle() + elif any_fist_made and not self.drawing: + # Closed fist made - delete last rectangle (undo) + self._undo_last_rectangle() + + # Update lock animation if needed + self._update_lock_animation() + + def _start_drawing(self, gestures): + """Start drawing a new rectangle.""" + if len(self.rectangles) >= self.max_rectangles: + print(f"Maximum number of rectangles ({self.max_rectangles}) reached.") + return + + # Get index finger tips for both hands + left_index_tip = gestures['left']['index_tip'] + right_index_tip = gestures['right']['index_tip'] + + if left_index_tip is None or right_index_tip is None: + print("Could not detect both index fingers") + return + + # Convert to normalized points + start_norm = (left_index_tip.x, left_index_tip.y) + end_norm = (right_index_tip.x, right_index_tip.y) + + # Determine effect type based on sequence + effect_index = len(self.rectangles) % len(self.effect_sequence) + effect_name, color = self.effect_sequence[effect_index] + + # Create a temporary rectangle + rect = Rectangle(start_norm, end_norm, effect_index, effect_name, color) + rect.set_temp(True) + self.rectangles.append(rect) + self.drawing = True + self.start_point = start_norm + self.current_point = end_norm + + def _update_drawing(self, gestures): + """Update the rectangle being drawn.""" + if not self.rectangles: + return + + # Get the last rectangle (the one being drawn) + rect = self.rectangles[-1] + if not rect.is_temp: + return + + # Get index finger tips for both hands + left_index_tip = gestures['left']['index_tip'] + right_index_tip = gestures['right']['index_tip'] + + if left_index_tip is None or right_index_tip is None: + return + + # Update the rectangle points + # According to spec: rectangle diagonal uses left index and right index + start_norm = (left_index_tip.x, left_index_tip.y) + end_norm = (right_index_tip.x, right_index_tip.y) + + rect.start_point = start_norm + rect.end_point = end_norm + self.start_point = start_norm + self.current_point = end_norm + + def _lock_rectangle(self): + """Lock the current rectangle and stop drawing.""" + if not self.rectangles or not self.drawing: + return + + rect = self.rectangles[-1] + if rect.is_temp: + rect.lock() + rect.set_temp(False) + self.drawing = False + self.lock_time = time.time() + print(f"Rectangle locked with effect: {rect.effect_name}") + + def _undo_last_rectangle(self): + """Delete the last rectangle (undo).""" + if not self.rectangles: + print("No rectangles to delete.") + return + + # Remove the last rectangle + removed = self.rectangles.pop() + print(f"Undo: Removed rectangle with effect {removed.effect_name}") + + def _update_lock_animation(self): + """Update lock animation (e.g., brief flash).""" + if self.lock_time is None: + return + + elapsed = time.time() - self.lock_time + if elapsed > self.animation_duration: + self.lock_time = None + + def get_rectangles(self): + """Get the list of rectangles.""" + return self.rectangles + + def get_locked_rectangles(self): + """Get only the locked rectangles.""" + return [rect for rect in self.rectangles if rect.is_locked] + + def get_temp_rectangle(self): + """Get the rectangle currently being drawn (if any).""" + for rect in reversed(self.rectangles): + if rect.is_temp: + return rect + return None + + def clear_all(self): + """Remove all rectangles.""" + self.rectangles.clear() + self.drawing = False + self.start_point = None + self.current_point = None + self.lock_time = None + print("All rectangles cleared.") + + def get_effect_manager(self): + """Get the effect manager.""" + return self.effect_manager \ No newline at end of file diff --git a/renderer.py b/renderer.py new file mode 100644 index 0000000..679bd2c --- /dev/null +++ b/renderer.py @@ -0,0 +1,61 @@ +import cv2 +import numpy as np + + +class Renderer: + def __init__(self): + self.font = cv2.FONT_HERSHEY_SIMPLEX + self.font_scale = 0.5 + self.font_thickness = 1 + self.text_color = (255, 255, 255) # White + self.bg_color = (0, 0, 0) # Black for text background + + def render(self, frame, hand_landmarks, temp_rectangle, locked_rectangles): + """ + Render the frame with all overlays. + :param frame: the input frame (BGR) + :param hand_landmarks: MediaPipe hands results (for drawing landmarks) + :param temp_rectangle: the Rectangle object being drawn (or None) + :param locked_rectangles: list of locked Rectangle objects + :return: the rendered frame (BGR) + """ + # Start with a copy of the frame + output = frame.copy() + + # Draw the temporary rectangle (being drawn) + if temp_rectangle and not temp_rectangle.is_locked: + temp_rectangle.draw_temp(output, output.shape[1], output.shape[0]) + + # Draw locked rectangles + for rect in locked_rectangles: + if rect.is_locked: + rect.draw_locked(output, output.shape[1], output.shape[0]) + + # Draw instructions + self._draw_instructions(output) + + return output + + def _draw_instructions(self, frame): + """Draw instructions on the frame.""" + h, w = frame.shape[:2] + # Semi-transparent background for text + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 80), (0, 0, 0), -1) + cv2.addWeighted(overlay, 0.6, frame, 0.4, 0, frame) + + lines = [ + "Controls:", + "Pinch both hands: Create/resize rectangle", + "Release pinch: Lock rectangle and apply effect", + "Closed fist: Undo last rectangle", + "R: Clear all rectangles", + "S: Save screenshot", + "Q: Quit" + ] + + y_offset = 20 + for line in lines: + cv2.putText(frame, line, (10, y_offset), self.font, self.font_scale, + self.text_color, self.font_thickness, cv2.LINE_AA) + y_offset += 20 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d6983fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +opencv-python>=4.5.0 +mediapipe>=0.8.0 +numpy>=1.19.0 \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..6994437 --- /dev/null +++ b/utils.py @@ -0,0 +1,32 @@ +import time +import cv2 + + +class FPS: + def __init__(self): + self._start = None + self._num_frames = 0 + self._fps = 0.0 + + def start(self): + self._start = time.time() + self._num_frames = 0 + + def update(self): + self._num_frames += 1 + if self._num_frames == 1: + self._start = time.time() + else: + elapsed = time.time() - self._start + if elapsed > 0: + self._fps = self._num_frames / elapsed + return self._fps + + def get_timestamp(self): + return int(time.time() * 1000) + + +def draw_fps(frame, fps): + """Draw FPS on the frame.""" + fps_text = f"FPS: {fps:.1f}" + cv2.putText(frame, fps_text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) \ No newline at end of file