Blind navigation MVP

.gitattributes

@@ -0,0 +1,2 @@
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.task filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,20 @@
+__pycache__/
+*.pyc
+venv/
+.gradio/
+test_images/
+results/
+video_test/
+new_images/
+*.mp4
+*.pptx
+*.pdf
+cert.pem
+key.pem
+# Old POC files
+/blind_nav.py
+/nav_dashboard.py
+/video_pipeline.py
+/full_pipeline.py
+/app.py
+/pose_slope_test.py

README.md

@@ -0,0 +1,61 @@
+# Blind Navigation Assistant
+
+Vision-based navigation system for visually impaired users. Uses camera input to detect obstacles, estimate terrain slope, and provide real-time voice guidance.
+
+## Models Used
+- **YOLOv8n** — obstacle detection
+- **Depth Anything** — monocular depth → slope estimation
+- **BlazePose** — gait analysis
+
+## Quick Start
+
+```bash
+# Clone
+git clone https://github.com/<your-username>/blind-nav.git
+cd blind-nav
+
+# Install
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+sudo apt-get install -y ffmpeg espeak
+
+# Run
+NAV_DEPTH_INPUT_SIZE=256 python app_live.py
+```
+
+Open: `http://<your-ip>:7860`
+
+## EC2 Deployment
+
+```bash
+git clone https://github.com/<your-username>/blind-nav.git
+cd blind-nav
+bash deploy_ec2.sh
+cd ~/blind_nav
+source venv/bin/activate
+NAV_DEPTH_INPUT_SIZE=256 python app_live.py
+```
+
+Ensure port 7860 is open in your EC2 security group.
+
+## Structure
+
+```
+config.py          — settings, model paths, thresholds
+core/
+  detector.py      — YOLO + ByteTrack
+  depth.py         — Depth Anything + slope smoothing
+  pose.py          — BlazePose gait
+  risk_engine.py   — contextual risk fusion
+  guidance.py      — voice guidance + throttling
+  tts.py           — live TTS (edge-tts/espeak)
+  tts_render.py    — offline TTS baking into video
+  camera.py        — threaded camera stream
+renderers/
+  overlay.py       — HUD overlay
+  blind_nav.py     — 2-panel view
+pipeline.py        — unified video/image/webcam pipeline
+app_live.py        — Gradio live streaming UI
+app_mobile.py      — OpenCV mobile camera UI
+```

app_live.py

@@ -0,0 +1,177 @@
+"""Live Phone Camera → Real-time Navigation Assistant.
+
+Phone streams camera on left, annotated output appears on right.
+Uses HTTPS so browser allows camera access.
+"""
+import gradio as gr
+import cv2
+import numpy as np
+from PIL import Image as PILImage
+import os, sys, math, time
+
+sys.path.insert(0, os.path.dirname(__file__))
+from core import detector, depth, pose, risk_engine
+from core.depth import SlopeSmoother
+from core.detector import ObstacleTracker
+from core.guidance import GuidanceEngine
+
+# Persistent state across frames
+_st = {}
+
+def _init():
+    if 'ready' not in _st:
+        _st['sm'] = SlopeSmoother()
+        _st['tr'] = ObstacleTracker()
+        _st['gu'] = GuidanceEngine()
+        _st['dn'] = None
+        _st['n'] = 0
+        _st['ready'] = True
+
+def _reset():
+    _st.clear()
+    return None, "Reset. Point your camera and start."
+
+
+def process_frame(frame):
+    if frame is None:
+        return None, "Waiting for camera..."
+
+    _init()
+    img = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)
+    h, w = img.shape[:2]
+    rgb = np.array(frame)
+
+    obs = detector.detect(img, track=True)
+    new_o, close_o, _ = _st['tr'].update(obs)
+    gait, lms, foot_y = pose.analyze(rgb, w, h)
+
+    _st['n'] += 1
+    if _st['n'] % 5 == 0 or _st['dn'] is None:
+        _st['dn'] = depth.estimate_depth(rgb, h, w)
+        rs, rd, rt, _ = depth.estimate_slope(_st['dn'], h, w, foot_y)
+        _st['sm'].update(rs, rd, rt)
+
+    sm = _st['sm']
+    sa, sd = sm.angle, sm.direction
+
+    risk_d = risk_engine.assess(sa, sd, gait, obs,
+        slope_trend=sm.trend, new_obstacles=new_o, closing_obstacles=close_o)
+    guid = _st['gu'].compute(sa, sd, sm.terrain, obs,
+        slope_trend=sm.trend, new_obstacles=new_o, closing_obstacles=close_o)
+
+    out = img.copy()
+
+    # ── Obstacles ──
+    for ob in obs:
+        x1,y1,x2,y2 = ob['box']
+        u = ob['proximity']
+        color = (0,0,255) if u>0.7 else (0,180,255) if u>0.4 else (0,200,0)
+        ov = out.copy()
+        cv2.rectangle(ov,(x1,y1),(x2,y2),color,-1)
+        cv2.addWeighted(ov, 0.35 if u>0.7 else 0.2, out, 0.65 if u>0.7 else 0.8, 0, out)
+        cv2.rectangle(out,(x1,y1),(x2,y2),color,3)
+        lbl = f'{ob["label"].upper()} {ob["dist"]}'
+        (tw,th),_ = cv2.getTextSize(lbl, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)
+        cv2.rectangle(out,(x1,y1-th-10),(x1+tw+6,y1),color,-1)
+        cv2.putText(out,lbl,(x1+3,y1-5),cv2.FONT_HERSHEY_SIMPLEX,0.6,(255,255,255),2)
+        cx,cy = ob['center']
+        if ob['direction']=='LEFT':
+            cv2.arrowedLine(out,(cx+50,cy),(cx-50,cy),(255,255,255),3,tipLength=0.4)
+        elif ob['direction']=='RIGHT':
+            cv2.arrowedLine(out,(cx-50,cy),(cx+50,cy),(255,255,255),3,tipLength=0.4)
+        else:
+            cv2.arrowedLine(out,(cx,cy-40),(cx,cy+40),(0,0,255),3,tipLength=0.4)
+        if u > 0.7:
+            cv2.putText(out,'!! CLOSE !!',(cx-45,y2+18),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),2)
+
+    # ── Path ──
+    blocked = [o for o in obs if o['box'][3] > h//2]
+    if not blocked:
+        pts = np.array([[w//4+30,h//2],[3*w//4-30,h//2],[3*w//4+20,h],[w//4-20,h]])
+        ov2 = out.copy()
+        cv2.fillPoly(ov2,[pts],(0,120,0))
+        cv2.addWeighted(ov2,0.15,out,0.85,0,out)
+        cv2.polylines(out,[pts],True,(0,255,0),2)
+        cv2.putText(out,'CLEAR',(w//2-30,h//2+30),cv2.FONT_HERSHEY_SIMPLEX,0.6,(0,255,0),2)
+    else:
+        lb = any(o['box'][0] < w//2 for o in blocked)
+        rb = any(o['box'][2] > w//2 for o in blocked)
+        if not lb:
+            cv2.arrowedLine(out,(w//2,h*3//4),(w//6,h*3//4),(0,255,0),4,tipLength=0.3)
+            cv2.putText(out,'GO LEFT',(20,h//2+30),cv2.FONT_HERSHEY_SIMPLEX,0.8,(0,255,0),2)
+        elif not rb:
+            cv2.arrowedLine(out,(w//2,h*3//4),(5*w//6,h*3//4),(0,255,0),4,tipLength=0.3)
+            cv2.putText(out,'GO RIGHT',(2*w//3-20,h//2+30),cv2.FONT_HERSHEY_SIMPLEX,0.8,(0,255,0),2)
+        else:
+            cv2.putText(out,'STOP',(w//2-40,h//2),cv2.FONT_HERSHEY_SIMPLEX,1.2,(0,0,255),3)
+
+    # ── Slope arrow ──
+    if abs(sa) > 3:
+        cx2,cy2 = w//2, h-50
+        ax = int(cx2 + math.cos(math.radians(sa))*40)
+        ay = int(cy2 - math.sin(math.radians(sa))*40)
+        cv2.arrowedLine(out,(cx2,cy2),(ax,ay),(0,255,255),3,tipLength=0.3)
+
+    # ── HUD ──
+    r = risk_d['risk']
+    rc = {'SAFE':(0,180,0),'LOW':(0,220,0),'MEDIUM':(0,180,220),'HIGH':(0,0,220)}[r]
+    cv2.rectangle(out,(0,0),(w,55),(0,0,0),-1)
+    cv2.rectangle(out,(5,5),(170,32),rc,-1)
+    cv2.putText(out,f'RISK: {r} ({risk_d["score"]})',(10,26),cv2.FONT_HERSHEY_SIMPLEX,0.5,(255,255,255),2)
+    cv2.putText(out,f'{sd} {sa:.0f}deg {sm.terrain}',(180,26),cv2.FONT_HERSHEY_SIMPLEX,0.38,(200,200,200),1)
+    cv2.putText(out,f'Step:{guid["step"]} Lean:{guid["lean"]} Knee:{guid["knee_rec"]}',
+                (5,48),cv2.FONT_HERSHEY_SIMPLEX,0.33,(0,255,255),1)
+
+    # ── Voice bar ──
+    cv2.rectangle(out,(0,h-30),(w,h),(20,20,40),-1)
+    cv2.putText(out,guid['voice'][:50],(8,h-10),cv2.FONT_HERSHEY_SIMPLEX,0.38,(255,255,255),1)
+
+    # Status text
+    status = f"🔊 **{guid['voice']}**\n\n"
+    status += f"Risk: **{r}** | Slope: {sd} {sa:.0f}° | Obstacles: {len(obs)} | Step: {guid['step']} | Lean: {guid['lean']}"
+
+    return PILImage.fromarray(cv2.cvtColor(out, cv2.COLOR_BGR2RGB)), status
+
+
+with gr.Blocks(title="Live Navigation", head="""
+<script>
+// Force rear camera on mobile
+document.addEventListener('DOMContentLoaded', () => {
+    const origGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+    navigator.mediaDevices.getUserMedia = (constraints) => {
+        if (constraints && constraints.video) {
+            constraints.video = { facingMode: { exact: 'environment' } };
+        }
+        return origGetUserMedia(constraints);
+    };
+});
+</script>
+""") as demo:
+    gr.Markdown("# 🦯 Live Navigation Assistant\nPoint your phone camera → see real-time obstacle & path guidance")
+
+    with gr.Row():
+        cam_in = gr.Image(sources=["webcam"], streaming=True, type="pil",
+                          label="📹 Live Camera")
+        cam_out = gr.Image(label="🎯 Navigation View")
+
+    status = gr.Markdown("Point camera and start streaming...")
+
+    cam_in.stream(fn=process_frame, inputs=cam_in, outputs=[cam_out, status])
+
+    gr.Button("🔄 Reset").click(fn=_reset, outputs=[cam_out, status])
+
+if __name__ == "__main__":
+    # Preload models
+    print("Preloading models...", flush=True)
+    dummy = np.zeros((100, 100, 3), dtype=np.uint8)
+    detector.detect(dummy)
+    depth.estimate_depth(dummy, 100, 100)
+    pose.analyze(dummy, 100, 100)
+    print("Models ready!", flush=True)
+
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=True,
+        show_error=True,
+    )

app_mobile.py

@@ -0,0 +1,284 @@
+"""Live Mobile Camera Navigation Assistant.
+
+Connects to phone camera via IP Webcam app and provides
+real-time obstacle detection, path analysis, slope estimation,
+and voice guidance.
+
+Usage:
+  1. Install "IP Webcam" app on Android (or similar on iPhone)
+  2. Start the app, note the URL (e.g., http://192.168.1.5:8080)
+  3. Run: python app_mobile.py http://192.168.1.5:8080
+     Or:  python app_mobile.py              (uses laptop webcam)
+"""
+import cv2
+import numpy as np
+import sys
+import os
+import time
+import math
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+from config import DEFAULT_DEPTH_EVERY
+from core import detector, depth, pose, risk_engine
+from core.depth import SlopeSmoother
+from core.detector import ObstacleTracker
+from core.guidance import GuidanceEngine
+from core.tts import TTSEngine
+from core.camera import CameraStream
+
+
+def draw_path_zone(frame, obstacles, slope_dir, slope_angle):
+    """Draw safe walking path overlay on the frame."""
+    h, w = frame.shape[:2]
+    overlay = frame.copy()
+
+    # Define path corridor (center third, bottom half)
+    path_left = w // 4
+    path_right = 3 * w // 4
+    path_top = h // 2
+    path_bottom = h
+
+    # Check if path is blocked
+    blocked_zones = []
+    for ob in obstacles:
+        x1, y1, x2, y2 = ob['box']
+        if y2 > path_top:  # obstacle in lower half
+            blocked_zones.append(ob)
+
+    if not blocked_zones:
+        # Green path — clear
+        pts = np.array([
+            [path_left + 30, path_top],
+            [path_right - 30, path_top],
+            [path_right + 20, path_bottom],
+            [path_left - 20, path_bottom],
+        ])
+        cv2.fillPoly(overlay, [pts], (0, 120, 0))
+        cv2.addWeighted(overlay, 0.2, frame, 0.8, 0, frame)
+        cv2.polylines(frame, [pts], True, (0, 255, 0), 2)
+        cv2.putText(frame, "CLEAR PATH", (w // 2 - 60, path_top + 30),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2)
+    else:
+        # Find which side is clear
+        left_blocked = any(ob['box'][0] < w // 2 for ob in blocked_zones)
+        right_blocked = any(ob['box'][2] > w // 2 for ob in blocked_zones)
+
+        if not left_blocked:
+            # Suggest left path
+            pts = np.array([[10, path_top], [w // 3, path_top],
+                            [w // 3 + 20, path_bottom], [10, path_bottom]])
+            cv2.fillPoly(overlay, [pts], (0, 120, 0))
+            cv2.addWeighted(overlay, 0.25, frame, 0.75, 0, frame)
+            cv2.arrowedLine(frame, (w // 2, h // 2), (w // 6, h // 2),
+                            (0, 255, 0), 3, tipLength=0.3)
+            cv2.putText(frame, "GO LEFT", (20, path_top + 30),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)
+        elif not right_blocked:
+            # Suggest right path
+            pts = np.array([[2 * w // 3, path_top], [w - 10, path_top],
+                            [w - 10, path_bottom], [2 * w // 3 - 20, path_bottom]])
+            cv2.fillPoly(overlay, [pts], (0, 120, 0))
+            cv2.addWeighted(overlay, 0.25, frame, 0.75, 0, frame)
+            cv2.arrowedLine(frame, (w // 2, h // 2), (5 * w // 6, h // 2),
+                            (0, 255, 0), 3, tipLength=0.3)
+            cv2.putText(frame, "GO RIGHT", (2 * w // 3, path_top + 30),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)
+        else:
+            # Both sides blocked
+            cv2.rectangle(overlay, (0, path_top), (w, path_bottom), (0, 0, 150), -1)
+            cv2.addWeighted(overlay, 0.3, frame, 0.7, 0, frame)
+            cv2.putText(frame, "STOP - PATH BLOCKED", (w // 2 - 120, h // 2),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
+
+    # Slope direction arrow at bottom center
+    if abs(slope_angle) > 3:
+        acx, acy = w // 2, h - 40
+        arad = math.radians(slope_angle)
+        ax = int(acx + math.cos(arad) * 35)
+        ay = int(acy - math.sin(arad) * 35)
+        cv2.arrowedLine(frame, (acx, acy), (ax, ay), (0, 255, 255), 3, tipLength=0.3)
+        cv2.putText(frame, f"{slope_dir} {slope_angle:.0f} deg",
+                    (acx - 60, acy + 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
+
+
+def draw_obstacles_bold(frame, obstacles):
+    """Draw highly visible obstacle markers."""
+    for ob in obstacles:
+        x1, y1, x2, y2 = ob['box']
+        u = ob['proximity']
+
+        # Color by urgency
+        if u > 0.7:
+            color = (0, 0, 255)      # red = NEAR
+            label_bg = (0, 0, 200)
+        elif u > 0.4:
+            color = (0, 180, 255)    # orange = MID
+            label_bg = (0, 140, 200)
+        else:
+            color = (0, 200, 0)      # green = FAR
+            label_bg = (0, 160, 0)
+
+        # Semi-transparent fill
+        overlay = frame.copy()
+        cv2.rectangle(overlay, (x1, y1), (x2, y2), color, -1)
+        alpha = 0.35 if u > 0.7 else 0.2
+        cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0, frame)
+
+        # Thick border
+        cv2.rectangle(frame, (x1, y1), (x2, y2), color, 3)
+
+        # Label
+        label = f"{ob['label'].upper()} {ob['dist']}"
+        if 'track_id' in ob:
+            label = f"#{ob['track_id']} {label}"
+        (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)
+        cv2.rectangle(frame, (x1, y1 - th - 10), (x1 + tw + 6, y1), label_bg, -1)
+        cv2.putText(frame, label, (x1 + 3, y1 - 5),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
+
+        # Direction arrow
+        cx, cy = ob['center']
+        if ob['direction'] == "LEFT":
+            cv2.arrowedLine(frame, (cx + 50, cy), (cx - 50, cy), (255, 255, 255), 3, tipLength=0.4)
+        elif ob['direction'] == "RIGHT":
+            cv2.arrowedLine(frame, (cx - 50, cy), (cx + 50, cy), (255, 255, 255), 3, tipLength=0.4)
+        else:
+            cv2.arrowedLine(frame, (cx, cy - 40), (cx, cy + 40), (0, 0, 255), 3, tipLength=0.4)
+
+        # Distance warning for NEAR
+        if u > 0.7:
+            cv2.putText(frame, "!! CLOSE !!", (cx - 40, y2 + 20),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
+
+
+def draw_hud(frame, risk_dict, guidance, fps_val):
+    """Draw navigation HUD."""
+    h, w = frame.shape[:2]
+
+    # Top bar
+    cv2.rectangle(frame, (0, 0), (w, 80), (0, 0, 0), -1)
+
+    # Risk badge
+    risk = risk_dict['risk']
+    rc = {'SAFE': (0, 180, 0), 'LOW': (0, 220, 0), 'MEDIUM': (0, 180, 220), 'HIGH': (0, 0, 220)}[risk]
+    cv2.rectangle(frame, (5, 5), (180, 38), rc, -1)
+    cv2.putText(frame, f"RISK: {risk} ({risk_dict['score']})", (10, 30),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2)
+
+    # Slope
+    cv2.putText(frame, f"Slope: {risk_dict['terrain']} {risk_dict['terrain_slope']:.0f} deg",
+                (190, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (200, 200, 200), 1)
+
+    # Guidance
+    cv2.putText(frame, f"Step: {guidance['step']} | Lean: {guidance['lean']} | Knee: {guidance['knee_rec']} deg",
+                (5, 55), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
+
+    # FPS
+    cv2.putText(frame, f"{fps_val:.0f}fps", (w - 60, 30),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.4, (100, 100, 100), 1)
+
+    # Bottom voice bar
+    cv2.rectangle(frame, (0, h - 45), (w, h), (20, 20, 40), -1)
+    cv2.putText(frame, "VOICE:", (8, h - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (100, 180, 255), 1)
+    cv2.putText(frame, guidance['voice'][:60], (75, h - 20),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
+
+
+def main():
+    # Parse camera source
+    if len(sys.argv) > 1 and sys.argv[1] != "--help":
+        source = sys.argv[1]
+        # IP Webcam app URLs
+        if source.startswith("http") and not source.endswith("/video"):
+            source = source.rstrip("/") + "/video"
+        print(f"Connecting to: {source}")
+    else:
+        source = 0
+        print("Using laptop webcam (pass phone URL as argument)")
+
+    try:
+        cam = CameraStream(source)
+    except RuntimeError as e:
+        print(f"Error: {e}")
+        print("\nUsage:")
+        print("  python app_mobile.py                          # laptop webcam")
+        print("  python app_mobile.py http://192.168.1.5:8080  # IP Webcam app")
+        return
+
+    # Init components
+    guide = GuidanceEngine()
+    smoother = SlopeSmoother()
+    tracker = ObstacleTracker()
+    tts = TTSEngine(enabled=True)
+
+    cached_depth = None
+    cached_depth_mini = None
+    frame_count = 0
+    fps_val = 0.0
+    t0 = time.time()
+
+    print(f"\n{'='*50}")
+    print(f"LIVE NAVIGATION ASSISTANT")
+    print(f"Camera: {cam.w}x{cam.h} @ {cam.native_fps:.0f}fps")
+    print(f"Press 'q' to quit")
+    print(f"{'='*50}\n")
+
+    while cam.is_open:
+        frame = cam.read()
+        if frame is None:
+            time.sleep(0.01)
+            continue
+
+        h, w = frame.shape[:2]
+        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        # Detect + track
+        obstacles = detector.detect(frame, track=True)
+        new_obs, closing_obs, _ = tracker.update(obstacles)
+
+        # Pose
+        gait, landmarks, foot_y = pose.analyze(rgb, w, h)
+
+        # Depth every 5th frame
+        frame_count += 1
+        if frame_count % DEFAULT_DEPTH_EVERY == 0 or cached_depth is None:
+            cached_depth = depth.estimate_depth(rgb, h, w)
+            raw_s, raw_d, raw_t, _ = depth.estimate_slope(cached_depth, h, w, foot_y)
+            smoother.update(raw_s, raw_d, raw_t)
+
+        # Risk + guidance
+        risk_dict = risk_engine.assess(
+            smoother.angle, smoother.direction, gait, obstacles,
+            slope_trend=smoother.trend, new_obstacles=new_obs, closing_obstacles=closing_obs)
+        guidance = guide.compute(
+            smoother.angle, smoother.direction, smoother.terrain, obstacles,
+            slope_trend=smoother.trend, new_obstacles=new_obs, closing_obstacles=closing_obs)
+
+        # Render
+        out = frame.copy()
+        draw_path_zone(out, obstacles, smoother.direction, smoother.angle)
+        draw_obstacles_bold(out, obstacles)
+        draw_hud(out, risk_dict, guidance, fps_val)
+
+        # TTS
+        if tts.enabled and guide.should_speak(guidance['voice'], smoother.angle):
+            tts.speak(guidance['voice'])
+
+        # FPS
+        if frame_count % 10 == 0:
+            fps_val = frame_count / (time.time() - t0)
+
+        cv2.imshow("Navigation Assistant", out)
+        key = cv2.waitKey(1) & 0xFF
+        if key == ord('q'):
+            break
+
+    cam.release()
+    cv2.destroyAllWindows()
+    tts.shutdown()
+    print(f"\nSession: {frame_count} frames in {time.time() - t0:.1f}s ({fps_val:.1f} fps)")
+
+
+if __name__ == "__main__":
+    main()

app_new.py

@@ -0,0 +1,138 @@
+"""Gradio UI — image analysis, video processing, and live webcam streaming."""
+import gradio as gr
+import cv2
+import numpy as np
+from PIL import Image as PILImage
+from pipeline import process_image, process_video
+from core import detector, depth, pose, risk_engine
+from core.depth import SlopeSmoother
+from core.detector import ObstacleTracker
+from core.guidance import GuidanceEngine
+from renderers import overlay as overlay_renderer
+
+# ─── Image handler ───
+
+def handle_image(input_image):
+    if input_image is None:
+        return None, "Upload an image."
+    img_bgr = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR)
+    rendered, risk_dict, guidance = process_image(img_bgr, mode="overlay")
+
+    rpt = f"## Risk: **{risk_dict['risk']}** (Score: {risk_dict['score']})\n\n"
+    rpt += f"**Terrain:** {risk_dict['terrain']} ({risk_dict['terrain_slope']:.1f}°)\n\n"
+    gs = risk_dict.get('gait_summary', {})
+    if gs:
+        rpt += "| Metric | Value |\n|---|---|\n"
+        for k, v in gs.items():
+            rpt += f"| {k.title()} | {v:.1f}° |\n"
+    rpt += "\n**Risk Factors:**\n"
+    rpt += "\n".join(f"- ⚠️ {r}" for r in risk_dict['reasons']) if risk_dict['reasons'] else "- ✅ None"
+    rpt += f"\n\n**Voice:** {guidance['voice']}"
+
+    return PILImage.fromarray(cv2.cvtColor(rendered, cv2.COLOR_BGR2RGB)), rpt
+
+
+# ─── Video handler ───
+
+def handle_video(video, mode):
+    if video is None:
+        return None
+    return process_video(video, mode=mode)
+
+
+# ─── Live webcam handler (Gradio streaming) ───
+
+# Persistent state for live stream
+_live_state = {
+    'guide': None,
+    'smoother': None,
+    'tracker': None,
+    'depth': None,
+    'depth_mini': None,
+    'counter': 0,
+}
+
+
+def _reset_live():
+    _live_state['guide'] = GuidanceEngine()
+    _live_state['smoother'] = SlopeSmoother()
+    _live_state['tracker'] = ObstacleTracker()
+    _live_state['depth'] = None
+    _live_state['depth_mini'] = None
+    _live_state['counter'] = 0
+
+
+def handle_webcam_frame(frame):
+    """Process a single webcam frame from Gradio's streaming input."""
+    if frame is None:
+        return None, ""
+
+    # Init state on first frame
+    if _live_state['guide'] is None:
+        _reset_live()
+
+    img_bgr = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)
+    h, w = img_bgr.shape[:2]
+    rgb = np.array(frame)
+
+    obstacles = detector.detect(img_bgr, track=True)
+    new_obs, closing_obs, _ = _live_state['tracker'].update(obstacles)
+    gait, landmarks, foot_y = pose.analyze(rgb, w, h)
+
+    _live_state['counter'] += 1
+    if _live_state['counter'] % 5 == 0 or _live_state['depth'] is None:
+        _live_state['depth'] = depth.estimate_depth(rgb, h, w)
+        raw_s, raw_d, raw_t, _ = depth.estimate_slope(_live_state['depth'], h, w, foot_y)
+        _live_state['smoother'].update(raw_s, raw_d, raw_t)
+        _live_state['depth_mini'] = overlay_renderer.render_depth_mini(_live_state['depth'], w, h)
+
+    sm = _live_state['smoother']
+    risk_dict = risk_engine.assess(sm.angle, sm.direction, gait, len(obstacles))
+    guidance = _live_state['guide'].compute(
+        sm.angle, sm.direction, sm.terrain, obstacles,
+        slope_trend=sm.trend, new_obstacles=new_obs, closing_obstacles=closing_obs)
+
+    rendered = overlay_renderer.render(
+        img_bgr, obstacles, gait, landmarks, risk_dict, guidance, _live_state['depth_mini'])
+
+    status = (f"**{risk_dict['risk']}** | Slope: {sm.direction} {sm.angle:.0f}° "
+              f"[{sm.trend}] | Obs: {len(obstacles)} | {guidance['voice'][:80]}")
+
+    return PILImage.fromarray(cv2.cvtColor(rendered, cv2.COLOR_BGR2RGB)), status
+
+
+# ─── Build UI ───
+
+with gr.Blocks(title="Navigation Assist") as demo:
+    gr.Markdown("# 🦯 Vision-Based Navigation Assistance")
+
+    with gr.Tab("📷 Image"):
+        with gr.Row():
+            img_in = gr.Image(type="pil", label="Upload Image")
+            img_out = gr.Image(label="Analysis")
+        report = gr.Markdown()
+        gr.Button("🔍 Analyze", variant="primary").click(
+            fn=handle_image, inputs=img_in, outputs=[img_out, report])
+
+    with gr.Tab("🎥 Video"):
+        with gr.Row():
+            vid_in = gr.Video(label="Upload Video")
+            vid_out = gr.Video(label="Output")
+        vid_mode = gr.Radio(["overlay", "blind_nav"], value="overlay", label="Render Mode")
+        gr.Button("🔍 Process", variant="primary").click(
+            fn=handle_video, inputs=[vid_in, vid_mode], outputs=vid_out)
+
+    with gr.Tab("📹 Live Camera"):
+        gr.Markdown("Enable your webcam below. Each frame is processed in real-time.")
+        with gr.Row():
+            cam_in = gr.Image(sources=["webcam"], streaming=True, label="Webcam")
+            cam_out = gr.Image(label="Live Analysis")
+        live_status = gr.Markdown("Waiting for camera...")
+        cam_in.stream(fn=handle_webcam_frame, inputs=cam_in, outputs=[cam_out, live_status])
+        gr.Button("🔄 Reset State").click(fn=lambda: (_reset_live(), None, "Reset."),
+                                           outputs=[cam_out, live_status])
+
+    gr.Markdown("---\n**Models:** YOLOv8n · BlazePose · Depth Anything · Rule-based risk fusion")
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860, show_error=True)

benchmark.py

@@ -0,0 +1,98 @@
+"""Benchmark each pipeline component to identify bottlenecks.
+
+Usage: python benchmark.py [image_path] [--depth-model small|base|v2-small|v2-base] [--size 256|384|512]
+"""
+import cv2
+import time
+import sys
+import os
+import numpy as np
+
+
+def bench(label, fn, runs=3):
+    """Run fn multiple times, print avg latency."""
+    times = []
+    for i in range(runs):
+        t0 = time.time()
+        result = fn()
+        times.append(time.time() - t0)
+    avg = sum(times) / len(times)
+    fps = 1.0 / avg if avg > 0 else 999
+    print(f"  {label:30s}  {avg*1000:7.1f}ms  ({fps:.1f} fps)")
+    return result, avg
+
+
+def main():
+    # Parse args
+    img_path = None
+    for a in sys.argv[1:]:
+        if not a.startswith("--") and os.path.exists(a):
+            img_path = a
+
+    if "--depth-model" in sys.argv:
+        idx = sys.argv.index("--depth-model")
+        os.environ["NAV_DEPTH_MODEL"] = sys.argv[idx + 1]
+
+    if "--size" in sys.argv:
+        idx = sys.argv.index("--size")
+        os.environ["NAV_DEPTH_INPUT_SIZE"] = sys.argv[idx + 1]
+
+    # Use a test image or generate one
+    if img_path:
+        img = cv2.imread(img_path)
+    else:
+        test_dir = "/mnt/c/Visual/test_images"
+        candidates = [f for f in os.listdir(test_dir) if f.endswith('.jpg') and '_pose' not in f]
+        if candidates:
+            img = cv2.imread(os.path.join(test_dir, candidates[0]))
+        else:
+            print("No test image found. Pass an image path.")
+            return
+
+    h, w = img.shape[:2]
+    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+    from config import DEPTH_MODEL, DEPTH_INPUT_SIZE, DEVICE
+    print(f"Image: {w}x{h}")
+    print(f"Depth model: {DEPTH_MODEL}  input_size: {DEPTH_INPUT_SIZE}  device: {DEVICE}")
+    print(f"{'='*60}")
+
+    # Warm up + benchmark each component
+    from core import detector, depth, pose, risk_engine
+
+    print("\n[1] YOLO Obstacle Detection")
+    obstacles, t_yolo = bench("yolo detect", lambda: detector.detect(img))
+
+    print("\n[2] YOLO + ByteTrack")
+    _, t_track = bench("yolo track", lambda: detector.detect(img, track=True))
+
+    print("\n[3] BlazePose Gait Analysis")
+    pose_result, t_pose = bench("pose analyze", lambda: pose.analyze(rgb, w, h))
+    gait, landmarks, foot_y = pose_result
+
+    print("\n[4] Depth Estimation")
+    depth_norm, t_depth = bench("depth estimate", lambda: depth.estimate_depth(rgb, h, w))
+
+    print("\n[5] Slope Analysis (on cached depth)")
+    _, t_slope = bench("slope estimate", lambda: depth.estimate_slope(depth_norm, h, w, foot_y))
+
+    print("\n[6] Risk Assessment")
+    _, t_risk = bench("risk assess", lambda: risk_engine.assess(0.0, "FLAT", gait, len(obstacles)))
+
+    print(f"\n{'='*60}")
+    total = t_yolo + t_pose + t_depth + t_slope + t_risk
+    print(f"  {'TOTAL (per frame)':30s}  {total*1000:7.1f}ms  ({1.0/total:.1f} fps)")
+    print(f"  {'Without depth':30s}  {(total-t_depth)*1000:7.1f}ms  ({1.0/(total-t_depth):.1f} fps)")
+    print(f"\nDepth is {t_depth/total*100:.0f}% of total latency.")
+
+    if t_depth > 0.15:
+        print("\nTips to speed up depth:")
+        print(f"  - Current input size: {DEPTH_INPUT_SIZE}. Try: NAV_DEPTH_INPUT_SIZE=256")
+        print(f"  - Current model: {DEPTH_MODEL}. 'small' is fastest.")
+        if DEVICE == "cpu":
+            print("  - Running on CPU. Set NAV_DEVICE=cuda if GPU available.")
+        print("  - Export to ONNX: set NAV_DEPTH_ONNX=/path/to/model.onnx")
+
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,71 @@
+"""Centralized configuration for the navigation system."""
+import os
+
+# Model paths
+YOLO_MODEL = os.environ.get("NAV_YOLO_MODEL", "yolov8n.pt")
+POSE_MODEL = os.environ.get("NAV_POSE_MODEL", "/mnt/c/Visual/pose_landmarker_heavy.task")
+# Depth model options: "small", "base", "v2-small", "v2-base"
+# v2 models are more accurate. "small" variants are faster.
+DEPTH_MODEL = os.environ.get("NAV_DEPTH_MODEL", "small")
+DEPTH_INPUT_SIZE = int(os.environ.get("NAV_DEPTH_INPUT_SIZE", 384))  # resize before inference
+DEVICE = os.environ.get("NAV_DEVICE", "cpu")
+
+# Auto-detect GPU if not explicitly set
+if "NAV_DEVICE" not in os.environ:
+    try:
+        import torch
+        if torch.cuda.is_available():
+            DEVICE = "cuda"
+    except ImportError:
+        pass
+
+# Detection
+YOLO_CONF = 0.35
+OBSTACLE_CLASSES = {
+    0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 5: 'bus', 7: 'truck',
+    9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign', 13: 'bench',
+    15: 'cat', 16: 'dog', 24: 'backpack', 25: 'umbrella', 56: 'chair',
+    57: 'couch', 58: 'potted plant', 60: 'dining table',
+}
+
+# Depth / slope
+GROUND_RATIO = 0.55          # default ground region starts at 55% of frame height
+SLOPE_MULTIPLIER = 20.0      # arctan scaling — calibrate on known slopes
+SLOPE_CLAMP = 35.0           # max slope angle (degrees)
+SLOPE_DEADZONE = 3.0         # angles below this → FLAT
+SLOPE_SMOOTHING = 0.7        # EMA alpha for temporal smoothing (0=no smoothing, 1=full cache)
+
+# Terrain roughness thresholds (std of ground depth)
+TERRAIN_ROCKY_THRESH = 0.25
+TERRAIN_ROUGH_THRESH = 0.15
+
+# Pose landmark indices
+LM = {
+    'L_SHOULDER': 11, 'R_SHOULDER': 12,
+    'L_HIP': 23, 'R_HIP': 24,
+    'L_KNEE': 25, 'R_KNEE': 26,
+    'L_ANKLE': 27, 'R_ANKLE': 28,
+    'L_HEEL': 29, 'R_HEEL': 30,
+    'L_FOOT': 31, 'R_FOOT': 32,
+}
+SKELETON_CONNS = [
+    (11, 13), (13, 15), (12, 14), (14, 16), (11, 12), (11, 23), (12, 24), (23, 24),
+    (23, 25), (25, 27), (27, 29), (27, 31), (29, 31),
+    (24, 26), (26, 28), (28, 30), (28, 32), (30, 32),
+]
+
+# Risk thresholds
+RISK_HIGH = 60
+RISK_MEDIUM = 30
+RISK_LOW = 10
+
+# User mobility profiles: "default", "elderly", "athletic"
+RISK_PROFILE = os.environ.get("NAV_RISK_PROFILE", "default")
+
+# Voice throttle
+VOICE_COOLDOWN_SEC = 2.5     # min seconds between repeated messages
+VOICE_SLOPE_DELTA = 5.0      # slope must change by this much to re-announce
+
+# Video processing
+DEFAULT_SKIP_FRAMES = 2
+DEFAULT_DEPTH_EVERY = 5

core/__init__.py

@@ -0,0 +1 @@
+

core/camera.py

@@ -0,0 +1,69 @@
+"""Live camera capture with threaded frame reading for consistent FPS.
+
+Supports:
+  - USB webcam: CameraStream(0)
+  - RTSP/IP cam: CameraStream("rtsp://user:pass@192.168.1.100:554/stream")
+  - Video file (treated as stream): CameraStream("/path/to/video.mp4")
+"""
+import cv2
+import threading
+import time
+
+
+class CameraStream:
+    """Threaded camera reader — always holds the latest frame, never blocks."""
+
+    def __init__(self, source=0, target_fps=15):
+        self.source = source
+        self.target_fps = target_fps
+        self._cap = cv2.VideoCapture(source)
+
+        if not self._cap.isOpened():
+            raise RuntimeError(f"Cannot open camera: {source}")
+
+        # Read one frame to get dimensions
+        ret, frame = self._cap.read()
+        if not ret:
+            raise RuntimeError(f"Cannot read from camera: {source}")
+
+        self.frame = frame
+        self.w = int(self._cap.get(3))
+        self.h = int(self._cap.get(4))
+        self.native_fps = self._cap.get(5) or 30.0
+
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._reader, daemon=True)
+        self._thread.start()
+
+        print(f"[Camera] Opened {source} ({self.w}x{self.h} @ {self.native_fps:.0f}fps)", flush=True)
+
+    def _reader(self):
+        """Continuously grab frames in background."""
+        while not self._stop.is_set():
+            ret, frame = self._cap.read()
+            if not ret:
+                # End of stream or disconnect — try reconnect for RTSP
+                if isinstance(self.source, str) and self.source.startswith("rtsp"):
+                    print("[Camera] Lost connection, reconnecting...", flush=True)
+                    time.sleep(2)
+                    self._cap.release()
+                    self._cap = cv2.VideoCapture(self.source)
+                    continue
+                break
+            with self._lock:
+                self.frame = frame
+
+    def read(self):
+        """Get the latest frame. Never blocks."""
+        with self._lock:
+            return self.frame.copy() if self.frame is not None else None
+
+    @property
+    def is_open(self):
+        return not self._stop.is_set() and self._cap.isOpened()
+
+    def release(self):
+        self._stop.set()
+        self._thread.join(timeout=3)
+        self._cap.release()

core/depth.py

@@ -0,0 +1,233 @@
+"""Monocular depth estimation with model selection, resolution scaling, and temporal smoothing.
+
+Supported models (set via NAV_DEPTH_MODEL env or config.py):
+  "small"    — Depth Anything V1 Small  (fastest)
+  "base"     — Depth Anything V1 Base
+  "v2-small" — Depth Anything V2 Small  (recommended for MVP)
+  "v2-base"  — Depth Anything V2 Base   (best accuracy)
+
+Performance knobs:
+  NAV_DEPTH_INPUT_SIZE — resize input before inference (default 384, try 256 for speed)
+  NAV_DEVICE           — "cpu" or "cuda"
+"""
+import cv2
+import numpy as np
+import math
+import time
+from config import (
+    DEPTH_MODEL, DEPTH_INPUT_SIZE, DEVICE, GROUND_RATIO, SLOPE_MULTIPLIER,
+    SLOPE_CLAMP, SLOPE_DEADZONE,
+    TERRAIN_ROCKY_THRESH, TERRAIN_ROUGH_THRESH,
+)
+
+_MODEL_MAP = {
+    "small":    "LiheYoung/depth-anything-small-hf",
+    "base":     "LiheYoung/depth-anything-base-hf",
+    "v2-small": "depth-anything/Depth-Anything-V2-Small-hf",
+    "v2-base":  "depth-anything/Depth-Anything-V2-Base-hf",
+}
+
+_depth_pipe = None
+_onnx_session = None
+_backend = None  # "hf" or "onnx"
+
+
+def _load():
+    global _depth_pipe, _backend
+    if _depth_pipe is not None:
+        return
+
+    model_id = _MODEL_MAP.get(DEPTH_MODEL, DEPTH_MODEL)
+
+    # Try ONNX first if available
+    if _try_load_onnx(model_id):
+        return
+
+    # Fall back to HuggingFace pipeline
+    from transformers import pipeline as hf_pipeline
+    t0 = time.time()
+    _depth_pipe = hf_pipeline("depth-estimation", model=model_id, device=DEVICE)
+    _backend = "hf"
+    print(f"[Depth] Loaded {model_id} on {DEVICE} ({time.time()-t0:.1f}s)", flush=True)
+
+
+def _try_load_onnx(model_id):
+    """Try loading an ONNX-exported model for faster CPU inference."""
+    global _onnx_session, _backend
+    try:
+        import onnxruntime as ort
+        import os
+        # Look for local ONNX file
+        onnx_path = os.environ.get("NAV_DEPTH_ONNX")
+        if not onnx_path:
+            return False
+        if not os.path.exists(onnx_path):
+            print(f"[Depth] ONNX path not found: {onnx_path}", flush=True)
+            return False
+
+        providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if DEVICE == "cuda" else ['CPUExecutionProvider']
+        t0 = time.time()
+        _onnx_session = ort.InferenceSession(onnx_path, providers=providers)
+        _backend = "onnx"
+        print(f"[Depth] Loaded ONNX model ({time.time()-t0:.1f}s) providers={_onnx_session.get_providers()}", flush=True)
+        return True
+    except ImportError:
+        return False
+
+
+def _infer_hf(rgb_small):
+    """Run HuggingFace pipeline on a (possibly resized) RGB image."""
+    from PIL import Image as PILImage
+    return np.array(_depth_pipe(PILImage.fromarray(rgb_small))["depth"]).astype(np.float32)
+
+
+def _infer_onnx(rgb_small):
+    """Run ONNX session on preprocessed input."""
+    # Standard normalization for Depth Anything
+    img = rgb_small.astype(np.float32) / 255.0
+    mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+    std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+    img = (img - mean) / std
+    img = np.transpose(img, (2, 0, 1))[np.newaxis]  # NCHW
+
+    input_name = _onnx_session.get_inputs()[0].name
+    result = _onnx_session.run(None, {input_name: img})
+    return result[0].squeeze().astype(np.float32)
+
+
+def estimate_depth(frame_rgb, h, w):
+    """Run depth model. Returns normalized depth map (0-1) at original frame resolution."""
+    _load()
+
+    # Resize for faster inference
+    inp_size = DEPTH_INPUT_SIZE
+    if h > inp_size or w > inp_size:
+        scale = inp_size / max(h, w)
+        sh, sw = int(h * scale), int(w * scale)
+        rgb_small = cv2.resize(frame_rgb, (sw, sh))
+    else:
+        rgb_small = frame_rgb
+
+    # Inference
+    if _backend == "onnx":
+        # ONNX needs exact square input for some exports
+        sq = cv2.resize(rgb_small, (inp_size, inp_size))
+        dm = _infer_onnx(sq)
+    else:
+        dm = _infer_hf(rgb_small)
+
+    # Resize back to original resolution
+    dm = cv2.resize(dm, (w, h))
+
+    dmin, dmax = dm.min(), dm.max()
+    if dmax - dmin < 1e-8:
+        return np.zeros((h, w), dtype=np.float32)
+    return (dm - dmin) / (dmax - dmin)
+
+
+def estimate_slope(depth_norm, h, w, foot_y=None):
+    """Compute slope angle and direction from depth map ground region.
+
+    Returns (slope_angle, slope_dir, terrain_type, ground_start_y).
+    """
+    if foot_y and foot_y > h * 0.3:
+        gs = max(0, int(foot_y) - int(h * 0.1))
+    else:
+        gs = int(h * GROUND_RATIO)
+
+    ground = depth_norm[gs:, :]
+    gh, gw = ground.shape
+    if gh < 10 or gw < 10:
+        return 0.0, "FLAT", "SMOOTH", gs
+
+    gy = cv2.Sobel(ground, cv2.CV_64F, 0, 1, ksize=5)
+    cl, cr = gw // 3, 2 * gw // 3
+
+    rows = []
+    sh = max(1, gh // 6)
+    for i in range(6):
+        y0, y1 = i * sh, min((i + 1) * sh, gh)
+        c = np.concatenate([gy[y0:y1, :cl].flatten(), gy[y0:y1, cr:].flatten()])
+        if len(c):
+            rows.append(float(np.median(c)))
+
+    if len(rows) < 3:
+        return 0.0, "FLAT", "SMOOTH", gs
+
+    trend = np.polyfit(np.arange(len(rows)), np.array(rows), 1)[0]
+    sa = float(np.clip(np.arctan(trend * SLOPE_MULTIPLIER) * 180 / math.pi,
+                        -SLOPE_CLAMP, SLOPE_CLAMP))
+    if abs(sa) < SLOPE_DEADZONE:
+        sa = 0.0
+
+    slope_dir = "FLAT" if abs(sa) < SLOPE_DEADZONE else ("UPHILL" if sa > 0 else "DOWNHILL")
+
+    gvar = float(np.std(ground))
+    if gvar > TERRAIN_ROCKY_THRESH:
+        terrain = "ROCKY"
+    elif gvar > TERRAIN_ROUGH_THRESH:
+        terrain = "ROUGH"
+    else:
+        terrain = "SMOOTH"
+
+    return sa, slope_dir, terrain, gs
+
+
+class SlopeSmoother:
+    """Temporal smoothing for slope estimates across video frames."""
+
+    def __init__(self, alpha=0.7, outlier_thresh=15.0):
+        self.alpha = alpha
+        self.outlier_thresh = outlier_thresh
+        self._angle = 0.0
+        self._dir = "FLAT"
+        self._terrain = "SMOOTH"
+        self._history = []
+        self._max_history = 10
+
+    def update(self, raw_angle, raw_dir, raw_terrain):
+        """Feed a new raw slope measurement. Returns smoothed (angle, dir, terrain)."""
+        self._history.append(raw_angle)
+        if len(self._history) > self._max_history:
+            self._history.pop(0)
+
+        # Outlier rejection
+        if len(self._history) >= 3:
+            median = float(np.median(self._history))
+            if abs(raw_angle - median) > self.outlier_thresh:
+                raw_angle = median
+
+        self._angle = self.alpha * self._angle + (1 - self.alpha) * raw_angle
+
+        if abs(self._angle) < SLOPE_DEADZONE:
+            self._angle = 0.0
+            self._dir = "FLAT"
+        else:
+            self._dir = "UPHILL" if self._angle > 0 else "DOWNHILL"
+
+        self._terrain = raw_terrain
+        return self._angle, self._dir, self._terrain
+
+    @property
+    def trend(self):
+        if len(self._history) < 4:
+            return "STABLE"
+        recent = self._history[-4:]
+        avg_diff = sum(recent[i+1] - recent[i] for i in range(len(recent)-1)) / (len(recent)-1)
+        if avg_diff > 2.0:
+            return "STEEPENING"
+        elif avg_diff < -2.0:
+            return "FLATTENING"
+        return "STABLE"
+
+    @property
+    def angle(self):
+        return self._angle
+
+    @property
+    def direction(self):
+        return self._dir
+
+    @property
+    def terrain(self):
+        return self._terrain

core/detector.py

@@ -0,0 +1,101 @@
+"""YOLO obstacle detection with ByteTrack tracking and state management."""
+from config import YOLO_MODEL, YOLO_CONF, OBSTACLE_CLASSES
+
+_yolo = None
+
+
+def _load():
+    global _yolo
+    if _yolo is None:
+        from ultralytics import YOLO
+        _yolo = YOLO(YOLO_MODEL)
+    return _yolo
+
+
+def detect(frame, track=False):
+    """Detect obstacles. If track=True, uses ByteTrack for persistent IDs."""
+    yolo = _load()
+    h, w = frame.shape[:2]
+
+    if track:
+        results = yolo.track(frame, conf=YOLO_CONF, verbose=False, persist=True)[0]
+    else:
+        results = yolo(frame, conf=YOLO_CONF, verbose=False)[0]
+
+    obstacles = []
+    for box in results.boxes:
+        cls_id = int(box.cls[0])
+        if cls_id not in OBSTACLE_CLASSES:
+            continue
+        x1, y1, x2, y2 = map(int, box.xyxy[0])
+
+        # Filter out full-frame false positives (box covers >50% of frame area)
+        box_area = (x2 - x1) * (y2 - y1)
+        if box_area > 0.5 * h * w:
+            continue
+
+        proximity = y2 / h
+        cx = (x1 + x2) / 2
+        direction = "LEFT" if cx < w * 0.33 else "RIGHT" if cx > w * 0.66 else "CENTER"
+
+        ob = {
+            'label': OBSTACLE_CLASSES[cls_id],
+            'conf': float(box.conf[0]),
+            'box': (x1, y1, x2, y2),
+            'center': ((x1 + x2) // 2, (y1 + y2) // 2),
+            'proximity': proximity,
+            'dist': "NEAR" if proximity > 0.7 else "MID" if proximity > 0.4 else "FAR",
+            'direction': direction,
+        }
+        if track and box.id is not None:
+            ob['track_id'] = int(box.id[0])
+        obstacles.append(ob)
+
+    return sorted(obstacles, key=lambda o: -o['proximity'])
+
+
+class ObstacleTracker:
+    """Tracks obstacles across frames, detects new/closing objects."""
+
+    def __init__(self):
+        self._prev = {}       # track_id -> previous obstacle dict
+        self._new_ids = set()  # track_ids that appeared this frame
+        self._lost_ids = set() # track_ids that disappeared this frame
+
+    def update(self, obstacles):
+        """Update tracker state. Call once per frame after detect(track=True).
+
+        Returns (new_obstacles, closing_obstacles, lost_ids).
+          - new_obstacles: obstacles with track_ids not seen before
+          - closing_obstacles: obstacles whose proximity increased significantly
+          - lost_ids: track_ids from previous frame no longer present
+        """
+        current = {}
+        new_obs = []
+        closing_obs = []
+
+        for ob in obstacles:
+            tid = ob.get('track_id')
+            if tid is None:
+                continue
+            current[tid] = ob
+
+            if tid not in self._prev:
+                new_obs.append(ob)
+            else:
+                # Check if closing (proximity increasing = getting nearer)
+                prev_prox = self._prev[tid]['proximity']
+                delta = ob['proximity'] - prev_prox
+                if delta > 0.05:  # moved noticeably closer
+                    ob['closing_rate'] = round(delta, 3)
+                    closing_obs.append(ob)
+
+        self._lost_ids = set(self._prev.keys()) - set(current.keys())
+        self._new_ids = set(current.keys()) - set(self._prev.keys())
+        self._prev = current
+
+        return new_obs, closing_obs, self._lost_ids
+
+    @property
+    def active_count(self):
+        return len(self._prev)

core/guidance.py

@@ -0,0 +1,141 @@
+"""Voice and movement guidance with throttling and tracker-aware alerts."""
+import time
+from config import VOICE_COOLDOWN_SEC, VOICE_SLOPE_DELTA
+
+
+class GuidanceEngine:
+    """Generates movement guidance and throttled voice messages."""
+
+    def __init__(self):
+        self._last_voice = ""
+        self._last_voice_time = 0.0
+        self._last_slope_announced = 0.0
+
+    def compute(self, slope_angle, slope_dir, terrain, obstacles,
+                slope_trend="STABLE", new_obstacles=None, closing_obstacles=None):
+        """Return guidance dict with movement recommendations and voice string."""
+        sa = abs(slope_angle)
+
+        knee_rec = 135 if sa > 20 else 145 if sa > 10 else 155 if sa > 5 else 165
+
+        if sa > 15 or terrain in ("ROCKY", "ROUGH"):
+            step, step_len = "SHORT", "30cm"
+        elif sa > 5:
+            step, step_len = "MEDIUM", "50cm"
+        else:
+            step, step_len = "NORMAL", "70cm"
+
+        if slope_dir == "UPHILL":
+            foot_adj = round(min(25, sa * 0.7), 1)
+            lean = "FORWARD"
+        elif slope_dir == "DOWNHILL":
+            foot_adj = round(-min(20, sa * 0.6), 1)
+            lean = "BACKWARD"
+        else:
+            foot_adj, lean = 0.0, "UPRIGHT"
+
+        near_obs = [o for o in obstacles if o['proximity'] > 0.6]
+        risk_score = min(100, int(sa * 1.5 + len(near_obs) * 20 +
+                                  (15 if terrain == "ROCKY" else 0)))
+
+        voice = self._build_voice(
+            slope_angle, slope_dir, terrain, obstacles, sa, step,
+            near_obs, slope_trend, new_obstacles, closing_obstacles)
+
+        return {
+            'knee_rec': knee_rec,
+            'step': step,
+            'step_len': step_len,
+            'foot_adj': foot_adj,
+            'lean': lean,
+            'risk_score': risk_score,
+            'voice': voice,
+            'obstacle_warning': self._obstacle_warning(near_obs),
+            'slope_trend': slope_trend,
+        }
+
+    def _build_voice(self, slope_angle, slope_dir, terrain, obstacles, sa, step,
+                     near_obs, slope_trend, new_obstacles, closing_obstacles):
+        parts = []
+
+        # Priority 1: NEW obstacles entering scene
+        if new_obstacles:
+            for ob in new_obstacles[:2]:
+                parts.append(f"New {ob['label']} on {ob['direction'].lower()}.")
+
+        # Priority 2: CLOSING obstacles (approaching fast)
+        if closing_obstacles:
+            for ob in closing_obstacles[:2]:
+                if ob['direction'] == "CENTER":
+                    parts.append(f"{ob['label']} approaching ahead!")
+                else:
+                    opp = "right" if ob['direction'] == "LEFT" else "left"
+                    parts.append(f"{ob['label']} closing from {ob['direction'].lower()}. Move {opp}.")
+
+        # Priority 3: Already-near obstacles
+        if not closing_obstacles:
+            for ob in near_obs[:2]:
+                if ob['direction'] == "CENTER":
+                    parts.append(f"{ob['label']} ahead! Stop.")
+                else:
+                    opp = "right" if ob['direction'] == "LEFT" else "left"
+                    parts.append(f"{ob['label']} on {ob['direction'].lower()}. Move {opp}.")
+
+        # Priority 4: Slope trend changes
+        if slope_trend == "STEEPENING" and sa > 5:
+            parts.append("Slope increasing. Slow down.")
+        elif slope_trend == "FLATTENING" and sa > 3:
+            parts.append("Slope easing.")
+
+        # Priority 5: Current slope guidance
+        if sa > 3:
+            parts.append(f"Slope {slope_angle:+.0f} degrees.")
+            if slope_dir == "UPHILL":
+                parts.append("Lean forward." if sa > 10 else "Slight forward lean.")
+            else:
+                parts.append("Lean back." if sa > 10 else "Slight backward lean.")
+
+        if sa > 15 or terrain in ("ROCKY", "ROUGH"):
+            parts.append("Short steps.")
+        elif sa > 5:
+            parts.append("Medium steps.")
+
+        if terrain == "ROCKY":
+            parts.append("Uneven ground.")
+        elif terrain == "ROUGH":
+            parts.append("Rough surface.")
+
+        if not parts:
+            parts.append("Path clear.")
+
+        return " ".join(parts)
+
+    def _obstacle_warning(self, near_obs):
+        if not near_obs:
+            return None
+        ob = near_obs[0]
+        if ob['direction'] == "CENTER":
+            return f"{ob['label']} AHEAD — STOP"
+        opp = "RIGHT" if ob['direction'] == "LEFT" else "LEFT"
+        return f"{ob['label']} on {ob['direction']} — move {opp}"
+
+    def should_speak(self, voice, slope_angle):
+        """Throttle: returns True if this message should be spoken aloud."""
+        now = time.time()
+        # Always speak obstacle warnings immediately
+        if any(kw in voice.lower() for kw in ["ahead", "stop", "new ", "closing", "approaching"]):
+            self._last_voice = voice
+            self._last_voice_time = now
+            return True
+        # Slope changed significantly
+        if abs(slope_angle - self._last_slope_announced) > VOICE_SLOPE_DELTA:
+            self._last_slope_announced = slope_angle
+            self._last_voice = voice
+            self._last_voice_time = now
+            return True
+        # Cooldown elapsed and message changed
+        if now - self._last_voice_time > VOICE_COOLDOWN_SEC and voice != self._last_voice:
+            self._last_voice = voice
+            self._last_voice_time = now
+            return True
+        return False

core/pose.py

@@ -0,0 +1,71 @@
+"""BlazePose gait analysis."""
+import numpy as np
+import math
+from config import LM, POSE_MODEL
+
+_pose_lm = None
+
+
+def _load():
+    global _pose_lm
+    if _pose_lm is None:
+        import mediapipe as mp
+        opts = mp.tasks.vision.PoseLandmarkerOptions(
+            base_options=mp.tasks.BaseOptions(model_asset_path=POSE_MODEL),
+            running_mode=mp.tasks.vision.RunningMode.IMAGE,
+            num_poses=1,
+            min_pose_detection_confidence=0.5,
+        )
+        _pose_lm = mp.tasks.vision.PoseLandmarker.create_from_options(opts)
+    return _pose_lm
+
+
+def _angle(a, b, c):
+    ba = np.array(a) - np.array(b)
+    bc = np.array(c) - np.array(b)
+    cos = np.dot(ba, bc) / (np.linalg.norm(ba) * np.linalg.norm(bc) + 1e-8)
+    return math.degrees(math.acos(np.clip(cos, -1, 1)))
+
+
+def _vert(top, bot):
+    return math.degrees(math.atan2(bot[0] - top[0], bot[1] - top[1]))
+
+
+def _foot_tilt(heel, toe):
+    dx = toe[0] - heel[0]
+    dy = -(toe[1] - heel[1])
+    return math.degrees(math.atan2(dy, dx))
+
+
+def analyze(frame_rgb, w, h):
+    """Run pose detection and extract gait metrics.
+
+    Returns (gait_dict, landmarks, foot_y) or (None, None, None).
+    """
+    import mediapipe as mp
+    lm_model = _load()
+    mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
+    res = lm_model.detect(mp_img)
+
+    if not res.pose_landmarks:
+        return None, None, None
+
+    lms = res.pose_landmarks[0]
+    foot_y = max(lms[i].y * h for i in [27, 28, 29, 30, 31, 32])
+
+    gait = {}
+    for side, pfx in [('L', 'L_'), ('R', 'R_')]:
+        p = {n: (lms[LM[f'{pfx}{n}']].x * w, lms[LM[f'{pfx}{n}']].y * h)
+             for n in ['SHOULDER', 'HIP', 'KNEE', 'ANKLE', 'HEEL', 'FOOT']}
+        gait[f'{side}_knee'] = round(_angle(p['HIP'], p['KNEE'], p['ANKLE']), 1)
+        gait[f'{side}_ankle'] = round(_angle(p['KNEE'], p['ANKLE'], p['FOOT']), 1)
+        gait[f'{side}_hip'] = round(_angle(p['SHOULDER'], p['HIP'], p['KNEE']), 1)
+        gait[f'{side}_shin'] = round(_vert(p['KNEE'], p['ANKLE']), 1)
+        gait[f'{side}_lean'] = round(_vert(p['SHOULDER'], p['HIP']), 1)
+        gait[f'{side}_foot_tilt'] = round(_foot_tilt(p['HEEL'], p['FOOT']), 1)
+
+    for k in ['knee', 'ankle', 'hip', 'shin', 'lean', 'foot_tilt']:
+        gait[f'avg_{k}'] = round((gait[f'L_{k}'] + gait[f'R_{k}']) / 2, 1)
+    gait['symmetry'] = round(abs(gait['L_knee'] - gait['R_knee']), 1)
+
+    return gait, lms, foot_y

core/risk_engine.py

@@ -0,0 +1,227 @@
+"""Risk fusion engine — contextual, trend-aware, with user profiles.
+
+Improvements over POC:
+  1. Contextual: bent knees on flat ground ≠ bent knees on a slope
+  2. Trend-based: steepening slope is riskier than steady slope
+  3. Obstacle velocity: closing obstacles score higher than static ones
+  4. User profiles: elderly/athletic/default adjust sensitivity
+  5. Temporal: risk decays slowly (no flicker between HIGH/LOW)
+"""
+from config import RISK_HIGH, RISK_MEDIUM, RISK_LOW, RISK_PROFILE
+
+# ─── User profiles: multipliers on base scores ───
+_PROFILES = {
+    "default":  {"slope": 1.0, "gait": 1.0, "obstacle": 1.0, "compound": 1.0},
+    "elderly":  {"slope": 1.5, "gait": 1.5, "obstacle": 1.3, "compound": 1.5},
+    "athletic": {"slope": 0.6, "gait": 0.5, "obstacle": 1.0, "compound": 0.5},
+}
+
+
+def _get_profile():
+    return _PROFILES.get(RISK_PROFILE, _PROFILES["default"])
+
+
+class RiskEngine:
+    """Stateful risk engine with temporal smoothing and context awareness."""
+
+    def __init__(self, profile=None):
+        self.profile = _PROFILES.get(profile or RISK_PROFILE, _PROFILES["default"])
+        self._prev_score = 0
+        self._prev_level = "SAFE"
+        self._decay = 0.7  # risk decays slowly to prevent flicker
+
+    def assess(self, slope_angle, slope_dir, gait, obstacles,
+               slope_trend="STABLE", new_obstacles=None, closing_obstacles=None):
+        """Compute contextual risk from all signals.
+
+        Returns dict with: risk, score, reasons, gait_summary, components.
+        """
+        sa = abs(slope_angle)
+        p = self.profile
+        score = 0
+        reasons = []
+        components = {}
+
+        # ── 1. Terrain slope ──
+        s_score = 0
+        if sa > 20:
+            s_score = 40
+            reasons.append(f"steep slope ({slope_angle:.0f}°)")
+        elif sa > 10:
+            s_score = 20
+            reasons.append(f"moderate slope ({slope_angle:.0f}°)")
+        elif sa > 3:
+            s_score = 5
+            reasons.append(f"mild slope ({slope_angle:.0f}°)")
+
+        # Trend bonus: steepening is riskier than steady
+        if slope_trend == "STEEPENING":
+            s_score = int(s_score * 1.4)
+            reasons.append("slope steepening")
+        elif slope_trend == "FLATTENING" and s_score > 0:
+            s_score = int(s_score * 0.7)
+
+        s_score = int(s_score * p["slope"])
+        components['slope'] = s_score
+        score += s_score
+
+        # ── 2. Gait analysis (contextual) ──
+        g_score = 0
+        gait_summary = {}
+        if gait:
+            kn = gait['avg_knee']
+            sh = abs(gait['avg_shin'])
+            ln = abs(gait['avg_lean'])
+            sy = gait['symmetry']
+            gait_summary = {'knee': kn, 'shin': sh, 'lean': ln, 'symmetry': sy}
+
+            # Bent knees: only risky if NOT on a slope (on slopes it's expected adaptation)
+            if kn < 130:
+                if sa < 5:
+                    # Bent knees on flat = potential instability
+                    g_score += 25
+                    reasons.append(f"heavily bent knees on flat ({kn:.0f}°)")
+                else:
+                    # Bent knees on slope = expected, mild concern only if extreme
+                    if kn < 110:
+                        g_score += 15
+                        reasons.append(f"extreme knee bend ({kn:.0f}°)")
+            elif kn < 150:
+                if sa < 5:
+                    g_score += 10
+                    reasons.append(f"bent knees on flat ({kn:.0f}°)")
+
+            # Shin tilt: contextual — expected to tilt on slopes
+            expected_shin = sa * 0.4  # rough expected shin tilt for slope
+            excess_shin = max(0, sh - expected_shin)
+            if excess_shin > 15:
+                g_score += 15
+                reasons.append(f"excess shin tilt ({sh:.0f}° vs expected {expected_shin:.0f}°)")
+            elif excess_shin > 8:
+                g_score += 8
+
+            # Body lean: expected on slopes, risky if opposite direction
+            if slope_dir == "UPHILL" and ln < -10:
+                g_score += 20
+                reasons.append(f"leaning backward on uphill ({ln:.0f}°)")
+            elif slope_dir == "DOWNHILL" and ln > 10:
+                g_score += 20
+                reasons.append(f"leaning forward on downhill ({ln:.0f}°)")
+            elif abs(ln) > 20:
+                g_score += 10
+                reasons.append(f"excessive lean ({ln:.0f}°)")
+
+            # Asymmetry: always concerning
+            if sy > 25:
+                g_score += 25
+                reasons.append(f"severe gait asymmetry ({sy:.0f}°)")
+            elif sy > 15:
+                g_score += 12
+                reasons.append(f"gait asymmetry ({sy:.0f}°)")
+
+            g_score = int(g_score * p["gait"])
+            components['gait'] = g_score
+            score += g_score
+
+        # ── 3. Obstacles ──
+        o_score = 0
+        num_obs = len(obstacles) if isinstance(obstacles, list) else obstacles
+
+        if isinstance(obstacles, list):
+            near = [o for o in obstacles if o.get('proximity', 0) > 0.6]
+            o_score += min(20, len(near) * 10)
+            if len(near) >= 2:
+                reasons.append(f"{len(near)} obstacles nearby")
+
+            # Closing obstacles are more dangerous
+            if closing_obstacles:
+                o_score += min(20, len(closing_obstacles) * 12)
+                for ob in closing_obstacles[:2]:
+                    rate = ob.get('closing_rate', 0)
+                    reasons.append(f"{ob['label']} closing ({rate:.0%}/frame)")
+
+            # New obstacles: brief awareness bump
+            if new_obstacles:
+                o_score += min(10, len(new_obstacles) * 5)
+        else:
+            o_score += min(30, num_obs * 10)
+
+        o_score = int(o_score * p["obstacle"])
+        components['obstacles'] = o_score
+        score += o_score
+
+        # ── 4. Compound risks ──
+        c_score = 0
+        if gait and sa > 10:
+            kn = gait['avg_knee']
+            sy = gait['symmetry']
+            if kn < 150 and sy > 15:
+                c_score += 20
+                reasons.append("slope + bent knees + asymmetry")
+            elif kn < 150:
+                c_score += 12
+                reasons.append("slope + bent knees")
+            elif sy > 15:
+                c_score += 12
+                reasons.append("slope + asymmetry")
+
+        if isinstance(obstacles, list):
+            near = [o for o in obstacles if o.get('proximity', 0) > 0.7]
+            if near and sa > 10:
+                c_score += 15
+                reasons.append("slope + near obstacle")
+
+        c_score = int(c_score * p["compound"])
+        components['compound'] = c_score
+        score += c_score
+
+        # ── 5. Temporal smoothing (prevent flicker) ──
+        raw_score = min(100, score)
+        smoothed = self._decay * self._prev_score + (1 - self._decay) * raw_score
+
+        # Snap up fast (danger), decay down slowly (safety)
+        if raw_score > self._prev_score:
+            smoothed = max(smoothed, raw_score * 0.85)  # jump up quickly
+
+        self._prev_score = smoothed
+        final_score = int(smoothed)
+
+        # Level
+        if final_score >= RISK_HIGH:
+            level = "HIGH"
+        elif final_score >= RISK_MEDIUM:
+            level = "MEDIUM"
+        elif final_score >= RISK_LOW:
+            level = "LOW"
+        else:
+            level = "SAFE"
+
+        self._prev_level = level
+
+        return {
+            'risk': level,
+            'score': final_score,
+            'raw_score': raw_score,
+            'terrain': slope_dir,
+            'terrain_slope': round(slope_angle, 1),
+            'slope_trend': slope_trend,
+            'reasons': reasons,
+            'gait_summary': gait_summary,
+            'components': components,
+            'profile': RISK_PROFILE,
+        }
+
+
+# ─── Backward-compatible module-level function ───
+_default_engine = None
+
+
+def assess(slope_angle, slope_dir, gait, num_obstacles,
+           slope_trend="STABLE", new_obstacles=None, closing_obstacles=None):
+    """Stateless convenience wrapper. For video, use RiskEngine class directly."""
+    global _default_engine
+    if _default_engine is None:
+        _default_engine = RiskEngine()
+    return _default_engine.assess(
+        slope_angle, slope_dir, gait, num_obstacles,
+        slope_trend, new_obstacles, closing_obstacles)

core/tts.py

@@ -0,0 +1,143 @@
+"""Text-to-speech engine with multiple backends and async playback.
+
+Backends (tried in order):
+  1. pyttsx3  — offline, cross-platform
+  2. edge-tts — Microsoft Edge TTS (async, high quality, needs internet)
+  3. espeak   — CLI fallback (Linux)
+
+Install one: pip install pyttsx3   OR   pip install edge-tts
+"""
+import threading
+import queue
+import time
+import os
+import tempfile
+
+
+class TTSEngine:
+    """Non-blocking TTS that speaks in a background thread."""
+
+    def __init__(self, enabled=True):
+        self.enabled = enabled
+        self._backend = None
+        self._queue = queue.Queue(maxsize=5)  # drop old messages if backed up
+        self._thread = None
+        self._stop = threading.Event()
+
+        if enabled:
+            self._backend = self._detect_backend()
+            if self._backend:
+                self._thread = threading.Thread(target=self._worker, daemon=True)
+                self._thread.start()
+            else:
+                print("[TTS] No backend available. Install: pip install pyttsx3")
+                self.enabled = False
+
+    def _detect_backend(self):
+        # Try edge-tts first — natural assistant voice
+        try:
+            import edge_tts
+            print("[TTS] Using edge-tts (GuyNeural)")
+            return ('edge_tts', None)
+        except ImportError:
+            pass
+
+        # Try pyttsx3
+        try:
+            import pyttsx3
+            engine = pyttsx3.init()
+            engine.setProperty('rate', 170)
+            engine.setProperty('volume', 1.0)
+            print("[TTS] Using pyttsx3")
+            return ('pyttsx3', engine)
+        except Exception:
+            pass
+
+        # Try espeak CLI
+        if os.system("which espeak > /dev/null 2>&1") == 0:
+            print("[TTS] Using espeak CLI")
+            return ('espeak', None)
+
+        return None
+
+    def speak(self, text):
+        """Queue text for speaking. Non-blocking. Drops if queue is full."""
+        if not self.enabled or not text:
+            return
+        try:
+            self._queue.put_nowait(text)
+        except queue.Full:
+            # Drop oldest, add new
+            try:
+                self._queue.get_nowait()
+            except queue.Empty:
+                pass
+            try:
+                self._queue.put_nowait(text)
+            except queue.Full:
+                pass
+
+    def _worker(self):
+        """Background thread that processes the speech queue."""
+        while not self._stop.is_set():
+            try:
+                text = self._queue.get(timeout=0.5)
+            except queue.Empty:
+                continue
+
+            # Drain queue — only speak the latest message
+            latest = text
+            while not self._queue.empty():
+                try:
+                    latest = self._queue.get_nowait()
+                except queue.Empty:
+                    break
+
+            self._speak_sync(latest)
+
+    def _speak_sync(self, text):
+        name, engine = self._backend
+        try:
+            if name == 'pyttsx3':
+                engine.say(text)
+                engine.runAndWait()
+            elif name == 'edge_tts':
+                self._speak_edge(text)
+            elif name == 'espeak':
+                # -s = speed (words per minute), -a = amplitude
+                safe = text.replace('"', '\\"').replace("'", "\\'")
+                os.system(f'espeak -s 170 -a 200 "{safe}" 2>/dev/null')
+        except Exception as e:
+            print(f"[TTS] Error: {e}")
+
+    def _speak_edge(self, text):
+        """edge-tts is async, run in a sync wrapper."""
+        import asyncio
+        import edge_tts
+
+        tmp = os.path.join(tempfile.gettempdir(), "nav_tts.mp3")
+
+        async def _gen():
+            comm = edge_tts.Communicate(text, "en-US-GuyNeural", rate="+15%")
+            await comm.save(tmp)
+
+        asyncio.run(_gen())
+
+        # Play with ffplay or aplay
+        if os.system(f"which ffplay > /dev/null 2>&1") == 0:
+            os.system(f"ffplay -nodisp -autoexit -loglevel error {tmp}")
+        elif os.system(f"which aplay > /dev/null 2>&1") == 0:
+            # Convert to wav first
+            wav = tmp.replace('.mp3', '.wav')
+            os.system(f"ffmpeg -y -i {tmp} {wav} -loglevel error 2>/dev/null")
+            os.system(f"aplay {wav} 2>/dev/null")
+
+    def shutdown(self):
+        self._stop.set()
+        if self._thread:
+            self._thread.join(timeout=2)
+        if self._backend and self._backend[0] == 'pyttsx3':
+            try:
+                self._backend[1].stop()
+            except Exception:
+                pass

core/tts_render.py

@@ -0,0 +1,175 @@
+"""Render TTS voice guidance as audio segments and merge into video.
+
+Generates .wav clips for each unique voice message, then composites
+them onto the video timeline using ffmpeg.
+"""
+import os
+import tempfile
+import json
+
+
+def render_voice_track(voice_events, output_audio_path, total_duration):
+    """Generate a single audio track from timestamped voice events."""
+    tmpdir = tempfile.mkdtemp(prefix="nav_tts_")
+    backend = _detect_backend()
+    if not backend:
+        print("[TTS-Render] No TTS backend. Install: pip install edge-tts")
+        return False
+
+    # Deduplicate and synthesize unique texts
+    unique_texts = list(set(t for _, t in voice_events))
+    clip_map = {}
+    for i, text in enumerate(unique_texts):
+        clip_path = os.path.join(tmpdir, f"clip_{i}.wav")
+        _synth(backend, text, clip_path)
+        if os.path.exists(clip_path) and os.path.getsize(clip_path) > 100:
+            clip_map[text] = clip_path
+
+    if not clip_map:
+        return False
+
+    # Build a concat file: silence gaps + voice clips at correct timestamps
+    segments = []
+    cursor = 0.0
+
+    for ts, text in sorted(voice_events, key=lambda x: x[0]):
+        if text not in clip_map:
+            continue
+        # Add silence gap before this clip
+        gap = ts - cursor
+        if gap > 0.05:
+            silence_path = os.path.join(tmpdir, f"silence_{len(segments)}.wav")
+            os.system(f'ffmpeg -y -f lavfi -i anullsrc=r=22050:cl=mono -t {gap:.3f} {silence_path} -loglevel error')
+            if os.path.exists(silence_path):
+                segments.append(silence_path)
+                cursor = ts
+
+        # Get clip duration
+        dur_str = os.popen(
+            f'ffprobe -i {clip_map[text]} -show_entries format=duration -v error -of csv=p=0'
+        ).read().strip()
+        clip_dur = float(dur_str) if dur_str else 2.0
+
+        segments.append(clip_map[text])
+        cursor = ts + clip_dur
+
+    # Add trailing silence to match video duration
+    if cursor < total_duration:
+        trail = os.path.join(tmpdir, "silence_trail.wav")
+        os.system(f'ffmpeg -y -f lavfi -i anullsrc=r=22050:cl=mono -t {total_duration - cursor:.3f} {trail} -loglevel error')
+        if os.path.exists(trail):
+            segments.append(trail)
+
+    if not segments:
+        return False
+
+    # Write concat list
+    concat_file = os.path.join(tmpdir, "concat.txt")
+    with open(concat_file, 'w') as f:
+        for seg in segments:
+            f.write(f"file '{seg}'\n")
+
+    # Concatenate all segments
+    os.system(f'ffmpeg -y -f concat -safe 0 -i {concat_file} -c:a pcm_s16le -ar 22050 -ac 1 {output_audio_path} -loglevel error')
+
+    # Cleanup
+    for f_path in os.listdir(tmpdir):
+        try:
+            os.remove(os.path.join(tmpdir, f_path))
+        except Exception:
+            pass
+    try:
+        os.rmdir(tmpdir)
+    except Exception:
+        pass
+
+    return os.path.exists(output_audio_path) and os.path.getsize(output_audio_path) > 100
+
+    # Cleanup
+    for f in os.listdir(tmpdir):
+        os.remove(os.path.join(tmpdir, f))
+    os.rmdir(tmpdir)
+
+    return os.path.exists(output_audio_path)
+
+
+def merge_voice_into_video(video_path, voice_events, total_duration):
+    """Add TTS voice track to an existing video file. Returns new path."""
+    tmpdir = tempfile.gettempdir()
+    voice_track = os.path.join(tmpdir, "nav_voice_track.wav")
+
+    if not render_voice_track(voice_events, voice_track, total_duration):
+        return video_path  # fallback: return original
+
+    output = video_path.replace('.mp4', '_voiced.mp4')
+
+    # Check if video already has audio
+    has_audio = os.popen(
+        f'ffprobe -i {video_path} -show_streams -select_streams a -loglevel error 2>&1'
+    ).read().strip()
+
+    if has_audio:
+        # Mix TTS with existing audio, use longest duration
+        os.system(
+            f'ffmpeg -y -i {video_path} -i {voice_track} '
+            f'-filter_complex "[0:a][1:a]amix=inputs=2:duration=longest:dropout_transition=0[a]" '
+            f'-map 0:v -map "[a]" -c:v copy -c:a aac -shortest '
+            f'{output} -loglevel error'
+        )
+    else:
+        # Add TTS as the only audio
+        os.system(
+            f'ffmpeg -y -i {video_path} -i {voice_track} '
+            f'-map 0:v -map 1:a -c:v copy -c:a aac -shortest '
+            f'{output} -loglevel error'
+        )
+
+    if os.path.exists(output) and os.path.getsize(output) > 0:
+        return output
+    return video_path
+
+
+def _detect_backend():
+    # Prefer edge-tts — natural assistant voice
+    try:
+        import edge_tts
+        return "edge_tts"
+    except ImportError:
+        pass
+    if os.system("which espeak > /dev/null 2>&1") == 0:
+        return "espeak"
+    try:
+        import pyttsx3
+        return "pyttsx3"
+    except ImportError:
+        pass
+    return None
+
+
+def _synth(backend, text, out_path):
+    """Synthesize text to a .wav file."""
+    try:
+        if backend == "edge_tts":
+            import asyncio, edge_tts
+            mp3 = out_path.replace('.wav', '.mp3')
+            async def _gen():
+                # en-US-GuyNeural: clear male assistant voice
+                # rate=+15% for snappy navigation feel
+                c = edge_tts.Communicate(text, "en-US-GuyNeural", rate="+15%")
+                await c.save(mp3)
+            asyncio.run(_gen())
+            os.system(f'ffmpeg -y -i {mp3} -ar 22050 -ac 1 {out_path} -loglevel error')
+            if os.path.exists(mp3):
+                os.remove(mp3)
+        elif backend == "espeak":
+            safe = text.replace('"', '\\"').replace("'", "\\'")
+            os.system(f'espeak -s 170 -w {out_path} "{safe}" 2>/dev/null')
+        elif backend == "pyttsx3":
+            import pyttsx3
+            engine = pyttsx3.init()
+            engine.setProperty('rate', 170)
+            engine.save_to_filename(out_path)
+            engine.say(text)
+            engine.runAndWait()
+    except Exception as e:
+        print(f"[TTS-Render] Synth error: {e}")

deploy_ec2.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Deploy Blind Navigation App to EC2
+# Run this ON the EC2 instance after uploading blind_nav_app.tar.gz
+
+set -e
+
+echo "=== Installing system dependencies ==="
+sudo apt-get update
+sudo apt-get install -y python3-pip python3-venv ffmpeg espeak libgl1-mesa-glx
+
+echo "=== Setting up app ==="
+mkdir -p ~/blind_nav && cd ~/blind_nav
+tar -xzf ~/blind_nav_app.tar.gz
+
+echo "=== Creating virtual environment ==="
+python3 -m venv venv
+source venv/bin/activate
+
+echo "=== Installing Python packages ==="
+pip install --upgrade pip
+pip install -r requirements.txt
+
+echo "=== Done! ==="
+echo ""
+echo "To run:"
+echo "  cd ~/blind_nav"
+echo "  source venv/bin/activate"
+echo "  python app_live.py"
+echo ""
+echo "App will be at: http://<your-ec2-public-ip>:7860"
+echo "Make sure port 7860 is open in your EC2 security group!"

pipeline.py

@@ -0,0 +1,298 @@
+"""Unified video/image/webcam pipeline using core modules."""
+import cv2
+import numpy as np
+import os
+import time
+import tempfile
+
+from config import DEFAULT_SKIP_FRAMES, DEFAULT_DEPTH_EVERY
+from core import detector, depth, pose, risk_engine
+from core.detector import ObstacleTracker
+from core.depth import SlopeSmoother
+from core.guidance import GuidanceEngine
+from core.tts import TTSEngine
+from renderers import overlay as overlay_renderer
+from renderers import blind_nav as blind_nav_renderer
+
+
+def process_image(image_bgr, mode="overlay"):
+    """Process a single image. Returns (rendered_bgr, risk_dict, guidance_dict)."""
+    h, w = image_bgr.shape[:2]
+    rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
+
+    obstacles = detector.detect(image_bgr)
+    gait, landmarks, foot_y = pose.analyze(rgb, w, h)
+    depth_norm = depth.estimate_depth(rgb, h, w)
+    slope_angle, slope_dir, terrain, gs = depth.estimate_slope(depth_norm, h, w, foot_y)
+    risk = risk_engine.assess(slope_angle, slope_dir, gait, len(obstacles))
+
+    guide_engine = GuidanceEngine()
+    guidance = guide_engine.compute(slope_angle, slope_dir, terrain, obstacles)
+
+    if mode == "blind_nav":
+        rendered = blind_nav_renderer.render(
+            image_bgr, obstacles, slope_angle, slope_dir, terrain, depth_norm, guidance)
+    else:
+        depth_mini = overlay_renderer.render_depth_mini(depth_norm, w, h)
+        rendered = overlay_renderer.render(
+            image_bgr, obstacles, gait, landmarks, risk, guidance, depth_mini)
+
+    return rendered, risk, guidance
+
+
+def process_video(video_path, mode="overlay", skip_frames=DEFAULT_SKIP_FRAMES,
+                  depth_every=DEFAULT_DEPTH_EVERY, track=True, tts=False):
+    """Process video file. Returns output video path."""
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        return None
+
+    w, h = int(cap.get(3)), int(cap.get(4))
+    fps = cap.get(5)
+    total = int(cap.get(7))
+    out_fps = fps / (skip_frames + 1)
+    original_path = video_path  # keep for audio mux later
+
+    if mode == "blind_nav":
+        out_size = (w * 2, h + 70)
+    else:
+        out_size = (w, h)
+
+    out_path = os.path.join(tempfile.gettempdir(), f"nav_{mode}.mp4")
+    writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*'mp4v'), out_fps, out_size)
+
+    # Stateful components
+    guide_engine = GuidanceEngine()
+    slope_smoother = SlopeSmoother()
+    obs_tracker = ObstacleTracker()
+    tts_engine = TTSEngine(enabled=tts)
+    voice_events = []  # (timestamp, text) for offline TTS rendering
+
+    cached_depth = np.zeros((h, w), dtype=np.float32)
+    cached_depth_mini = None
+
+    frame_idx = processed = 0
+    t0 = time.time()
+    print(f"Processing {total} frames ({w}x{h}) mode={mode}...", flush=True)
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+        frame_idx += 1
+        if frame_idx % (skip_frames + 1) != 0:
+            continue
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        # Obstacles with tracking
+        obstacles = detector.detect(frame, track=track)
+        new_obs, closing_obs, lost_ids = obs_tracker.update(obstacles)
+
+        # Pose
+        gait, landmarks, foot_y = pose.analyze(rgb, w, h)
+
+        # Depth (every N frames)
+        if frame_idx % (depth_every * (skip_frames + 1)) == 0 or frame_idx <= skip_frames + 1:
+            cached_depth = depth.estimate_depth(rgb, h, w)
+            raw_slope, raw_dir, raw_terrain, gs = depth.estimate_slope(
+                cached_depth, h, w, foot_y)
+            slope_smoother.update(raw_slope, raw_dir, raw_terrain)
+            cached_depth_mini = overlay_renderer.render_depth_mini(cached_depth, w, h)
+
+        # Use smoothed values
+        s_angle = slope_smoother.angle
+        s_dir = slope_smoother.direction
+        s_terrain = slope_smoother.terrain
+        s_trend = slope_smoother.trend
+
+        # Risk + guidance
+        risk = risk_engine.assess(
+            s_angle, s_dir, gait, obstacles,
+            slope_trend=s_trend,
+            new_obstacles=new_obs,
+            closing_obstacles=closing_obs)
+        guidance = guide_engine.compute(
+            s_angle, s_dir, s_terrain, obstacles,
+            slope_trend=s_trend,
+            new_obstacles=new_obs,
+            closing_obstacles=closing_obs)
+
+        # Render
+        if mode == "blind_nav":
+            rendered = blind_nav_renderer.render(
+                frame, obstacles, s_angle, s_dir, s_terrain,
+                cached_depth, guidance)
+        else:
+            rendered = overlay_renderer.render(
+                frame, obstacles, gait, landmarks, risk, guidance, cached_depth_mini)
+
+        writer.write(rendered)
+        processed += 1
+
+        # TTS: speak if throttle allows
+        if guide_engine.should_speak(guidance['voice'], s_angle):
+            timestamp = frame_idx / fps
+            voice_events.append((timestamp, guidance['voice']))
+            if tts_engine.enabled:
+                tts_engine.speak(guidance['voice'])
+
+        if processed % 20 == 0:
+            el = time.time() - t0
+            new_str = f" new={len(new_obs)}" if new_obs else ""
+            close_str = f" closing={len(closing_obs)}" if closing_obs else ""
+            print(f"  {frame_idx}/{total} | {processed / el:.1f}fps | "
+                  f"{s_dir} {s_angle:.0f}° [{s_trend}] | {s_terrain} | "
+                  f"obs={len(obstacles)}{new_str}{close_str} | risk={risk['risk']}",
+                  flush=True)
+
+    cap.release()
+    writer.release()
+    tts_engine.shutdown()
+
+    # Re-encode to H.264 and mux audio from original
+    h264 = out_path.replace('.mp4', '_h264.mp4')
+    # First: encode video to H.264
+    os.system(f'ffmpeg -y -i {out_path} -c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p {h264} -loglevel error')
+    if os.path.exists(h264) and os.path.getsize(h264) > 0:
+        os.remove(out_path)
+        out_path = h264
+
+    # Mux original audio (tempo-adjusted for frame skipping)
+    if original_path and os.path.exists(original_path):
+        with_audio = out_path.replace('.mp4', '_audio.mp4')
+        tempo = skip_frames + 1  # audio needs to speed up to match skipped video
+        os.system(
+            f'ffmpeg -y -i {out_path} -i {original_path} '
+            f'-filter_complex "[1:a]atempo={tempo}[a]" '
+            f'-map 0:v -map "[a]" -c:v copy -c:a aac -shortest '
+            f'{with_audio} -loglevel error'
+        )
+        if os.path.exists(with_audio) and os.path.getsize(with_audio) > 0:
+            os.remove(out_path)
+            out_path = with_audio
+
+    print(f"Done! {processed} frames in {time.time() - t0:.1f}s", flush=True)
+
+    # Bake TTS voice into video if we have events
+    if voice_events:
+        from core.tts_render import merge_voice_into_video
+        duration = total / fps
+        print(f"Rendering {len(voice_events)} voice events into video...", flush=True)
+        out_path = merge_voice_into_video(out_path, voice_events, duration)
+
+    return out_path
+
+
+def run_webcam(mode="overlay", camera_id=0, tts=True):
+    """Live webcam/RTSP processing loop. Press 'q' to quit.
+
+    Args:
+        camera_id: 0 for USB webcam, or "rtsp://..." for IP camera
+    """
+    from core.camera import CameraStream
+
+    try:
+        cam = CameraStream(camera_id)
+    except RuntimeError as e:
+        print(f"Error: {e}")
+        return
+
+    guide_engine = GuidanceEngine()
+    slope_smoother = SlopeSmoother()
+    obs_tracker = ObstacleTracker()
+    tts_engine = TTSEngine(enabled=tts)
+    cached_depth = None
+    cached_depth_mini = None
+    depth_counter = 0
+    frame_count = 0
+    t0 = time.time()
+
+    print(f"Live mode={mode} tts={tts}. Press 'q' to quit.", flush=True)
+    while cam.is_open:
+        frame = cam.read()
+        if frame is None:
+            time.sleep(0.01)
+            continue
+
+        h, w = frame.shape[:2]
+        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        obstacles = detector.detect(frame, track=True)
+        new_obs, closing_obs, _ = obs_tracker.update(obstacles)
+        gait, landmarks, foot_y = pose.analyze(rgb, w, h)
+
+        depth_counter += 1
+        if depth_counter % DEFAULT_DEPTH_EVERY == 0 or cached_depth is None:
+            cached_depth = depth.estimate_depth(rgb, h, w)
+            raw_slope, raw_dir, raw_terrain, _ = depth.estimate_slope(cached_depth, h, w, foot_y)
+            slope_smoother.update(raw_slope, raw_dir, raw_terrain)
+            cached_depth_mini = overlay_renderer.render_depth_mini(cached_depth, w, h)
+
+        risk = risk_engine.assess(
+            slope_smoother.angle, slope_smoother.direction, gait, obstacles,
+            slope_trend=slope_smoother.trend,
+            new_obstacles=new_obs,
+            closing_obstacles=closing_obs)
+        guidance = guide_engine.compute(
+            slope_smoother.angle, slope_smoother.direction, slope_smoother.terrain,
+            obstacles, slope_trend=slope_smoother.trend,
+            new_obstacles=new_obs, closing_obstacles=closing_obs)
+
+        if mode == "blind_nav":
+            rendered = blind_nav_renderer.render(
+                frame, obstacles, slope_smoother.angle, slope_smoother.direction,
+                slope_smoother.terrain, cached_depth if cached_depth is not None
+                else np.zeros((h, w), dtype=np.float32), guidance)
+        else:
+            rendered = overlay_renderer.render(
+                frame, obstacles, gait, landmarks, risk, guidance, cached_depth_mini)
+
+        if tts_engine.enabled and guide_engine.should_speak(guidance['voice'], slope_smoother.angle):
+            tts_engine.speak(guidance['voice'])
+
+        frame_count += 1
+        if frame_count % 30 == 0:
+            fps = frame_count / (time.time() - t0)
+            cv2.setWindowTitle("Navigation", f"Navigation | {fps:.1f} FPS | {risk['risk']}")
+
+        cv2.imshow("Navigation", rendered)
+        key = cv2.waitKey(1) & 0xFF
+        if key == ord('q'):
+            break
+
+    cam.release()
+    cv2.destroyAllWindows()
+    tts_engine.shutdown()
+    print(f"Session: {frame_count} frames in {time.time() - t0:.1f}s", flush=True)
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage:")
+        print("  python pipeline.py <video.mp4> [--mode overlay|blind_nav] [--tts]")
+        print("  python pipeline.py --webcam [--cam 0] [--mode overlay|blind_nav] [--tts]")
+        print("  python pipeline.py --webcam --cam rtsp://user:pass@ip:554/stream [--tts]")
+        sys.exit(1)
+
+    mode = "overlay"
+    if "--mode" in sys.argv:
+        idx = sys.argv.index("--mode")
+        mode = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else "overlay"
+
+    use_tts = "--tts" in sys.argv
+
+    cam_source = 0
+    if "--cam" in sys.argv:
+        idx = sys.argv.index("--cam")
+        val = sys.argv[idx + 1] if idx + 1 < len(sys.argv) else "0"
+        cam_source = val if val.startswith("rtsp") else int(val)
+
+    if sys.argv[1] == "--webcam":
+        run_webcam(mode=mode, camera_id=cam_source, tts=use_tts)
+    else:
+        out = process_video(sys.argv[1], mode=mode, tts=use_tts)
+        if out:
+            print(f"Output: {out}")

pose_landmarker_heavy.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64437af838a65d18e5ba7a0d39b465540069bc8aae8308de3e318aad31fcbc7b
+size 30664242

renderers/__init__.py

@@ -0,0 +1 @@
+

renderers/blind_nav.py

@@ -0,0 +1,96 @@
+"""2-panel blind navigation renderer (live feed + terrain map)."""
+import cv2
+import numpy as np
+import math
+from config import LM
+
+
+def render(frame, obstacles, slope_angle, slope_dir, terrain, depth_norm, guidance):
+    """Render 2-panel layout: left=live+obstacles, right=depth+slope, bottom=voice bar."""
+    h, w = frame.shape[:2]
+    bar_h = 70
+    canvas = np.zeros((h + bar_h, w * 2, 3), dtype=np.uint8)
+
+    # Left panel: live feed + obstacle highlights
+    left = frame.copy()
+    for ob in obstacles:
+        x1, y1, x2, y2 = ob['box']
+        u = ob['proximity']
+        color = (0, 0, 180) if u > 0.7 else (0, 160, 200) if u > 0.4 else (0, 160, 0)
+        overlay = left.copy()
+        cv2.rectangle(overlay, (x1, y1), (x2, y2), color, -1)
+        left = cv2.addWeighted(overlay, 0.35 if u > 0.7 else 0.2, left, 0.65 if u > 0.7 else 0.8, 0)
+        cv2.rectangle(left, (x1, y1), (x2, y2), color, 3)
+        # Label with background
+        label = f"{ob['label']} {ob['dist']}"
+        (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.55, 2)
+        cv2.rectangle(left, (x1, y1 - th - 8), (x1 + tw + 4, y1), color, -1)
+        cv2.putText(left, label, (x1 + 2, y1 - 5),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2)
+        # Direction arrow
+        cx, cy = ob['center']
+        if ob['direction'] != "CENTER":
+            opp_x = cx - 50 if ob['direction'] == "LEFT" else cx + 50
+            cv2.arrowedLine(left, (cx, cy), (opp_x, cy), (255, 255, 255), 3, tipLength=0.4)
+        else:
+            cv2.arrowedLine(left, (cx, cy - 40), (cx, cy + 40), (0, 0, 255), 3, tipLength=0.4)
+
+    oc = (0, 255, 0) if len(obstacles) == 0 else (0, 200, 255) if len(obstacles) <= 2 else (0, 0, 255)
+    cv2.rectangle(left, (0, 0), (w, 28), (0, 0, 0), -1)
+    cv2.putText(left, f"LIVE FEED  |  Obstacles: {len(obstacles)}", (8, 20),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.5, oc, 1)
+    canvas[:h, :w] = left
+
+    # Right panel: depth + slope
+    depth_color = cv2.applyColorMap((depth_norm * 255).astype(np.uint8), cv2.COLORMAP_INFERNO)
+    right = depth_color.copy()
+    sa = abs(slope_angle)
+    slope_tint = (0, 0, 200) if sa > 15 else (0, 180, 220) if sa > 5 else (0, 180, 0)
+
+    gs = int(h * 0.55)
+    overlay = right.copy()
+    cv2.rectangle(overlay, (0, gs), (w, h), slope_tint, -1)
+    right = cv2.addWeighted(overlay, 0.25, right, 0.75, 0)
+    cv2.line(right, (0, gs), (w, gs), (0, 255, 0), 1)
+
+    # Slope arrow
+    acx, acy = w // 2, (gs + h) // 2
+    arad = math.radians(slope_angle)
+    ax = int(acx + math.cos(arad) * 50)
+    ay = int(acy - math.sin(arad) * 50)
+    cv2.arrowedLine(right, (acx, acy), (ax, ay), (0, 255, 0), 3, tipLength=0.3)
+
+    cv2.rectangle(right, (0, 0), (w, 80), (0, 0, 0), -1)
+    cv2.putText(right, f"TERRAIN: {slope_dir} {slope_angle:.1f} deg", (8, 20),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.5, slope_tint, 2)
+    cv2.putText(right, f"Surface: {terrain}", (8, 42),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.45, (200, 200, 200), 1)
+
+    if sa > 3:
+        cv2.putText(right, f"Foot adjust: {guidance['foot_adj']:+.0f} deg  |  Knee: {guidance['knee_rec']} deg",
+                    (8, 65), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 255), 1)
+    else:
+        cv2.putText(right, "Foot: level  |  Knee: normal",
+                    (8, 65), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 200, 0), 1)
+    canvas[:h, w:] = right
+
+    # Bottom voice bar
+    cv2.rectangle(canvas, (0, h), (w * 2, h + bar_h), (25, 25, 35), -1)
+    voice = guidance['voice']
+    cv2.putText(canvas, "VOICE:", (10, h + 22), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (100, 180, 255), 1)
+    cv2.putText(canvas, voice[:100], (75, h + 22), cv2.FONT_HERSHEY_SIMPLEX, 0.42, (255, 255, 255), 1)
+
+    step = guidance['step']
+    sc = (0, 255, 0) if step == "NORMAL" else (0, 200, 255) if step == "MEDIUM" else (0, 100, 255)
+    cv2.putText(canvas, f"Step: {step}", (w * 2 - 300, h + 22),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.5, sc, 2)
+
+    risk = guidance['risk_score']
+    rc = (0, 200, 0) if risk < 30 else (0, 200, 200) if risk < 60 else (0, 0, 255)
+    bar_x = w * 2 - 130
+    cv2.putText(canvas, "Risk:", (bar_x, h + 22), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (180, 180, 180), 1)
+    cv2.rectangle(canvas, (bar_x + 40, h + 10), (bar_x + 120, h + 25), (60, 60, 60), -1)
+    fill = int(80 * min(1.0, risk / 100.0))
+    cv2.rectangle(canvas, (bar_x + 40, h + 10), (bar_x + 40 + fill, h + 25), rc, -1)
+
+    return canvas

renderers/overlay.py

@@ -0,0 +1,110 @@
+"""Simple HUD overlay renderer — single frame with annotations."""
+import cv2
+import math
+import numpy as np
+from config import LM, SKELETON_CONNS
+
+
+def draw_obstacles(frame, obstacles):
+    h, w = frame.shape[:2]
+    for ob in obstacles:
+        x1, y1, x2, y2 = ob['box']
+        u = ob['proximity']
+        color = (0, int(255 * (1 - u)), int(255 * u))
+
+        # Semi-transparent fill
+        overlay = frame.copy()
+        cv2.rectangle(overlay, (x1, y1), (x2, y2), color, -1)
+        alpha = 0.35 if u > 0.7 else 0.2
+        cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0, frame)
+
+        # Thick border
+        cv2.rectangle(frame, (x1, y1), (x2, y2), color, 3)
+
+        # Label with background
+        label = f"{ob['label']} {ob['dist']}"
+        if 'track_id' in ob:
+            label = f"#{ob['track_id']} {label}"
+        (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.55, 2)
+        cv2.rectangle(frame, (x1, y1 - th - 8), (x1 + tw + 4, y1), color, -1)
+        cv2.putText(frame, label, (x1 + 2, y1 - 5),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2)
+
+        # Direction arrow
+        cx, cy = ob['center']
+        if ob['direction'] == "LEFT":
+            cv2.arrowedLine(frame, (cx + 40, cy), (cx - 40, cy), (255, 255, 255), 3, tipLength=0.4)
+        elif ob['direction'] == "RIGHT":
+            cv2.arrowedLine(frame, (cx - 40, cy), (cx + 40, cy), (255, 255, 255), 3, tipLength=0.4)
+        else:
+            cv2.arrowedLine(frame, (cx, cy - 40), (cx, cy + 40), (0, 0, 255), 3, tipLength=0.4)
+
+
+def draw_skeleton(frame, landmarks, gait, w, h):
+    if not landmarks:
+        return
+    for i, j in SKELETON_CONNS:
+        p1 = (int(landmarks[i].x * w), int(landmarks[i].y * h))
+        p2 = (int(landmarks[j].x * w), int(landmarks[j].y * h))
+        cv2.line(frame, p1, p2, (0, 255, 0), 2)
+    for i in LM.values():
+        cv2.circle(frame, (int(landmarks[i].x * w), int(landmarks[i].y * h)), 4, (0, 0, 255), -1)
+    if gait:
+        for side, pfx in [('L', 'L_'), ('R', 'R_')]:
+            ki = LM[f'{pfx}KNEE']
+            kx, ky = int(landmarks[ki].x * w), int(landmarks[ki].y * h)
+            c = (0, 255, 255) if side == 'L' else (255, 255, 0)
+            cv2.putText(frame, f"{gait[f'{side}_knee']:.0f}°", (kx + 5, ky - 5),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.35, c, 1)
+
+
+def draw_hud(frame, risk_result, guidance):
+    h, w = frame.shape[:2]
+    cv2.rectangle(frame, (0, 0), (w, 70), (0, 0, 0), -1)
+
+    risk = risk_result['risk']
+    rc = {'SAFE': (0, 180, 0), 'LOW': (0, 220, 0), 'MEDIUM': (0, 180, 220), 'HIGH': (0, 0, 220)}[risk]
+    cv2.rectangle(frame, (5, 5), (170, 35), rc, -1)
+    cv2.putText(frame, f"RISK: {risk} ({risk_result['score']})", (10, 28),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2)
+
+    slope_text = f"Slope: {risk_result['terrain']} {risk_result['terrain_slope']:.1f}°"
+    if guidance and guidance.get('slope_trend', 'STABLE') != 'STABLE':
+        slope_text += f" [{guidance['slope_trend']}]"
+    cv2.putText(frame, slope_text, (180, 28), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (200, 200, 200), 1)
+
+    gs = risk_result.get('gait_summary', {})
+    if gs:
+        cv2.putText(frame, f"Knee:{gs.get('knee', 0):.0f} Shin:{gs.get('shin', 0):.0f} "
+                    f"Lean:{gs.get('lean', 0):.0f} Sym:{gs.get('symmetry', 0):.0f}",
+                    (5, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.38, (180, 180, 180), 1)
+
+    # Risk component breakdown
+    comps = risk_result.get('components', {})
+    if comps:
+        parts = [f"{k[0].upper()}:{v}" for k, v in comps.items() if v > 0]
+        if parts:
+            cv2.putText(frame, " ".join(parts), (w - 200, 50),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.3, rc, 1)
+
+    if guidance:
+        cv2.putText(frame, guidance['voice'][:80], (5, 68),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.3, (100, 200, 255), 1)
+
+
+def render_depth_mini(depth_norm, w, h):
+    dc = cv2.applyColorMap((depth_norm * 255).astype(np.uint8), cv2.COLORMAP_INFERNO)
+    return cv2.resize(dc, (w // 4, h // 4))
+
+
+def render(frame, obstacles, gait, landmarks, risk_result, guidance, depth_mini=None):
+    """Render single-frame overlay with HUD, skeleton, obstacles, and optional depth minimap."""
+    h, w = frame.shape[:2]
+    out = frame.copy()
+    draw_obstacles(out, obstacles)
+    draw_skeleton(out, landmarks, gait, w, h)
+    draw_hud(out, risk_result, guidance)
+    if depth_mini is not None:
+        mh, mw = depth_mini.shape[:2]
+        out[h - mh - 5:h - 5, w - mw - 5:w - 5] = depth_mini
+    return out

requirements.txt

@@ -0,0 +1,10 @@
+ultralytics>=8.0
+transformers>=4.30
+torch>=2.0
+mediapipe>=0.10
+opencv-python-headless>=4.8
+gradio>=4.0
+numpy>=1.24
+Pillow>=9.0
+edge-tts>=6.0
+lapx>=0.5

yolov8n.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f59b3d833e2ff32e194b5bb8e08d211dc7c5bdf144b90d2c8412c47ccfc83b36
+size 6549796