From b846cc4e6d2fbce57cfff19b29114b472e603cad Mon Sep 17 00:00:00 2001 From: AadarshJena Date: Tue, 20 Oct 2026 20:04:23 -0400 Subject: [PATCH 1/2] V3: extract FaceTracker into GraptionVision, stable f1/f2 IDs, largest-face speaker guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move tracking (OneEuroFilter, TrackedFace, matching) out of CameraView into a new GraptionVision package with unit tests. - Match by IoU >= 0.3 first, then center distance; keep a track 1 s unseen, show it for 2 missed frames; IDs are f1, f2, ... per session (schema FaceId). - Add SpeakerScorer protocol with LargestFaceScorer stub; camera view labels boxes with their ID and highlights the guessed speaker. - Link the package in the Xcode project, run its tests in iOS CI, update §6. Closes #10 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ios-ci.yml | 3 + apps/ios/Graption.xcodeproj/project.pbxproj | 12 + apps/ios/Graption/CameraView.swift | 217 ++++------------- apps/ios/Packages/GraptionVision/.gitignore | 8 + .../ios/Packages/GraptionVision/Package.swift | 24 ++ .../Sources/GraptionVision/FaceTracker.swift | 218 ++++++++++++++++++ .../GraptionVision/OneEuroFilter.swift | 58 +++++ .../GraptionVision/SpeakerScorer.swift | 43 ++++ .../FaceTrackerTests.swift | 122 ++++++++++ .../SpeakerScorerTests.swift | 33 +++ docs/GRAPTION_CONTEXT.md | 10 +- 11 files changed, 567 insertions(+), 181 deletions(-) create mode 100644 apps/ios/Packages/GraptionVision/.gitignore create mode 100644 apps/ios/Packages/GraptionVision/Package.swift create mode 100644 apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift create mode 100644 apps/ios/Packages/GraptionVision/Sources/GraptionVision/OneEuroFilter.swift create mode 100644 apps/ios/Packages/GraptionVision/Sources/GraptionVision/SpeakerScorer.swift create mode 100644 apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift create mode 100644 apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/SpeakerScorerTests.swift diff --git a/.github/workflows/ios-ci.yml b/.github/workflows/ios-ci.yml index 2a5a293..ebdf09c 100644 --- a/.github/workflows/ios-ci.yml +++ b/.github/workflows/ios-ci.yml @@ -29,4 +29,7 @@ jobs: CODE_SIGNING_ALLOWED=NO \ build + - name: Test GraptionVision + run: swift test --package-path apps/ios/Packages/GraptionVision + #mac-os latest or xcode-27 for the mac server to host and run the Github Actions Server on? diff --git a/apps/ios/Graption.xcodeproj/project.pbxproj b/apps/ios/Graption.xcodeproj/project.pbxproj index d75ace9..dff897c 100644 --- a/apps/ios/Graption.xcodeproj/project.pbxproj +++ b/apps/ios/Graption.xcodeproj/project.pbxproj @@ -11,6 +11,7 @@ 5A863841E58064FDABEE6763 /* GraptionUI in Frameworks */ = {isa = PBXBuildFile; productRef = 1504F8340E9DC8A84ACE6184 /* GraptionUI */; }; 19DABF99307713050033A27A /* MediaPipeTasksAudio in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF98307713050033A27A /* MediaPipeTasksAudio */; }; 19DABF9B307713050033A27A /* MediaPipeTasksVision in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF9A307713050033A27A /* MediaPipeTasksVision */; }; + 19DABF9E307713050033A27A /* GraptionVision in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF9D307713050033A27A /* GraptionVision */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -47,6 +48,7 @@ 5A863841E58064FDABEE6763 /* GraptionUI in Frameworks */, 19DABF9B307713050033A27A /* MediaPipeTasksVision in Frameworks */, 19DABF99307713050033A27A /* MediaPipeTasksAudio in Frameworks */, + 19DABF9E307713050033A27A /* GraptionVision in Frameworks */, ); }; /* End PBXFrameworksBuildPhase section */ @@ -112,6 +114,7 @@ 1504F8340E9DC8A84ACE6184 /* GraptionUI */, 19DABF98307713050033A27A /* MediaPipeTasksAudio */, 19DABF9A307713050033A27A /* MediaPipeTasksVision */, + 19DABF9D307713050033A27A /* GraptionVision */, ); productName = Graption; productReference = 9835CFA1306C55A900F6C1EC /* Graption.app */; @@ -145,6 +148,7 @@ 65CCECAE3074877500093FF3 /* XCLocalSwiftPackageReference "Packages/GraptionAudio" */, 6CF0856E124649A4FFEEDDAC /* XCLocalSwiftPackageReference "Packages/GraptionUI" */, 19DABF97307713050033A27A /* XCRemoteSwiftPackageReference "mediapipe" */, + 19DABF9C307713050033A27A /* XCLocalSwiftPackageReference "Packages/GraptionVision" */, ); preferredProjectObjectVersion = 77; productRefGroup = 9835CFA2306C55A900F6C1EC /* Products */; @@ -396,6 +400,10 @@ isa = XCLocalSwiftPackageReference; relativePath = Packages/GraptionUI; }; + 19DABF9C307713050033A27A /* XCLocalSwiftPackageReference "Packages/GraptionVision" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = Packages/GraptionVision; + }; /* End XCLocalSwiftPackageReference section */ /* Begin XCRemoteSwiftPackageReference section */ @@ -429,6 +437,10 @@ package = 19DABF97307713050033A27A /* XCRemoteSwiftPackageReference "mediapipe" */; productName = MediaPipeTasksVision; }; + 19DABF9D307713050033A27A /* GraptionVision */ = { + isa = XCSwiftPackageProductDependency; + productName = GraptionVision; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 9835CF99306C55A900F6C1EC /* Project object */; diff --git a/apps/ios/Graption/CameraView.swift b/apps/ios/Graption/CameraView.swift index 017b091..96b66c1 100644 --- a/apps/ios/Graption/CameraView.swift +++ b/apps/ios/Graption/CameraView.swift @@ -12,128 +12,7 @@ import Combine import UIKit import Vision import MediaPipeTasksVision - -nonisolated private struct OneEuroFilter { - private let minCutoff: Double - private let beta: Double - private let derivativeCutoff: Double - private var previousValue: Double? - private var previousDerivative = 0.0 - private var previousTimestamp: Double? - - // Inputs are normalized (0...1) image coordinates, so beta is much larger than the - // pixel-space defaults from the paper; otherwise the speed term never reduces lag. - init(minCutoff: Double = 1.25, beta: Double = 8.0, derivativeCutoff: Double = 1.0) { - self.minCutoff = minCutoff - self.beta = beta - self.derivativeCutoff = derivativeCutoff - } - - mutating func filter(_ value: Double, at timestamp: Double) -> Double { - guard let previousValue, let previousTimestamp else { - self.previousValue = value - self.previousTimestamp = timestamp - return value - } - - let deltaTime = max(timestamp - previousTimestamp, 1.0 / 120.0) - let rawDerivative = (value - previousValue) / deltaTime - let derivativeAlpha = smoothingFactor(deltaTime, cutoff: derivativeCutoff) - let derivative = exponentialFilter( - rawDerivative, - previous: self.previousDerivative, - alpha: derivativeAlpha - ) - let valueAlpha = smoothingFactor(deltaTime, cutoff: minCutoff + beta * abs(derivative)) - let filteredValue = exponentialFilter(value, previous: previousValue, alpha: valueAlpha) - - self.previousValue = filteredValue - self.previousDerivative = derivative - self.previousTimestamp = timestamp - return filteredValue - } - - private func smoothingFactor(_ deltaTime: Double, cutoff: Double) -> Double { - let tau = 1.0 / (2.0 * .pi * cutoff) - return 1.0 / (1.0 + tau / deltaTime) - } - - private func exponentialFilter(_ value: Double, previous: Double, alpha: Double) -> Double { - alpha * value + (1.0 - alpha) * previous - } -} - -/// One face as published to the UI. `box` and `landmarks` are normalized to the full, -/// portrait-oriented frame with a top-left origin. `landmarks` is empty when the face was -/// detected but too small or occluded for MediaPipe to fit a mesh. -nonisolated struct FaceTrack: Identifiable, Sendable { - let id: Int - let box: CGRect - let landmarks: [SIMD3] - - var hasLandmarks: Bool { !landmarks.isEmpty } -} - -nonisolated private struct FaceMeasurement { - let box: CGRect - let landmarks: [SIMD3] -} - -nonisolated private struct TrackedFace { - let id: Int - var box: CGRect - var landmarks: [SIMD3] = [] - var velocity = CGPoint.zero - var missedFrames = 0 - var xFilter = OneEuroFilter() - var yFilter = OneEuroFilter() - var widthFilter = OneEuroFilter() - var heightFilter = OneEuroFilter() - private var lastTimestamp: Double? - - init(id: Int, measurement: FaceMeasurement, timestamp: Double) { - self.id = id - self.box = measurement.box - update(with: measurement, timestamp: timestamp) - } - - mutating func update(with measurement: FaceMeasurement, timestamp: Double) { - let filtered = CGRect( - x: xFilter.filter(measurement.box.minX, at: timestamp), - y: yFilter.filter(measurement.box.minY, at: timestamp), - width: widthFilter.filter(measurement.box.width, at: timestamp), - height: heightFilter.filter(measurement.box.height, at: timestamp) - ) - if let lastTimestamp { - let timeScale = max(timestamp - lastTimestamp, 1.0 / 120.0) - velocity = CGPoint( - x: (filtered.midX - box.midX) / timeScale, - y: (filtered.midY - box.midY) / timeScale - ) - } - box = filtered - landmarks = measurement.landmarks - missedFrames = 0 - lastTimestamp = timestamp - } - - mutating func predict(at timestamp: Double) { - let deltaTime = max(timestamp - (lastTimestamp ?? timestamp), 1.0 / 120.0) - let decay = pow(0.65, deltaTime * 15.0) - box = box.offsetBy( - dx: velocity.x * deltaTime * decay, - dy: velocity.y * deltaTime * decay - ) - velocity.x *= decay - velocity.y *= decay - // Landmarks from an older frame would no longer line up with the predicted box. - landmarks = [] - missedFrames += 1 - lastTimestamp = timestamp - } - - var output: FaceTrack { FaceTrack(id: id, box: box, landmarks: landmarks) } -} +import GraptionVision /// Where MediaPipe looks for landmarks. Detection always uses Apple Vision. nonisolated enum LandmarkMode: Sendable { @@ -162,8 +41,9 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam private static let minFaceConfidence: Float = 0.5 let session = AVCaptureSession() - /// Called on the video queue after every processed frame, with that frame's pixel size. - var onFaces: (([FaceTrack], CGSize) -> Void)? + /// called on the video queue after every frame with the tracked faces. the guessed speaker + /// id (nil if none). and the frame's pixel size. + var onFaces: (([FaceTrack], String?, CGSize) -> Void)? private var cameraPosition: AVCaptureDevice.Position = .back private var videoRotationAngle: CGFloat = 90 @@ -182,8 +62,10 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam private let sequenceHandler = VNSequenceRequestHandler() private var faceLandmarker: FaceLandmarker? private var cropPool: CVPixelBufferPool? - private var trackedFaces: [TrackedFace] = [] - private var nextTrackID = 0 + // v3. tracking lives in GraptionVision/FaceTracker.swift. + private var tracker = FaceTracker() + // largest face guess until the core ml speaker model replaces it behind SpeakerScorer. + private var speakerScorer: any SpeakerScorer = LargestFaceScorer() private var publishedEmpty = false private var lastLandmarkerTimestamp = -1 @@ -209,7 +91,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam session.stopRunning() } videoQueue.async { [self] in - trackedFaces.removeAll() + // camera stopped means session over. ids restart at f1. + tracker.reset() } } } @@ -225,7 +108,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam // Existing tracks are in the old orientation's coordinates. guard changed else { return } videoQueue.async { [self] in - trackedFaces.removeAll() + // not reset() since restarting at f1 mid session could hand an old id to someone new. + tracker.forgetTracks() } } } @@ -259,7 +143,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam session.commitConfiguration() configureCamera(for: device, fps: 15) videoQueue.async { [self] in - trackedFaces.removeAll() + // other camera means other people. new ids but numbering keeps counting. + tracker.forgetTracks() publishedEmpty = false } completion(true) @@ -351,11 +236,12 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam guard timestamp.isFinite else { return } let frameSize = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer)) - let faces = updateTracking(with: detectFaces(in: pixelBuffer, timestamp: timestamp), timestamp: timestamp) + let faces = tracker.update(with: detectFaces(in: pixelBuffer, timestamp: timestamp), at: timestamp) + let speaker = speakerScorer.scores(for: faces, at: timestamp).likelySpeaker() // Skip repeated empty updates so an empty room doesn't re-render SwiftUI at 15 fps. if faces.isEmpty && publishedEmpty { return } publishedEmpty = faces.isEmpty - onFaces?(faces, frameSize) + onFaces?(faces, speaker, frameSize) } // MARK: Detection @@ -557,51 +443,6 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam return pool } - // MARK: Tracking - - private func updateTracking(with measurements: [FaceMeasurement], timestamp: Double) -> [FaceTrack] { - // Greedy assignment over all pairs, closest first, so one face can't steal another's - // track just because it was detected earlier in the list. - var candidates: [(track: Int, measurement: Int, distance: CGFloat)] = [] - for trackIndex in trackedFaces.indices { - let track = trackedFaces[trackIndex].box - let maxDistance = max(track.width, track.height) * 1.5 - for measurementIndex in measurements.indices { - let distance = distance(from: track, to: measurements[measurementIndex].box) - if distance <= maxDistance { - candidates.append((trackIndex, measurementIndex, distance)) - } - } - } - candidates.sort { $0.distance < $1.distance } - - var unmatchedTracks = Set(trackedFaces.indices) - var unmatchedMeasurements = Set(measurements.indices) - for candidate in candidates - where unmatchedTracks.contains(candidate.track) && unmatchedMeasurements.contains(candidate.measurement) { - trackedFaces[candidate.track].update(with: measurements[candidate.measurement], timestamp: timestamp) - unmatchedTracks.remove(candidate.track) - unmatchedMeasurements.remove(candidate.measurement) - } - - for trackIndex in unmatchedTracks { - trackedFaces[trackIndex].predict(at: timestamp) - } - for measurementIndex in unmatchedMeasurements.sorted() { - trackedFaces.append( - TrackedFace(id: nextTrackID, measurement: measurements[measurementIndex], timestamp: timestamp) - ) - nextTrackID += 1 - } - trackedFaces.removeAll { $0.missedFrames > 2 || $0.box.maxX < 0 || $0.box.minX > 1 || - $0.box.maxY < 0 || $0.box.minY > 1 } - return trackedFaces.map(\.output) - } - - private func distance(from lhs: CGRect, to rhs: CGRect) -> CGFloat { - hypot(lhs.midX - rhs.midX, lhs.midY - rhs.midY) - } - // MARK: Camera /// Rotation and mirroring are set on the analyzed frames, not only the preview. @@ -659,6 +500,8 @@ final class CameraController: ObservableObject { @Published private(set) var status: Status = .notDetermined @Published private(set) var faces: [FaceTrack] = [] @Published private(set) var cameraPosition: AVCaptureDevice.Position = .back + /// id of the face drawn as the speaker. nil when no faces. + @Published private(set) var speakerID: String? /// Width / height of the analyzed frames; flips when the phone rotates. @Published private(set) var captureAspectRatio: CGFloat = 9.0 / 16.0 private var isSwitchingCamera = false @@ -669,9 +512,10 @@ final class CameraController: ObservableObject { var session: AVCaptureSession { engine.session } init() { - engine.onFaces = { [weak self] faces, frameSize in + engine.onFaces = { [weak self] faces, speakerID, frameSize in DispatchQueue.main.async { self?.faces = faces + self?.speakerID = speakerID self?.captureAspectRatio = frameSize.width / frameSize.height } } @@ -703,6 +547,7 @@ final class CameraController: ObservableObject { func stop() { engine.stop() faces = [] + speakerID = nil } func setVideoRotationAngle(_ angle: CGFloat) { @@ -720,6 +565,7 @@ final class CameraController: ObservableObject { guard success else { return } self.cameraPosition = next self.faces = [] + self.speakerID = nil } } } @@ -835,8 +681,10 @@ struct CameraView: View { ForEach(controller.faces) { face in FaceBox( + id: face.id, box: face.box, hasLandmarks: face.hasLandmarks, + isSpeaker: face.id == controller.speakerID, size: geometry.size, sourceAspectRatio: controller.captureAspectRatio ) @@ -917,9 +765,12 @@ struct CameraView: View { /// Solid when MediaPipe fit a mesh to the face; dashed when only Vision detected it /// (usually a face too small or turned away for landmarks). +/// v3. green marks the guessed speaker. the label shows the track id so id stability is visible. private struct FaceBox: View { + let id: String let box: CGRect let hasLandmarks: Bool + let isSpeaker: Bool let size: CGSize let sourceAspectRatio: CGFloat @@ -930,12 +781,22 @@ private struct FaceBox: View { let horizontalCrop = (renderedWidth - size.width) / 2 let verticalCrop = (renderedHeight - size.height) / 2 + let color: Color = isSpeaker ? .green : .yellow + Rectangle() - .stroke(.yellow, style: StrokeStyle(lineWidth: 3, dash: hasLandmarks ? [] : [8, 6])) + .stroke(color, style: StrokeStyle(lineWidth: 3, dash: hasLandmarks ? [] : [8, 6])) .frame( width: box.width * renderedWidth, height: box.height * renderedHeight ) + .overlay(alignment: .topLeading) { + Text(id) + .font(.caption.weight(.bold)) + .foregroundStyle(.black) + .padding(.horizontal, 4) + .background(color) + .offset(y: -18) + } .position( x: box.midX * renderedWidth - horizontalCrop, y: box.midY * renderedHeight - verticalCrop diff --git a/apps/ios/Packages/GraptionVision/.gitignore b/apps/ios/Packages/GraptionVision/.gitignore new file mode 100644 index 0000000..0023a53 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/apps/ios/Packages/GraptionVision/Package.swift b/apps/ios/Packages/GraptionVision/Package.swift new file mode 100644 index 0000000..edadac3 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Package.swift @@ -0,0 +1,24 @@ +// swift-tools-version: 6.0 +// v3 (#10). pure tracking logic with no camera or ui so it can be unit tested. +// tools 6.0 not 6.4 so `swift test` also runs on older xcode toolchains. + +import PackageDescription + +let package = Package( + name: "GraptionVision", + // macos so the tests run with plain `swift test` on a mac. ci runs them in ios-ci.yml. + platforms: [.iOS(.v17), .macOS(.v14)], + products: [ + .library( + name: "GraptionVision", + targets: ["GraptionVision"] + ), + ], + targets: [ + .target(name: "GraptionVision"), + .testTarget( + name: "GraptionVisionTests", + dependencies: ["GraptionVision"] + ), + ] +) diff --git a/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift new file mode 100644 index 0000000..0aef755 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift @@ -0,0 +1,218 @@ +// +// FaceTracker.swift +// GraptionVision +// +// v3 (#10). gives each face a stable per session id (f1 f2 ...). +// moved out of Graption/CameraView.swift so it can be unit tested without a camera. +// spec: GRAPTION_CONTEXT.md §6 FaceTracker. +// + +import CoreGraphics +import Foundation + +/// one detected face in one frame before tracking. +/// `box` and `landmarks` are normalized to the full portrait frame with a top left origin. +public struct FaceMeasurement: Sendable, Equatable { + public let box: CGRect + public let landmarks: [SIMD3] + + public init(box: CGRect, landmarks: [SIMD3] = []) { + self.box = box + self.landmarks = landmarks + } +} + +/// one tracked face as published to the app. +/// `landmarks` is empty when mediapipe found no mesh or the face was missed this frame. +public struct FaceTrack: Identifiable, Sendable, Equatable { + /// `f1` `f2` ... unique within a session. matches `FaceId` in schemas/events.schema.json. + public let id: String + public let box: CGRect + public let landmarks: [SIMD3] + + public var hasLandmarks: Bool { !landmarks.isEmpty } + + public init(id: String, box: CGRect, landmarks: [SIMD3] = []) { + self.id = id + self.box = box + self.landmarks = landmarks + } +} + +/// matches each frame's faces to existing tracks so ids stay put as people move. +/// +/// matching is greedy over all pairs best first so detection order can't swap ids. +/// pass 1 pairs by iou >= `minIoU`. pass 2 pairs the leftovers by center distance +/// to catch fast movers whose boxes no longer overlap. +/// +/// a track lives `maxUnseen` seconds without a match so a face that drops out briefly +/// gets its old id back. it is only published for `maxCoastFrames` missed frames +/// so stale boxes don't linger on screen. +/// +/// changes from the v2 tracker: iou pass added. ids are strings not ints. +/// tracks live 1 s instead of being dropped after 2 missed frames. +public struct FaceTracker: Sendable { + public var minIoU: CGFloat = 0.3 + /// center distance limit as a multiple of the track's larger side. + public var maxCenterDistance: CGFloat = 1.5 + /// seconds a track survives without a match before its id is retired. + public var maxUnseen: Double = 1.0 + /// missed frames during which a track is still published at its predicted spot. + public var maxCoastFrames = 2 + + private var tracks: [TrackedFace] = [] + /// next id number. only `reset()` rewinds it so ids are never reused in a session. + private var nextNumber = 1 + + public init() {} + + /// updates tracks with this frame's faces and returns the faces to show. oldest track first. + /// `timestamp` is in seconds and must not decrease between calls. + public mutating func update(with measurements: [FaceMeasurement], at timestamp: Double) -> [FaceTrack] { + var unmatchedTracks = Set(tracks.indices) + var unmatchedMeasurements = Set(measurements.indices) + + // takes pairs in the given order and skips any whose track or face is already taken. + func assign(_ candidates: [(track: Int, measurement: Int)]) { + for candidate in candidates + where unmatchedTracks.contains(candidate.track) && unmatchedMeasurements.contains(candidate.measurement) { + tracks[candidate.track].update(with: measurements[candidate.measurement], timestamp: timestamp) + unmatchedTracks.remove(candidate.track) + unmatchedMeasurements.remove(candidate.measurement) + } + } + + // pass 1. overlap. highest iou first. + var overlaps: [(track: Int, measurement: Int, iou: CGFloat)] = [] + for trackIndex in tracks.indices { + for measurementIndex in measurements.indices { + let overlap = iou(tracks[trackIndex].box, measurements[measurementIndex].box) + if overlap >= minIoU { + overlaps.append((trackIndex, measurementIndex, overlap)) + } + } + } + assign(overlaps.sorted { $0.iou > $1.iou }.map { ($0.track, $0.measurement) }) + + // pass 2. center distance for whatever is left. closest first. + var nearby: [(track: Int, measurement: Int, distance: CGFloat)] = [] + for trackIndex in unmatchedTracks { + let box = tracks[trackIndex].box + let radius = max(box.width, box.height) * maxCenterDistance + for measurementIndex in unmatchedMeasurements { + let distance = centerDistance(box, measurements[measurementIndex].box) + if distance <= radius { + nearby.append((trackIndex, measurementIndex, distance)) + } + } + } + assign(nearby.sorted { $0.distance < $1.distance }.map { ($0.track, $0.measurement) }) + + // unmatched tracks coast. unmatched faces become new tracks. + for trackIndex in unmatchedTracks { + tracks[trackIndex].predict(at: timestamp) + } + for measurementIndex in unmatchedMeasurements.sorted() { + tracks.append( + TrackedFace(id: "f\(nextNumber)", measurement: measurements[measurementIndex], timestamp: timestamp) + ) + nextNumber += 1 + } + + // retire tracks unseen too long or coasted fully out of frame. + tracks.removeAll { track in + timestamp - track.lastSeen > maxUnseen || + track.box.maxX < 0 || track.box.minX > 1 || track.box.maxY < 0 || track.box.minY > 1 + } + // still alive but missed for a while means remembered not shown. + return tracks.filter { $0.missedFrames <= maxCoastFrames }.map(\.output) + } + + /// forgets every track and restarts ids at `f1`. call when a session ends. + public mutating func reset() { + tracks.removeAll() + nextNumber = 1 + } + + /// forgets every track mid session. used after the frame rotates since old boxes no longer + /// line up. faces come back with new ids and old ids are not reused. + public mutating func forgetTracks() { + tracks.removeAll() + } +} + +/// intersection over union of two boxes. 0 when they don't touch. +func iou(_ lhs: CGRect, _ rhs: CGRect) -> CGFloat { + let intersection = lhs.intersection(rhs) + guard !intersection.isNull, !intersection.isEmpty else { return 0 } + let intersectionArea = intersection.width * intersection.height + let unionArea = lhs.width * lhs.height + rhs.width * rhs.height - intersectionArea + return unionArea > 0 ? intersectionArea / unionArea : 0 +} + +private func centerDistance(_ lhs: CGRect, _ rhs: CGRect) -> CGFloat { + hypot(lhs.midX - rhs.midX, lhs.midY - rhs.midY) +} + +/// internal state for one track. smoothing and prediction are unchanged from v2. +private struct TrackedFace: Sendable { + let id: String + var box: CGRect + var landmarks: [SIMD3] = [] + var velocity = CGPoint.zero + var missedFrames = 0 + /// last time a face was matched. drives the `maxUnseen` cutoff. + private(set) var lastSeen: Double + var xFilter = OneEuroFilter() + var yFilter = OneEuroFilter() + var widthFilter = OneEuroFilter() + var heightFilter = OneEuroFilter() + private var lastTimestamp: Double? + + init(id: String, measurement: FaceMeasurement, timestamp: Double) { + self.id = id + self.box = measurement.box + self.lastSeen = timestamp + update(with: measurement, timestamp: timestamp) + } + + /// matched this frame. smooth the box and record velocity for coasting. + mutating func update(with measurement: FaceMeasurement, timestamp: Double) { + let filtered = CGRect( + x: xFilter.filter(measurement.box.minX, at: timestamp), + y: yFilter.filter(measurement.box.minY, at: timestamp), + width: widthFilter.filter(measurement.box.width, at: timestamp), + height: heightFilter.filter(measurement.box.height, at: timestamp) + ) + if let lastTimestamp { + let timeScale = max(timestamp - lastTimestamp, 1.0 / 120.0) + velocity = CGPoint( + x: (filtered.midX - box.midX) / timeScale, + y: (filtered.midY - box.midY) / timeScale + ) + } + box = filtered + landmarks = measurement.landmarks + missedFrames = 0 + lastSeen = timestamp + lastTimestamp = timestamp + } + + /// missed this frame. drift along the last velocity which decays toward zero. + mutating func predict(at timestamp: Double) { + let deltaTime = max(timestamp - (lastTimestamp ?? timestamp), 1.0 / 120.0) + let decay = pow(0.65, deltaTime * 15.0) + box = box.offsetBy( + dx: velocity.x * deltaTime * decay, + dy: velocity.y * deltaTime * decay + ) + velocity.x *= decay + velocity.y *= decay + // landmarks from an older frame would not line up with the predicted box. + landmarks = [] + missedFrames += 1 + lastTimestamp = timestamp + } + + var output: FaceTrack { FaceTrack(id: id, box: box, landmarks: landmarks) } +} diff --git a/apps/ios/Packages/GraptionVision/Sources/GraptionVision/OneEuroFilter.swift b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/OneEuroFilter.swift new file mode 100644 index 0000000..25d87c6 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/OneEuroFilter.swift @@ -0,0 +1,58 @@ +// +// OneEuroFilter.swift +// GraptionVision +// +// smooths jittery box coordinates. moved unchanged from Graption/CameraView.swift (v2). +// + +import Foundation + +struct OneEuroFilter: Sendable { + private let minCutoff: Double + private let beta: Double + private let derivativeCutoff: Double + private var previousValue: Double? + private var previousDerivative = 0.0 + private var previousTimestamp: Double? + + // Inputs are normalized (0...1) image coordinates, so beta is much larger than the + // pixel-space defaults from the paper; otherwise the speed term never reduces lag. + init(minCutoff: Double = 1.25, beta: Double = 8.0, derivativeCutoff: Double = 1.0) { + self.minCutoff = minCutoff + self.beta = beta + self.derivativeCutoff = derivativeCutoff + } + + mutating func filter(_ value: Double, at timestamp: Double) -> Double { + guard let previousValue, let previousTimestamp else { + self.previousValue = value + self.previousTimestamp = timestamp + return value + } + + let deltaTime = max(timestamp - previousTimestamp, 1.0 / 120.0) + let rawDerivative = (value - previousValue) / deltaTime + let derivativeAlpha = smoothingFactor(deltaTime, cutoff: derivativeCutoff) + let derivative = exponentialFilter( + rawDerivative, + previous: self.previousDerivative, + alpha: derivativeAlpha + ) + let valueAlpha = smoothingFactor(deltaTime, cutoff: minCutoff + beta * abs(derivative)) + let filteredValue = exponentialFilter(value, previous: previousValue, alpha: valueAlpha) + + self.previousValue = filteredValue + self.previousDerivative = derivative + self.previousTimestamp = timestamp + return filteredValue + } + + private func smoothingFactor(_ deltaTime: Double, cutoff: Double) -> Double { + let tau = 1.0 / (2.0 * .pi * cutoff) + return 1.0 / (1.0 + tau / deltaTime) + } + + private func exponentialFilter(_ value: Double, previous: Double, alpha: Double) -> Double { + alpha * value + (1.0 - alpha) * previous + } +} diff --git a/apps/ios/Packages/GraptionVision/Sources/GraptionVision/SpeakerScorer.swift b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/SpeakerScorer.swift new file mode 100644 index 0000000..c6211f8 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/SpeakerScorer.swift @@ -0,0 +1,43 @@ +// +// SpeakerScorer.swift +// GraptionVision +// +// v3 (#10). the speaker model interface plus a stand in guess. +// the core ml model (GRAPTION_CONTEXT.md §6 SpeakerModel) will replace +// `LargestFaceScorer` behind this protocol without touching callers. +// + +import CoreGraphics + +public protocol SpeakerScorer { + /// p(speaking) from 0 to 1 for each face keyed by face id. called once per frame in order. + /// `mutating` so real models can keep per face history. + mutating func scores(for faces: [FaceTrack], at timestamp: Double) -> [String: Double] +} + +/// placeholder until the speaker model ships. the largest face gets 1 and the rest get 0. +/// ties go to the older track (first in `faces`). +public struct LargestFaceScorer: SpeakerScorer, Sendable { + public init() {} + + public func scores(for faces: [FaceTrack], at timestamp: Double) -> [String: Double] { + // strict > keeps the first face on a tie. + var largest: FaceTrack? + for face in faces where largest.map({ area(face.box) > area($0.box) }) ?? true { + largest = face + } + return Dictionary(uniqueKeysWithValues: faces.map { ($0.id, $0.id == largest?.id ? 1.0 : 0.0) }) + } + + private func area(_ box: CGRect) -> CGFloat { box.width * box.height } +} + +extension Dictionary where Key == String, Value == Double { + /// the face most likely speaking or nil if none reaches `threshold`. + /// for the live camera highlight only. caption to speaker matching is fusion's job (p7 #28). + public func likelySpeaker(threshold: Double = 0.5) -> String? { + guard let best = self.max(by: { $0.value < $1.value || ($0.value == $1.value && $0.key > $1.key) }), + best.value >= threshold else { return nil } + return best.key + } +} diff --git a/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift new file mode 100644 index 0000000..03ea5b2 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift @@ -0,0 +1,122 @@ +// v3 (#10) done when: ids don't jump around when people move. +// boxes are normalized 0 to 1 and frames are 1/15 s apart like the live camera. +// run: swift test --package-path apps/ios/Packages/GraptionVision + +import CoreGraphics +import XCTest +@testable import GraptionVision + +final class FaceTrackerTests: XCTestCase { + private let frame = 1.0 / 15.0 + + private func face(_ x: CGFloat, _ y: CGFloat, size: CGFloat = 0.2) -> FaceMeasurement { + FaceMeasurement(box: CGRect(x: x, y: y, width: size, height: size)) + } + + private func ids(_ tracks: [FaceTrack]) -> [String] { tracks.map(\.id) } + + func testNewFacesGetSequentialIDsStartingAtF1() { + var tracker = FaceTracker() + let tracks = tracker.update(with: [face(0.1, 0.1), face(0.6, 0.1), face(0.1, 0.6)], at: 0) + XCTAssertEqual(ids(tracks), ["f1", "f2", "f3"]) + } + + func testIDsFollowFacesAsTheyMove() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.4), face(0.6, 0.4)], at: 0) + + // Both walk right ~1% of the frame per frame for 3 seconds. + var tracks: [FaceTrack] = [] + for step in 1...45 { + let dx = CGFloat(step) * 0.006 + tracks = tracker.update(with: [face(0.1 + dx, 0.4), face(0.6 + dx * 0.5, 0.4)], at: Double(step) * frame) + } + XCTAssertEqual(ids(tracks), ["f1", "f2"]) + XCTAssertLessThan(tracks[0].box.minX, tracks[1].box.minX) + } + + func testDetectionOrderDoesNotSwapIDs() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.4), face(0.6, 0.4)], at: 0) + let tracks = tracker.update(with: [face(0.61, 0.4), face(0.11, 0.4)], at: frame) + let left = tracks.first { $0.box.minX < 0.5 } + let right = tracks.first { $0.box.minX > 0.5 } + XCTAssertEqual(left?.id, "f1") + XCTAssertEqual(right?.id, "f2") + } + + func testFastMoverWithoutOverlapKeepsIDViaCenterDistance() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.4)], at: 0) + // Jumps a full face width: IoU is 0, but the center is within 1.5 face sizes. + let tracks = tracker.update(with: [face(0.3, 0.4)], at: frame) + XCTAssertEqual(ids(tracks), ["f1"]) + } + + func testFaceFarAwayGetsNewID() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.0, 0.0)], at: 0) + let tracks = tracker.update(with: [face(0.7, 0.7)], at: frame) + // f1 still coasts at its old spot; the new face is f2. + XCTAssertEqual(ids(tracks), ["f1", "f2"]) + XCTAssertEqual(tracks[1].box.minX, 0.7, accuracy: 1e-9) + } + + func testBriefDropoutKeepsIDButHidesStaleBox() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.4, 0.4)], at: 0) + + // Missed for 2 frames: still shown (predicted), without landmarks. + XCTAssertEqual(ids(tracker.update(with: [], at: frame)), ["f1"]) + XCTAssertEqual(ids(tracker.update(with: [], at: 2 * frame)), ["f1"]) + // 3rd missed frame: hidden but not forgotten. + XCTAssertEqual(ids(tracker.update(with: [], at: 3 * frame)), []) + + // Back after 0.6 s: same id. + XCTAssertEqual(ids(tracker.update(with: [face(0.41, 0.4)], at: 0.6)), ["f1"]) + } + + func testTrackRetiredAfterOneSecondUnseen() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.4, 0.4)], at: 0) + _ = tracker.update(with: [], at: 1.05) + XCTAssertEqual(ids(tracker.update(with: [face(0.4, 0.4)], at: 1.1)), ["f2"]) + } + + func testIDsAreNeverReusedWithinASession() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.1), face(0.6, 0.6)], at: 0) + _ = tracker.update(with: [], at: 2) + XCTAssertEqual(ids(tracker.update(with: [face(0.1, 0.1)], at: 2.1)), ["f3"]) + } + + func testResetStartsANewSession() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.1), face(0.6, 0.6)], at: 0) + tracker.reset() + XCTAssertEqual(ids(tracker.update(with: [face(0.6, 0.6)], at: 0.1)), ["f1"]) + } + + func testForgetTracksKeepsNumbering() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.1, 0.1), face(0.6, 0.6)], at: 0) + tracker.forgetTracks() + XCTAssertEqual(ids(tracker.update(with: [face(0.1, 0.1)], at: 0.1)), ["f3"]) + } + + func testMissedFrameClearsLandmarks() { + var tracker = FaceTracker() + let mesh = [SIMD3(0.5, 0.5, 0)] + let first = tracker.update(with: [FaceMeasurement(box: CGRect(x: 0.4, y: 0.4, width: 0.2, height: 0.2), landmarks: mesh)], at: 0) + XCTAssertTrue(first[0].hasLandmarks) + let missed = tracker.update(with: [], at: frame) + XCTAssertFalse(missed[0].hasLandmarks) + } + + func testIoU() { + let a = CGRect(x: 0, y: 0, width: 1, height: 1) + XCTAssertEqual(iou(a, a), 1, accuracy: 1e-9) + XCTAssertEqual(iou(a, CGRect(x: 0.5, y: 0, width: 1, height: 1)), 1.0 / 3.0, accuracy: 1e-9) + XCTAssertEqual(iou(a, CGRect(x: 2, y: 2, width: 1, height: 1)), 0) + } +} diff --git a/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/SpeakerScorerTests.swift b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/SpeakerScorerTests.swift new file mode 100644 index 0000000..8e93b53 --- /dev/null +++ b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/SpeakerScorerTests.swift @@ -0,0 +1,33 @@ +// v3 (#10). checks the largest face guess and the speaker pick used for the live highlight. + +import CoreGraphics +import XCTest +@testable import GraptionVision + +final class SpeakerScorerTests: XCTestCase { + private func track(_ id: String, size: CGFloat) -> FaceTrack { + FaceTrack(id: id, box: CGRect(x: 0, y: 0, width: size, height: size)) + } + + func testLargestFaceScoresOneOthersZero() { + let scores = LargestFaceScorer().scores(for: [track("f1", size: 0.1), track("f2", size: 0.3), track("f3", size: 0.2)], at: 0) + XCTAssertEqual(scores, ["f1": 0, "f2": 1, "f3": 0]) + XCTAssertEqual(scores.likelySpeaker(), "f2") + } + + func testTieGoesToOlderTrack() { + let scores = LargestFaceScorer().scores(for: [track("f1", size: 0.2), track("f2", size: 0.2)], at: 0) + XCTAssertEqual(scores.likelySpeaker(), "f1") + } + + func testNoFacesNoSpeaker() { + let scores = LargestFaceScorer().scores(for: [], at: 0) + XCTAssertTrue(scores.isEmpty) + XCTAssertNil(scores.likelySpeaker()) + } + + func testLikelySpeakerRespectsThreshold() { + XCTAssertNil(["f1": 0.4, "f2": 0.3].likelySpeaker()) + XCTAssertEqual(["f1": 0.4, "f2": 0.7].likelySpeaker(), "f2") + } +} diff --git a/docs/GRAPTION_CONTEXT.md b/docs/GRAPTION_CONTEXT.md index 2b90d19..8b2d3ce 100644 --- a/docs/GRAPTION_CONTEXT.md +++ b/docs/GRAPTION_CONTEXT.md @@ -239,9 +239,13 @@ For now the app has two ways to get landmarks, chosen by `LandmarkMode`. **`face - **Training must match the selected mode.** With `faceCrops` selected, `ml/asd` extraction must crop the same way (2.2×, 256 px) around the AVA face boxes instead of running on full frames (see §8). Until it does, train/test landmarks differ slightly. Features are nose-centered and divided by inter-ocular distance, so the expected drift is small, but it hasn't been measured. Head pose from the transformation matrix assumes the full camera view, so compute pose from full-frame landmarks (or crop identically in training) before enabling matrices. - **Switching modes:** edit one line in `CameraController` in `apps/ios/Graption/CameraView.swift`, `FaceCaptureEngine(landmarkMode: .faceCrops)` → `.fullFrame` (or back), rebuild, and update this section and the extraction to match. -**FaceTracker** -- Greedy IoU matching of the current frame's faces to existing tracks (IoU ≥ 0.3), then a centroid-distance fallback. -- Drop a track after 1s unseen. IDs are `f1`, `f2`, … per session. +**FaceTracker** (`GraptionVision/FaceTracker.swift`, unit-tested with `swift test`) +- Greedy IoU matching of the current frame's faces to existing tracks (IoU ≥ 0.3), then a centroid-distance fallback (within 1.5× the face size) for fast movers. +- Boxes are smoothed with a One Euro filter. A missed face coasts on its last velocity and is still shown for 2 frames, without landmarks. +- Drop a track after 1s unseen; until then a returning face gets its old ID back. IDs are `f1`, `f2`, … per session, never reused; they restart when the camera stops. Rotating the phone forgets all tracks (their boxes are in the old orientation), so faces get new IDs. + +**Speaker guess (until SpeakerModel ships)** +- `LargestFaceScorer` implements the `SpeakerScorer` protocol: the largest face gets `p = 1`, the rest `0`. The Core ML model replaces it behind the same protocol. **FeatureBuilder** Must match `ml/common/features.py` exactly: same order, same normalization. It produces one vector per face per frame: From 37619e718d1281ba87e06b3323630c0ce93b0dd3 Mon Sep 17 00:00:00 2001 From: AadarshJena Date: Tue, 27 Oct 2026 18:23:00 -0400 Subject: [PATCH 2/2] V3: keep face IDs for 2 s and label the guessed speaker "talking" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-device test: turning the head away long enough to lose detection gave the face a new ID on return, and the green-only speaker highlight was hard to read. - FaceTracker keeps an unseen face's ID for 2 s (was 1 s) - Speaker box label reads "fX talking" - Tests for a 2 s dropout keeping its ID and retirement after 2 s - GRAPTION_CONTEXT.md §6 updated with the new value and tradeoff Co-Authored-By: Claude Opus 5.5 --- apps/ios/Graption/CameraView.swift | 3 ++- .../Sources/GraptionVision/FaceTracker.swift | 6 ++++-- .../FaceTrackerTests.swift | 21 ++++++++++++++----- docs/GRAPTION_CONTEXT.md | 4 ++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/apps/ios/Graption/CameraView.swift b/apps/ios/Graption/CameraView.swift index 96b66c1..c841c3b 100644 --- a/apps/ios/Graption/CameraView.swift +++ b/apps/ios/Graption/CameraView.swift @@ -790,7 +790,8 @@ private struct FaceBox: View { height: box.height * renderedHeight ) .overlay(alignment: .topLeading) { - Text(id) + // v3. spell out the guess since green alone was hard to read on device. + Text(isSpeaker ? "\(id) talking" : id) .font(.caption.weight(.bold)) .foregroundStyle(.black) .padding(.horizontal, 4) diff --git a/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift index 0aef755..7820f85 100644 --- a/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift +++ b/apps/ios/Packages/GraptionVision/Sources/GraptionVision/FaceTracker.swift @@ -50,13 +50,15 @@ public struct FaceTrack: Identifiable, Sendable, Equatable { /// so stale boxes don't linger on screen. /// /// changes from the v2 tracker: iou pass added. ids are strings not ints. -/// tracks live 1 s instead of being dropped after 2 missed frames. +/// tracks live 2 s instead of being dropped after 2 missed frames. public struct FaceTracker: Sendable { public var minIoU: CGFloat = 0.3 /// center distance limit as a multiple of the track's larger side. public var maxCenterDistance: CGFloat = 1.5 /// seconds a track survives without a match before its id is retired. - public var maxUnseen: Double = 1.0 + /// 2 s not 1 s so a face can turn away and back without a new id (on device test). + /// tradeoff: someone new stepping into that spot within 2 s inherits the id. + public var maxUnseen: Double = 2.0 /// missed frames during which a track is still published at its predicted spot. public var maxCoastFrames = 2 diff --git a/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift index 03ea5b2..b32d6e3 100644 --- a/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift +++ b/apps/ios/Packages/GraptionVision/Tests/GraptionVisionTests/FaceTrackerTests.swift @@ -76,18 +76,29 @@ final class FaceTrackerTests: XCTestCase { XCTAssertEqual(ids(tracker.update(with: [face(0.41, 0.4)], at: 0.6)), ["f1"]) } - func testTrackRetiredAfterOneSecondUnseen() { + func testBoxGoneForTwoSecondsKeepsID() { var tracker = FaceTracker() _ = tracker.update(with: [face(0.4, 0.4)], at: 0) - _ = tracker.update(with: [], at: 1.05) - XCTAssertEqual(ids(tracker.update(with: [face(0.4, 0.4)], at: 1.1)), ["f2"]) + // head turns to profile: box narrows and shifts then detection is lost. + _ = tracker.update(with: [FaceMeasurement(box: CGRect(x: 0.45, y: 0.4, width: 0.1, height: 0.2))], at: frame) + var time = 2 * frame + while time < 2 + frame { _ = tracker.update(with: [], at: time); time += frame } + // a full 2 s after it was last seen the face turns back. + XCTAssertEqual(ids(tracker.update(with: [face(0.4, 0.4)], at: 2 + frame)), ["f1"]) + } + + func testTrackRetiredAfterTwoSecondsUnseen() { + var tracker = FaceTracker() + _ = tracker.update(with: [face(0.4, 0.4)], at: 0) + _ = tracker.update(with: [], at: 2.05) + XCTAssertEqual(ids(tracker.update(with: [face(0.4, 0.4)], at: 2.1)), ["f2"]) } func testIDsAreNeverReusedWithinASession() { var tracker = FaceTracker() _ = tracker.update(with: [face(0.1, 0.1), face(0.6, 0.6)], at: 0) - _ = tracker.update(with: [], at: 2) - XCTAssertEqual(ids(tracker.update(with: [face(0.1, 0.1)], at: 2.1)), ["f3"]) + _ = tracker.update(with: [], at: 4) + XCTAssertEqual(ids(tracker.update(with: [face(0.1, 0.1)], at: 4.1)), ["f3"]) } func testResetStartsANewSession() { diff --git a/docs/GRAPTION_CONTEXT.md b/docs/GRAPTION_CONTEXT.md index 8b2d3ce..5a3df04 100644 --- a/docs/GRAPTION_CONTEXT.md +++ b/docs/GRAPTION_CONTEXT.md @@ -242,10 +242,10 @@ For now the app has two ways to get landmarks, chosen by `LandmarkMode`. **`face **FaceTracker** (`GraptionVision/FaceTracker.swift`, unit-tested with `swift test`) - Greedy IoU matching of the current frame's faces to existing tracks (IoU ≥ 0.3), then a centroid-distance fallback (within 1.5× the face size) for fast movers. - Boxes are smoothed with a One Euro filter. A missed face coasts on its last velocity and is still shown for 2 frames, without landmarks. -- Drop a track after 1s unseen; until then a returning face gets its old ID back. IDs are `f1`, `f2`, … per session, never reused; they restart when the camera stops. Rotating the phone forgets all tracks (their boxes are in the old orientation), so faces get new IDs. +- Drop a track after 2s unseen; until then a returning face gets its old ID back. (Was 1s; on-device testing showed turning the head away and back often takes longer, and the face came back with a new ID. Tradeoff: someone new stepping into the same spot within 2s inherits the ID.) IDs are `f1`, `f2`, … per session, never reused; they restart when the camera stops. Rotating the phone forgets all tracks (their boxes are in the old orientation), so faces get new IDs. **Speaker guess (until SpeakerModel ships)** -- `LargestFaceScorer` implements the `SpeakerScorer` protocol: the largest face gets `p = 1`, the rest `0`. The Core ML model replaces it behind the same protocol. +- `LargestFaceScorer` implements the `SpeakerScorer` protocol: the largest face gets `p = 1`, the rest `0`. The camera view labels that face "talking" and draws it green. The Core ML model replaces it behind the same protocol. **FeatureBuilder** Must match `ml/common/features.py` exactly: same order, same normalization. It produces one vector per face per frame: