Skip to content

10 V3 give each face an id guess the speaker #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ajaymuru26 merged 2 commits into from
Oct 28, 2026
Merged

10 V3 give each face an id guess the speaker #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b846cc4
V3: extract FaceTracker into GraptionVision, stable f1/f2 IDs, larges…
AadarshJena Oct 21, 2026
37619e7
V3: keep face IDs for 2 s and label the guessed speaker "talking"
AadarshJena Oct 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ios-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,7 @@ jobs:
CODE_SIGNING_ALLOWED=NO \
build

- name: Test GraptionVision
run: swift test --package-path apps/ios/Packages/GraptionVision

#mac-os latest or xcode-27 for the mac server to host and run the Github Actions Server on?
12 changes: 12 additions & 0 deletions apps/ios/Graption.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
5A863841E58064FDABEE6763 /* GraptionUI in Frameworks */ = {isa = PBXBuildFile; productRef = 1504F8340E9DC8A84ACE6184 /* GraptionUI */; };
19DABF99307713050033A27A /* MediaPipeTasksAudio in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF98307713050033A27A /* MediaPipeTasksAudio */; };
19DABF9B307713050033A27A /* MediaPipeTasksVision in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF9A307713050033A27A /* MediaPipeTasksVision */; };
19DABF9E307713050033A27A /* GraptionVision in Frameworks */ = {isa = PBXBuildFile; productRef = 19DABF9D307713050033A27A /* GraptionVision */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
Expand Down Expand Up @@ -47,6 +48,7 @@
5A863841E58064FDABEE6763 /* GraptionUI in Frameworks */,
19DABF9B307713050033A27A /* MediaPipeTasksVision in Frameworks */,
19DABF99307713050033A27A /* MediaPipeTasksAudio in Frameworks */,
19DABF9E307713050033A27A /* GraptionVision in Frameworks */,
);
};
/* End PBXFrameworksBuildPhase section */
Expand Down Expand Up @@ -112,6 +114,7 @@
1504F8340E9DC8A84ACE6184 /* GraptionUI */,
19DABF98307713050033A27A /* MediaPipeTasksAudio */,
19DABF9A307713050033A27A /* MediaPipeTasksVision */,
19DABF9D307713050033A27A /* GraptionVision */,
);
productName = Graption;
productReference = 9835CFA1306C55A900F6C1EC /* Graption.app */;
Expand Down Expand Up @@ -145,6 +148,7 @@
65CCECAE3074877500093FF3 /* XCLocalSwiftPackageReference "Packages/GraptionAudio" */,
6CF0856E124649A4FFEEDDAC /* XCLocalSwiftPackageReference "Packages/GraptionUI" */,
19DABF97307713050033A27A /* XCRemoteSwiftPackageReference "mediapipe" */,
19DABF9C307713050033A27A /* XCLocalSwiftPackageReference "Packages/GraptionVision" */,
);
preferredProjectObjectVersion = 77;
productRefGroup = 9835CFA2306C55A900F6C1EC /* Products */;
Expand Down Expand Up @@ -396,6 +400,10 @@
isa = XCLocalSwiftPackageReference;
relativePath = Packages/GraptionUI;
};
19DABF9C307713050033A27A /* XCLocalSwiftPackageReference "Packages/GraptionVision" */ = {
isa = XCLocalSwiftPackageReference;
relativePath = Packages/GraptionVision;
};
/* End XCLocalSwiftPackageReference section */

/* Begin XCRemoteSwiftPackageReference section */
Expand Down Expand Up @@ -429,6 +437,10 @@
package = 19DABF97307713050033A27A /* XCRemoteSwiftPackageReference "mediapipe" */;
productName = MediaPipeTasksVision;
};
19DABF9D307713050033A27A /* GraptionVision */ = {
isa = XCSwiftPackageProductDependency;
productName = GraptionVision;
};
/* End XCSwiftPackageProductDependency section */
};
rootObject = 9835CF99306C55A900F6C1EC /* Project object */;
Expand Down
218 changes: 40 additions & 178 deletions apps/ios/Graption/CameraView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,128 +12,7 @@ import Combine
import UIKit
import Vision
import MediaPipeTasksVision

nonisolated private struct OneEuroFilter {
private let minCutoff: Double
private let beta: Double
private let derivativeCutoff: Double
private var previousValue: Double?
private var previousDerivative = 0.0
private var previousTimestamp: Double?

// Inputs are normalized (0...1) image coordinates, so beta is much larger than the
// pixel-space defaults from the paper; otherwise the speed term never reduces lag.
init(minCutoff: Double = 1.25, beta: Double = 8.0, derivativeCutoff: Double = 1.0) {
self.minCutoff = minCutoff
self.beta = beta
self.derivativeCutoff = derivativeCutoff
}

mutating func filter(_ value: Double, at timestamp: Double) -> Double {
guard let previousValue, let previousTimestamp else {
self.previousValue = value
self.previousTimestamp = timestamp
return value
}

let deltaTime = max(timestamp - previousTimestamp, 1.0 / 120.0)
let rawDerivative = (value - previousValue) / deltaTime
let derivativeAlpha = smoothingFactor(deltaTime, cutoff: derivativeCutoff)
let derivative = exponentialFilter(
rawDerivative,
previous: self.previousDerivative,
alpha: derivativeAlpha
)
let valueAlpha = smoothingFactor(deltaTime, cutoff: minCutoff + beta * abs(derivative))
let filteredValue = exponentialFilter(value, previous: previousValue, alpha: valueAlpha)

self.previousValue = filteredValue
self.previousDerivative = derivative
self.previousTimestamp = timestamp
return filteredValue
}

private func smoothingFactor(_ deltaTime: Double, cutoff: Double) -> Double {
let tau = 1.0 / (2.0 * .pi * cutoff)
return 1.0 / (1.0 + tau / deltaTime)
}

private func exponentialFilter(_ value: Double, previous: Double, alpha: Double) -> Double {
alpha * value + (1.0 - alpha) * previous
}
}

/// One face as published to the UI. `box` and `landmarks` are normalized to the full,
/// portrait-oriented frame with a top-left origin. `landmarks` is empty when the face was
/// detected but too small or occluded for MediaPipe to fit a mesh.
nonisolated struct FaceTrack: Identifiable, Sendable {
let id: Int
let box: CGRect
let landmarks: [SIMD3<Float>]

var hasLandmarks: Bool { !landmarks.isEmpty }
}

nonisolated private struct FaceMeasurement {
let box: CGRect
let landmarks: [SIMD3<Float>]
}

nonisolated private struct TrackedFace {
let id: Int
var box: CGRect
var landmarks: [SIMD3<Float>] = []
var velocity = CGPoint.zero
var missedFrames = 0
var xFilter = OneEuroFilter()
var yFilter = OneEuroFilter()
var widthFilter = OneEuroFilter()
var heightFilter = OneEuroFilter()
private var lastTimestamp: Double?

init(id: Int, measurement: FaceMeasurement, timestamp: Double) {
self.id = id
self.box = measurement.box
update(with: measurement, timestamp: timestamp)
}

mutating func update(with measurement: FaceMeasurement, timestamp: Double) {
let filtered = CGRect(
x: xFilter.filter(measurement.box.minX, at: timestamp),
y: yFilter.filter(measurement.box.minY, at: timestamp),
width: widthFilter.filter(measurement.box.width, at: timestamp),
height: heightFilter.filter(measurement.box.height, at: timestamp)
)
if let lastTimestamp {
let timeScale = max(timestamp - lastTimestamp, 1.0 / 120.0)
velocity = CGPoint(
x: (filtered.midX - box.midX) / timeScale,
y: (filtered.midY - box.midY) / timeScale
)
}
box = filtered
landmarks = measurement.landmarks
missedFrames = 0
lastTimestamp = timestamp
}

mutating func predict(at timestamp: Double) {
let deltaTime = max(timestamp - (lastTimestamp ?? timestamp), 1.0 / 120.0)
let decay = pow(0.65, deltaTime * 15.0)
box = box.offsetBy(
dx: velocity.x * deltaTime * decay,
dy: velocity.y * deltaTime * decay
)
velocity.x *= decay
velocity.y *= decay
// Landmarks from an older frame would no longer line up with the predicted box.
landmarks = []
missedFrames += 1
lastTimestamp = timestamp
}

var output: FaceTrack { FaceTrack(id: id, box: box, landmarks: landmarks) }
}
import GraptionVision

/// Where MediaPipe looks for landmarks. Detection always uses Apple Vision.
nonisolated enum LandmarkMode: Sendable {
Expand Down Expand Up @@ -162,8 +41,9 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
private static let minFaceConfidence: Float = 0.5

let session = AVCaptureSession()
/// Called on the video queue after every processed frame, with that frame's pixel size.
var onFaces: (([FaceTrack], CGSize) -> Void)?
/// called on the video queue after every frame with the tracked faces. the guessed speaker
/// id (nil if none). and the frame's pixel size.
var onFaces: (([FaceTrack], String?, CGSize) -> Void)?
private var cameraPosition: AVCaptureDevice.Position = .back
private var videoRotationAngle: CGFloat = 90

Expand All @@ -182,8 +62,10 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
private let sequenceHandler = VNSequenceRequestHandler()
private var faceLandmarker: FaceLandmarker?
private var cropPool: CVPixelBufferPool?
private var trackedFaces: [TrackedFace] = []
private var nextTrackID = 0
// v3. tracking lives in GraptionVision/FaceTracker.swift.
private var tracker = FaceTracker()
// largest face guess until the core ml speaker model replaces it behind SpeakerScorer.
private var speakerScorer: any SpeakerScorer = LargestFaceScorer()
private var publishedEmpty = false
private var lastLandmarkerTimestamp = -1

Expand All @@ -209,7 +91,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
session.stopRunning()
}
videoQueue.async { [self] in
trackedFaces.removeAll()
// camera stopped means session over. ids restart at f1.
tracker.reset()
}
}
}
Expand All @@ -225,7 +108,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
// Existing tracks are in the old orientation's coordinates.
guard changed else { return }
videoQueue.async { [self] in
trackedFaces.removeAll()
// not reset() since restarting at f1 mid session could hand an old id to someone new.
tracker.forgetTracks()
}
}
}
Expand Down Expand Up @@ -259,7 +143,8 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
session.commitConfiguration()
configureCamera(for: device, fps: 15)
videoQueue.async { [self] in
trackedFaces.removeAll()
// other camera means other people. new ids but numbering keeps counting.
tracker.forgetTracks()
publishedEmpty = false
}
completion(true)
Expand Down Expand Up @@ -351,11 +236,12 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
guard timestamp.isFinite else { return }

let frameSize = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
let faces = updateTracking(with: detectFaces(in: pixelBuffer, timestamp: timestamp), timestamp: timestamp)
let faces = tracker.update(with: detectFaces(in: pixelBuffer, timestamp: timestamp), at: timestamp)
let speaker = speakerScorer.scores(for: faces, at: timestamp).likelySpeaker()
// Skip repeated empty updates so an empty room doesn't re-render SwiftUI at 15 fps.
if faces.isEmpty && publishedEmpty { return }
publishedEmpty = faces.isEmpty
onFaces?(faces, frameSize)
onFaces?(faces, speaker, frameSize)
}

// MARK: Detection
Expand Down Expand Up @@ -557,51 +443,6 @@ nonisolated final class FaceCaptureEngine: NSObject, AVCaptureVideoDataOutputSam
return pool
}

// MARK: Tracking

private func updateTracking(with measurements: [FaceMeasurement], timestamp: Double) -> [FaceTrack] {
// Greedy assignment over all pairs, closest first, so one face can't steal another's
// track just because it was detected earlier in the list.
var candidates: [(track: Int, measurement: Int, distance: CGFloat)] = []
for trackIndex in trackedFaces.indices {
let track = trackedFaces[trackIndex].box
let maxDistance = max(track.width, track.height) * 1.5
for measurementIndex in measurements.indices {
let distance = distance(from: track, to: measurements[measurementIndex].box)
if distance <= maxDistance {
candidates.append((trackIndex, measurementIndex, distance))
}
}
}
candidates.sort { $0.distance < $1.distance }

var unmatchedTracks = Set(trackedFaces.indices)
var unmatchedMeasurements = Set(measurements.indices)
for candidate in candidates
where unmatchedTracks.contains(candidate.track) && unmatchedMeasurements.contains(candidate.measurement) {
trackedFaces[candidate.track].update(with: measurements[candidate.measurement], timestamp: timestamp)
unmatchedTracks.remove(candidate.track)
unmatchedMeasurements.remove(candidate.measurement)
}

for trackIndex in unmatchedTracks {
trackedFaces[trackIndex].predict(at: timestamp)
}
for measurementIndex in unmatchedMeasurements.sorted() {
trackedFaces.append(
TrackedFace(id: nextTrackID, measurement: measurements[measurementIndex], timestamp: timestamp)
)
nextTrackID += 1
}
trackedFaces.removeAll { $0.missedFrames > 2 || $0.box.maxX < 0 || $0.box.minX > 1 ||
$0.box.maxY < 0 || $0.box.minY > 1 }
return trackedFaces.map(\.output)
}

private func distance(from lhs: CGRect, to rhs: CGRect) -> CGFloat {
hypot(lhs.midX - rhs.midX, lhs.midY - rhs.midY)
}

// MARK: Camera

/// Rotation and mirroring are set on the analyzed frames, not only the preview.
Expand Down Expand Up @@ -659,6 +500,8 @@ final class CameraController: ObservableObject {
@Published private(set) var status: Status = .notDetermined
@Published private(set) var faces: [FaceTrack] = []
@Published private(set) var cameraPosition: AVCaptureDevice.Position = .back
/// id of the face drawn as the speaker. nil when no faces.
@Published private(set) var speakerID: String?
/// Width / height of the analyzed frames; flips when the phone rotates.
@Published private(set) var captureAspectRatio: CGFloat = 9.0 / 16.0
private var isSwitchingCamera = false
Expand All @@ -669,9 +512,10 @@ final class CameraController: ObservableObject {
var session: AVCaptureSession { engine.session }

init() {
engine.onFaces = { [weak self] faces, frameSize in
engine.onFaces = { [weak self] faces, speakerID, frameSize in
DispatchQueue.main.async {
self?.faces = faces
self?.speakerID = speakerID
self?.captureAspectRatio = frameSize.width / frameSize.height
}
}
Expand Down Expand Up @@ -703,6 +547,7 @@ final class CameraController: ObservableObject {
func stop() {
engine.stop()
faces = []
speakerID = nil
}

func setVideoRotationAngle(_ angle: CGFloat) {
Expand All @@ -720,6 +565,7 @@ final class CameraController: ObservableObject {
guard success else { return }
self.cameraPosition = next
self.faces = []
self.speakerID = nil
}
}
}
Expand Down Expand Up @@ -835,8 +681,10 @@ struct CameraView: View {

ForEach(controller.faces) { face in
FaceBox(
id: face.id,
box: face.box,
hasLandmarks: face.hasLandmarks,
isSpeaker: face.id == controller.speakerID,
size: geometry.size,
sourceAspectRatio: controller.captureAspectRatio
)
Expand Down Expand Up @@ -917,9 +765,12 @@ struct CameraView: View {

/// Solid when MediaPipe fit a mesh to the face; dashed when only Vision detected it
/// (usually a face too small or turned away for landmarks).
/// v3. green marks the guessed speaker. the label shows the track id so id stability is visible.
private struct FaceBox: View {
let id: String
let box: CGRect
let hasLandmarks: Bool
let isSpeaker: Bool
let size: CGSize
let sourceAspectRatio: CGFloat

Expand All @@ -930,12 +781,23 @@ private struct FaceBox: View {
let horizontalCrop = (renderedWidth - size.width) / 2
let verticalCrop = (renderedHeight - size.height) / 2

let color: Color = isSpeaker ? .green : .yellow

Rectangle()
.stroke(.yellow, style: StrokeStyle(lineWidth: 3, dash: hasLandmarks ? [] : [8, 6]))
.stroke(color, style: StrokeStyle(lineWidth: 3, dash: hasLandmarks ? [] : [8, 6]))
.frame(
width: box.width * renderedWidth,
height: box.height * renderedHeight
)
.overlay(alignment: .topLeading) {
// v3. spell out the guess since green alone was hard to read on device.
Text(isSpeaker ? "\(id) talking" : id)
.font(.caption.weight(.bold))
.foregroundStyle(.black)
.padding(.horizontal, 4)
.background(color)
.offset(y: -18)
}
.position(
x: box.midX * renderedWidth - horizontalCrop,
y: box.midY * renderedHeight - verticalCrop
Expand Down
Loading
Loading