@remotion/canvas-capture: Add interactivity promo video

packages/canvas-capture/src/index.tsx

@@ -1,3 +1,13 @@
+import {
+	ALL_FORMATS,
+	BufferSource,
+	BufferTarget,
+	CanvasSource,
+	Conversion,
+	Input,
+	Output,
+	WebMOutputFormat,
+} from 'mediabunny';
 import React, {
 	forwardRef,
 	useCallback,
@@ -53,6 +63,11 @@ type MouseMovement = {
 	readonly cursor: string;
 };
 
+type PointerClick = {
+	readonly timeInSeconds: number;
+	readonly type: 'pointer-down' | 'pointer-up';
+};
+
 type CaptureMetadata = {
 	readonly density: number;
 	readonly contentRect: {
@@ -79,6 +94,7 @@ type RecordingState = {
 	readonly source: CanvasVideoSource;
 	readonly startedAt: number;
 	readonly mouseMovements: MouseMovement[];
+	readonly pointerClicks: PointerClick[];
 	lastTimestampInSeconds: number | null;
 	lastFramePromise: Promise<void>;
 	frameCount: number;
@@ -94,12 +110,10 @@ export type HtmlInCanvasCaptureHandle = {
 
 type HtmlInCanvasCaptureProps = {
 	readonly children: React.ReactNode;
-	readonly density: number;
 	readonly filename: string;
 };
 
 type WithHtmlInCanvasCaptureProps = {
-	readonly density: number;
 	readonly filename: string;
 };
 
@@ -201,9 +215,10 @@ const downloadBlob = (blob: Blob, filename: string) => {
 	URL.revokeObjectURL(url);
 };
 
-const getJsonFilename = (filename: string) => {
-	return filename.replace(/\.[^.]+$/, '') + '.json';
-};
+const CAPTURE_METADATA_TAG_KEY = 'REMOTION_CAPTURE_DATA';
+const CAPTURE_DENSITY = 2;
+
+export {CAPTURE_METADATA_TAG_KEY, CAPTURE_DENSITY};
 
 const logCaptureError = (message: string, err: unknown) => {
 	// eslint-disable-next-line no-console
@@ -254,37 +269,45 @@ const finalizeRecording = async (
 		throw new Error('Mediabunny did not return an output buffer.');
 	}
 
-	downloadBlob(
-		new Blob([recording.target.buffer], {type: 'video/webm'}),
-		filename,
-	);
-	downloadBlob(
-		new Blob(
-			[
-				JSON.stringify(
-					{
-						startedAt: recording.startedAt,
-						endedAt: performance.now(),
-						captureMetadata: recording.captureMetadata,
-						mouseMovements: recording.mouseMovements,
-					},
-					null,
-					2,
-				),
-			],
-			{type: 'application/json'},
-		),
-		getJsonFilename(filename),
-	);
+	const captureData = JSON.stringify({
+		startedAt: recording.startedAt,
+		endedAt: performance.now(),
+		captureMetadata: recording.captureMetadata,
+		mouseMovements: recording.mouseMovements,
+		pointerClicks: recording.pointerClicks,
+	});
+
+	const remuxInput = new Input({
+		formats: ALL_FORMATS,
+		source: new BufferSource(recording.target.buffer),
+	});
+	const remuxTarget = new BufferTarget();
+	const remuxOutput = new Output({
+		format: new WebMOutputFormat(),
+		target: remuxTarget,
+	});
+	const conversion = await Conversion.init({
+		input: remuxInput,
+		output: remuxOutput,
+		tags: {
+			raw: {[CAPTURE_METADATA_TAG_KEY]: captureData},
+		},
+		showWarnings: false,
+	});
+	await conversion.execute();
+
+	if (!remuxTarget.buffer) {
+		throw new Error('Mediabunny remux did not return an output buffer.');
+	}
+
+	downloadBlob(new Blob([remuxTarget.buffer], {type: 'video/webm'}), filename);
 };
 
 export const HtmlInCanvasCapture = forwardRef<
 	HtmlInCanvasCaptureHandle,
 	HtmlInCanvasCaptureProps
->(({children, density, filename}, ref) => {
-	if (!Number.isFinite(density) || density <= 0) {
-		throw new Error('HTML-in-canvas capture density must be greater than 0.');
-	}
+>(({children, filename}, ref) => {
+	const density = CAPTURE_DENSITY;
 
 	const isSupported = useMemo(() => isHtmlInCanvasAvailable(), []);
 	const canvasRef = useRef<HtmlInCanvasElement | null>(null);
@@ -307,8 +330,6 @@ export const HtmlInCanvasCapture = forwardRef<
 			return;
 		}
 
-		const {BufferTarget, CanvasSource, Output, WebMOutputFormat} =
-			await import('mediabunny');
 		const target = new BufferTarget();
 		const output = new Output({
 			format: new WebMOutputFormat(),
@@ -330,6 +351,7 @@ export const HtmlInCanvasCapture = forwardRef<
 			source,
 			startedAt: performance.now(),
 			mouseMovements: [],
+			pointerClicks: [],
 			lastTimestampInSeconds: null,
 			lastFramePromise: Promise.resolve(),
 			frameCount: 0,
@@ -461,6 +483,40 @@ export const HtmlInCanvasCapture = forwardRef<
 		};
 	}, [density]);
 
+	useEffect(() => {
+		const onPointerDown = () => {
+			const recording = recordingRef.current;
+			if (!recording || recording.isFinalizing) {
+				return;
+			}
+
+			recording.pointerClicks.push({
+				timeInSeconds: (performance.now() - recording.startedAt) / 1000,
+				type: 'pointer-down',
+			});
+		};
+
+		const onPointerUp = () => {
+			const recording = recordingRef.current;
+			if (!recording || recording.isFinalizing) {
+				return;
+			}
+
+			recording.pointerClicks.push({
+				timeInSeconds: (performance.now() - recording.startedAt) / 1000,
+				type: 'pointer-up',
+			});
+		};
+
+		window.addEventListener('pointerdown', onPointerDown, true);
+		window.addEventListener('pointerup', onPointerUp, true);
+
+		return () => {
+			window.removeEventListener('pointerdown', onPointerDown, true);
+			window.removeEventListener('pointerup', onPointerUp, true);
+		};
+	}, []);
+
 	useEffect(() => {
 		if (!isSupported) {
 			return;
@@ -549,9 +605,9 @@ export const withHtmlInCanvasCapture = <Props extends object>(
 	return forwardRef<
 		HtmlInCanvasCaptureHandle,
 		Props & WithHtmlInCanvasCaptureProps
-	>(({density, filename, ...props}, ref) => {
+	>(({filename, ...props}, ref) => {
 		return (
-			<HtmlInCanvasCapture ref={ref} density={density} filename={filename}>
+			<HtmlInCanvasCapture ref={ref} filename={filename}>
 				<Component {...(props as Props)} />
 			</HtmlInCanvasCapture>
 		);