diff --git a/packages/dev/inspector-v2/src/extensibility/defaultInspectorExtensionFeed.ts b/packages/dev/inspector-v2/src/extensibility/defaultInspectorExtensionFeed.ts index 3ef10e970e1..9f1fb3beafc 100644 --- a/packages/dev/inspector-v2/src/extensibility/defaultInspectorExtensionFeed.ts +++ b/packages/dev/inspector-v2/src/extensibility/defaultInspectorExtensionFeed.ts @@ -35,4 +35,12 @@ export const DefaultInspectorExtensionFeed = new BuiltInsExtensionFeed("Inspecto ...BabylonWebResources, getExtensionModuleAsync: async () => await import("../services/panes/tools/reflectorService"), }, + { + name: "Animation Retargeting", + description: "Retarget animations from one skeleton to another using AnimatorAvatar. Includes a dedicated 3D viewport with dual-camera preview.", + keywords: ["animation", "retargeting", "skeleton", "avatar", "bones"], + ...BabylonWebResources, + author: { name: "Babylon.js", forumUserName: "" }, + getExtensionModuleAsync: async () => await import("../extensions/animationRetargeting/animationRetargetingExtension"), + }, ]); diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animation.ts b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animation.ts new file mode 100644 index 00000000000..a9b9fefda21 --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animation.ts @@ -0,0 +1,570 @@ +import { type Nullable } from "core/types"; +import { type ArcRotateCamera } from "core/Cameras/arcRotateCamera"; +import { type ShadowGenerator } from "core/Lights/Shadows/shadowGenerator"; +import { type Scene } from "core/scene"; +import { type AnimationGroup } from "core/Animations/animationGroup"; +import { type AbstractMesh } from "core/Meshes/abstractMesh"; +import { type TransformNode } from "core/Meshes/transformNode"; +import { type Vector3, type Quaternion, Matrix } from "core/Maths/math.vector"; +import { type Skeleton } from "core/Bones/skeleton"; +import { type RestPoseDataUpdate } from "./avatarManager"; + +import { Color3 } from "core/Maths/math.color"; +import { StandardMaterial } from "core/Materials/standardMaterial"; +import { MeshBuilder } from "core/Meshes/meshBuilder"; +import { Mesh } from "core/Meshes/mesh"; +import { Observable } from "core/Misc/observable"; +import { GizmoManager } from "core/Gizmos/gizmoManager"; +import { GizmoCoordinatesMode } from "core/Gizmos/gizmo"; +import { UtilityLayerRenderer } from "core/Rendering/utilityLayerRenderer"; +import { SkeletonViewer } from "core/Debug/skeletonViewer"; +import { AppendSceneAsync } from "core/Loading/sceneLoader"; +import { CreateSkeletonFromTransformNodeHierarchy } from "core/Bones/skeleton.functions"; + +// Side-effect import needed for scene.createPickingRay (prototype-augmented) +import "core/Culling/ray"; + +import { DistancePointToLine } from "./helperFunctions"; +import { type GizmoType } from "./avatar"; + +const ShadowLayerMask = 0x20000000; + +type TransformNodeTransformations = Map; + +/** + * Manages the right-side animation source viewport. + * Mirrors the original animation.ts from the playground (no AnimatorAvatar — uses TransformNodes directly). + */ +export class AnimationSource { + private _animationGroup: Nullable = null; + private _skeletonViewer: Nullable = null; + private _skeletonMesh: Nullable = null; + private _rootNode: Nullable = null; + private _gizmoManager: GizmoManager; + private _gizmoSelectedNode: Nullable = null; + private _selectedTransformNode: Nullable = null; + private _inRestPose = false; + private _initialTransformNodeTransformations: TransformNodeTransformations = new Map(); + private _restPoseTransformNodeTransformations: TransformNodeTransformations = new Map(); + private _retargetedTransformNodeTransformations: TransformNodeTransformations = new Map(); + + /** + * + */ + public readonly onLoadedObservable = new Observable(); + /** + * + */ + public readonly onGizmoNodeSelectedObservable = new Observable(); + /** + * + */ + public readonly onPlayingObservable = new Observable(); + + public get animationGroup(): Nullable { + return this._animationGroup; + } + public get isLoaded(): boolean { + return this._animationGroup !== null; + } + /** True when the animation is actively playing (not in rest pose). */ + public get isPlaying(): boolean { + return this._animationGroup !== null && !this._inRestPose; + } + public get selectedTransformNode(): Nullable { + return this._selectedTransformNode; + } + + public constructor( + private readonly _scene: Scene, + private readonly _camera: ArcRotateCamera, + private readonly _shadowGenerator: ShadowGenerator + ) { + this._gizmoManager = new GizmoManager(this._scene, undefined, new UtilityLayerRenderer(this._scene), new UtilityLayerRenderer(this._scene)); + this._gizmoManager.keepDepthUtilityLayer.setRenderCamera(this._camera); + this._gizmoManager.utilityLayer.setRenderCamera(this._camera); + this._gizmoManager.usePointerToAttachGizmos = false; + this._gizmoManager.coordinatesMode = GizmoCoordinatesMode.Local; + } + + public async loadAsync(path: string, restPoseUpdate?: RestPoseDataUpdate, skeletonRootNodeName?: string, animationGroupIndex = 0): Promise { + this._cleanScene(); + await this._loadFileAsync(path, restPoseUpdate, skeletonRootNodeName, animationGroupIndex); + } + + public clearScene(): void { + this._cleanScene(); + } + + /** Returns a sorted list of transform node names targeted by the animation group. + * @returns Sorted list of transform node names. + */ + public getTransformNodeNames(): string[] { + if (!this._animationGroup || !this._rootNode) { + return []; + } + + const listUnsorted = new Set(); + const listSorted: string[] = []; + const listAllSorted: string[] = []; + + for (const ta of this._animationGroup.targetedAnimations) { + if ((ta.target as TransformNode).getClassName?.() === "TransformNode") { + listUnsorted.add((ta.target as TransformNode).name); + } + } + + this._rootNode.getChildTransformNodes(false).forEach((node) => { + if (node.getClassName() === "TransformNode") { + listAllSorted.push(node.name); + } + }); + + for (const name of listAllSorted) { + if (listUnsorted.has(name)) { + listSorted.push(name); + } + } + + return listSorted; + } + + /** Returns transform node options in hierarchy order, with indented labels for parenting. + * @returns Array of label/value pairs. + */ + public getTransformNodeOptions(): { label: string; value: string }[] { + if (!this._animationGroup || !this._rootNode) { + return []; + } + + const allNames = new Set(this.getTransformNodeNames()); + // getChildTransformNodes(false) traverses in DFS order (parent before children) + const result: { label: string; value: string }[] = []; + + this._rootNode.getChildTransformNodes(false).forEach((node) => { + if (!allNames.has(node.name)) { + return; + } + // Compute depth relative to _rootNode + let depth = 0; + let p = node.parent; + while (p && p !== this._rootNode) { + depth++; + p = p.parent; + } + const indent = "\u00A0\u00A0".repeat(depth); + result.push({ label: indent + node.name, value: node.name }); + }); + + return result; + } + + /** Called just before retargeting; saves the current rest-pose state. */ + public prepareRetargeting(): void { + if (this._inRestPose) { + this._restPoseTransformNodeTransformations.clear(); + this._saveTransformNodes(this._restPoseTransformNodeTransformations); + } else { + this._restoreTransformNodes(this._restPoseTransformNodeTransformations); + } + this._retargetedTransformNodeTransformations.clear(); + this._saveTransformNodes(); + } + + /** Builds the rest-pose export data for the playground code generator. + * @param restPoseUpdate - Optional existing rest-pose data. + * @returns The rest-pose data update array. + */ + public buildExportData(restPoseUpdate?: RestPoseDataUpdate): RestPoseDataUpdate { + return restPoseUpdate ? [...restPoseUpdate] : []; + } + + public play(speed: number): void { + if (!this._animationGroup) { + return; + } + this._animationGroup.stop(); + this._animationGroup.start(true, speed); + this._inRestPose = false; + this.onPlayingObservable.notifyObservers(true); + } + + public setAnimSpeed(speed: number): void { + if (this._animationGroup) { + this._animationGroup.speedRatio = speed; + } + } + + public returnToRest(): void { + if (!this._animationGroup) { + return; + } + this._animationGroup.stop(); + this._restoreTransformNodes(this._restPoseTransformNodeTransformations); + this._inRestPose = true; + this.onPlayingObservable.notifyObservers(false); + } + + /** + * Captures the current transform node transformations (while in rest pose, possibly edited via gizmo) + * as a `RestPoseDataUpdate`. All transform node transformations are saved as absolute values. + * @returns The rest-pose data update array. + */ + public saveAsRestPose(): RestPoseDataUpdate { + const result: RestPoseDataUpdate = []; + if (!this._rootNode) { + return result; + } + this._rootNode.getChildTransformNodes(false).forEach((node) => { + if (!node.rotationQuaternion) { + return; + } + const data: { position?: number[]; scaling?: number[]; quaternion?: number[] } = {}; + data.position = node.position.asArray(); + data.scaling = node.scaling.asArray(); + data.quaternion = node.rotationQuaternion.asArray(); + result.push({ name: node.name, data }); + }); + // Update internal rest pose baseline + this._restPoseTransformNodeTransformations.clear(); + this._saveTransformNodes(this._restPoseTransformNodeTransformations); + return result; + } + + public setSkeletonVisible(visible: boolean): void { + this._skeletonShow(visible); + } + + public setSkeletonLocalAxes(visible: boolean): void { + this._skeletonShowLocalAxes(visible); + } + + public setGizmo(enabled: boolean, type: GizmoType): void { + this._updateGizmoEnabled(enabled, type); + } + + /** Attaches the gizmo to the transform node with the given name (called when user picks from the dropdown). + * @param nodeName - The name of the transform node to attach the gizmo to. + */ + public attachGizmoToTransformNode(nodeName: string): void { + if (!this._rootNode) { + return; + } + const node = this._rootNode.getChildTransformNodes(false, (n) => n.name === nodeName)[0]; + if (!node) { + return; + } + if (this._gizmoSelectedNode) { + this._gizmoSelectedNode.position.copyFrom(node.absolutePosition); + } + this._selectedTransformNode = node; + this._gizmoManager.attachToNode(node); + } + + public handleBoneClick(x: number, y: number): void { + if (!this._rootNode) { + return; + } + const ray = this._scene.createPickingRay(x, y, Matrix.Identity(), this._camera, false); + let selectedNode: Nullable = null; + let minDistance = Number.POSITIVE_INFINITY; + + this._rootNode.getChildTransformNodes(false).forEach((node) => { + const d = DistancePointToLine(node.absolutePosition, ray.origin, ray.direction); + if (d < minDistance) { + minDistance = d; + selectedNode = node; + if (this._gizmoSelectedNode) { + this._gizmoSelectedNode.position.copyFrom(node.absolutePosition); + } + } + }); + + if (selectedNode) { + this._selectedTransformNode = selectedNode as TransformNode; + this._gizmoManager.attachToNode(selectedNode as TransformNode); + this.onGizmoNodeSelectedObservable.notifyObservers((selectedNode as TransformNode).name); + } + } + + public dispose(): void { + this._cleanScene(); + this._gizmoSelectedNode?.dispose(); + this._gizmoSelectedNode = null; + this._gizmoManager.dispose(); + } + + private async _loadFileAsync(path: string, restPoseUpdate?: RestPoseDataUpdate, skeletonRootNodeName?: string, animationGroupIndex = 0): Promise { + // Track meshes before loading so we can identify newly added ones + const meshesBefore = new Set(this._scene.meshes); + + if (path.startsWith("file:")) { + await AppendSceneAsync(path.substring(5), this._scene, { rootUrl: "file:" }); + } else { + await AppendSceneAsync(path, this._scene); + } + + // Find the newly added root node (avoid collision with existing nodes named "__root__") + this._rootNode = null; + for (const mesh of this._scene.meshes) { + if (!meshesBefore.has(mesh) && !mesh.parent) { + this._rootNode = mesh; + break; + } + } + if (!this._rootNode) { + return; + } + this._rootNode.name = "reference"; + + // Dispose ALL newly added meshes except the animation root node itself. + // The animation side only needs transform nodes for the skeleton, not meshes. + // This also cleans up duplicate avatar meshes from serialized PG scene data. + for (const mesh of [...this._scene.meshes]) { + if (!meshesBefore.has(mesh) && mesh !== this._rootNode) { + mesh.dispose(false, false); + } + } + + // Keep only our 2 cameras; remove any cameras loaded from the file + while (this._scene.cameras.length > 2) { + this._scene.cameras[2].dispose(); + } + + // Keep only the animation group at the requested index (skip "avatar" retargeted ones) + this._animationGroup = null; + const lstAnimDelete = new Set(); + let currentIndex = 0; + for (const animGroup of this._scene.animationGroups) { + if (animGroup.name !== "avatar") { + if (currentIndex === animationGroupIndex) { + this._animationGroup = animGroup; + } else { + lstAnimDelete.add(animGroup); + } + currentIndex++; + } + } + lstAnimDelete.forEach((anim) => anim.dispose()); + + if (!this._animationGroup) { + return; + } + this._animationGroup.name = "reference"; + this._animationGroup.stop(); + + // Delete all skeletons loaded from the animation file (we don't need them) + const skeletonsDelete = new Set(); + for (const skeleton of this._scene.skeletons) { + if (skeleton.name !== "avatar") { + skeletonsDelete.add(skeleton); + } + } + skeletonsDelete.forEach((skeleton) => skeleton.dispose()); + + // Delete all meshes under the root node (animation files only need transform nodes) + this._rootNode + .getChildren((node) => node instanceof Mesh, false) + .forEach((mesh) => { + (mesh as Mesh).dispose(); + }); + + // Use the user-selected root node, or fall back to hips heuristic + let skeletonRoot: TransformNode = this._rootNode; + if (skeletonRootNodeName) { + const found = this._rootNode.getChildTransformNodes(false, (node) => node.name === skeletonRootNodeName)[0]; + if (found) { + skeletonRoot = found; + } + } else { + // Legacy heuristic: find hips/pelvis node + let hipsNode: Nullable = this._rootNode.getChildTransformNodes(false, (node) => node.name === "mixamorig:Hips")?.[0] ?? null; + if (!hipsNode) { + hipsNode = this._rootNode.getChildTransformNodes(false, (node) => node.name === "Hips")[0] ?? null; + } + if (!hipsNode) { + hipsNode = this._rootNode.getChildTransformNodes(false, (node) => node.name.toLowerCase().indexOf("hips") >= 0)[0] ?? null; + } + if (!hipsNode) { + hipsNode = this._rootNode.getChildTransformNodes(false, (node) => node.name.toLowerCase().indexOf("pelvis") >= 0)[0] ?? null; + } + if (hipsNode) { + skeletonRoot = hipsNode; + } + } + + // Create a skeleton + visualization mesh from the transform node hierarchy + const meshOptions: { name?: string; boneMeshSize?: number; createMesh?: boolean; mesh?: Mesh } = { + name: "reference", + boneMeshSize: 0.0001, + createMesh: true, + }; + + const animSkeleton = CreateSkeletonFromTransformNodeHierarchy(skeletonRoot, this._scene, meshOptions); + const meshAnim = meshOptions.mesh!; + this._skeletonMesh = meshAnim; + meshAnim.layerMask = ShadowLayerMask; + meshAnim.isVisible = false; + + // Auto-scale the animation hierarchy to fit the viewport (~2 units) + const boundingVectors = meshAnim.getHierarchyBoundingVectors(true); + const extendSize = boundingVectors.max.subtract(boundingVectors.min); + const maxDim = Math.max(extendSize.x, extendSize.y, extendSize.z); + + if (maxDim > 3 || maxDim < 0.1) { + const scaleRatio = 2 / maxDim; + const s = this._rootNode.scaling; + this._rootNode.scaling.set(Math.sign(s.x) * scaleRatio, Math.sign(s.y) * scaleRatio, Math.sign(s.z) * scaleRatio); + } + + const initialNode = skeletonRoot; + this._selectedTransformNode = initialNode; + this._gizmoManager.attachToNode(initialNode); + this.onGizmoNodeSelectedObservable.notifyObservers(initialNode.name); + + const meshAnimExtendSize = meshAnim.getBoundingInfo().boundingBox.extendSizeWorld; + + const skeletonViewer = new SkeletonViewer(animSkeleton, meshAnim, this._scene, undefined, 0, { + displayMode: SkeletonViewer.DISPLAY_SPHERE_AND_SPURS, + displayOptions: { + showLocalAxes: false, + localAxesSize: Math.max(meshAnimExtendSize.x, meshAnimExtendSize.y, meshAnimExtendSize.z) * 5, + }, + }); + this._skeletonViewer = skeletonViewer; + skeletonViewer.isEnabled = false; + skeletonViewer.utilityLayer!.setRenderCamera(this._camera); + this._skeletonDebugMeshSet(); + + const utilScene = skeletonViewer.utilityLayer!.utilityLayerScene!; + const mat = new StandardMaterial("selectedBoneMatAnim", utilScene); + mat.emissiveColor = Color3.Red(); + mat.disableLighting = true; + + this._gizmoSelectedNode?.dispose(); + this._gizmoSelectedNode = MeshBuilder.CreateSphere("selectedTransformNode", { diameter: 0.07, segments: 10 }, utilScene); + this._gizmoSelectedNode.setEnabled(false); + this._gizmoSelectedNode.material = mat; + this._gizmoSelectedNode.layerMask = ShadowLayerMask; + this._gizmoSelectedNode.renderingGroupId = 3; + this._gizmoSelectedNode.position.copyFrom(initialNode.absolutePosition); + + if (restPoseUpdate && restPoseUpdate.length > 0) { + this._applyRestPoseUpdate(restPoseUpdate); + } + + this._initialTransformNodeTransformations.clear(); + this._restPoseTransformNodeTransformations.clear(); + this._saveTransformNodes(this._initialTransformNodeTransformations); + this._saveTransformNodes(this._restPoseTransformNodeTransformations); + this._animationGroup.speedRatio = 1; + + this._inRestPose = false; + + this.onLoadedObservable.notifyObservers(); + } + + private _skeletonShow(visible: boolean): void { + if (!this._skeletonViewer) { + return; + } + this._skeletonViewer.isEnabled = visible; + this._skeletonDebugMeshSet(); + } + + private _skeletonShowLocalAxes(visible: boolean): void { + if (!this._skeletonViewer) { + return; + } + this._skeletonViewer.changeDisplayOptions("showLocalAxes", visible); + this._skeletonDebugMeshSet(); + } + + private _gizmoShow(show: boolean): void { + if (this._gizmoSelectedNode) { + this._gizmoSelectedNode.setEnabled(show); + } + } + + private _updateGizmoEnabled(enabled: boolean, type: GizmoType): void { + this._gizmoManager.positionGizmoEnabled = enabled && type === "Position"; + this._gizmoManager.rotationGizmoEnabled = enabled && type === "Rotation"; + this._gizmoManager.scaleGizmoEnabled = enabled && type === "Scale"; + this._gizmoShow(enabled); + } + + private _skeletonDebugMeshSet(): void { + if (!this._skeletonViewer?.debugMesh) { + return; + } + this._skeletonViewer.debugMesh.layerMask = ShadowLayerMask; + this._skeletonViewer.debugMesh.alwaysSelectAsActiveMesh = true; + this._shadowGenerator.addShadowCaster(this._skeletonViewer.debugMesh); + if (this._skeletonViewer.debugLocalAxesMesh) { + this._skeletonViewer.debugLocalAxesMesh.layerMask = ShadowLayerMask; + this._skeletonViewer.debugLocalAxesMesh.alwaysSelectAsActiveMesh = true; + } + } + + private _saveTransformNodes(map?: TransformNodeTransformations): void { + if (!this._rootNode) { + return; + } + map = map ?? this._retargetedTransformNodeTransformations; + this._rootNode.getChildTransformNodes(false).forEach((node) => { + if (!node.rotationQuaternion) { + return; + } + map!.set(node, { + position: node.position.clone(), + scaling: node.scaling.clone(), + quaternion: node.rotationQuaternion.clone(), + }); + }); + } + + private _restoreTransformNodes(map?: TransformNodeTransformations): void { + map = map ?? this._retargetedTransformNodeTransformations; + map.forEach((transformation, tn) => { + tn.position = transformation.position.clone(); + tn.scaling = transformation.scaling.clone(); + tn.rotationQuaternion = transformation.quaternion.clone(); + tn.computeWorldMatrix(true); + }); + } + + private _applyRestPoseUpdate(restPoseUpdate: RestPoseDataUpdate): void { + if (!this._rootNode) { + return; + } + for (const dataBlock of restPoseUpdate) { + const node = this._rootNode.getChildTransformNodes(false, (node) => node.name === dataBlock.name)[0]; + if (node) { + if (dataBlock.data.position) { + node.position.fromArray(dataBlock.data.position); + } + if (dataBlock.data.scaling) { + node.scaling.fromArray(dataBlock.data.scaling); + } + if (dataBlock.data.quaternion) { + node.rotationQuaternion!.fromArray(dataBlock.data.quaternion); + } + } + } + } + + private _cleanScene(): void { + this._animationGroup?.dispose(); + if (this._skeletonViewer) { + this._skeletonViewer.skeleton.dispose(); + this._skeletonViewer.dispose(); + this._skeletonViewer = null; + } + this._skeletonMesh?.dispose(); + this._skeletonMesh = null; + this._rootNode?.dispose(false, true); + this._animationGroup = null; + this._rootNode = null; + this._selectedTransformNode = null; + } +} diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationManager.ts b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationManager.ts new file mode 100644 index 00000000000..f3f76bfa8c3 --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationManager.ts @@ -0,0 +1,516 @@ +import { type ISettingsStore, type SettingDescriptor } from "../../services/settingsStore"; +import { type RestPoseDataUpdate } from "./avatarManager"; + +const IDBName = "BabylonInspector_AnimRetargeting"; +const IDBStore = "animationFiles"; + +// ─── Types ──────────────────────────────────────────────────────────────────── + +/** Maps an animation group (by its index in scene.animationGroups) to a user-chosen display name. */ +export type AnimationGroupMapping = { + /** The index of the AnimationGroup in scene.animationGroups — stable across reloads of the same file. */ + index: number; + /** The original AnimationGroup.name from the file. */ + groupName: string; + /** User-chosen display name shown in the main UI dropdown. Empty = not included. */ + displayName: string; +}; + +export type StoredAnimation = { + /** Unique, immutable identifier generated at creation time. Used as the IndexedDB key prefix. */ + id: string; + /** User-chosen name for this animation file entry (shown in the list). */ + name: string; + /** + * + */ + source: "url" | "file" | "scene"; + /** + * + */ + url?: string; + /** + * + */ + fileNames?: string[]; + /** + * + */ + namingScheme: string; + /** User-selected root node for skeleton creation. */ + rootNodeName: string; + /** One entry per animation group in the file. */ + animations: AnimationGroupMapping[]; + /** + * + */ + restPoseUpdate?: RestPoseDataUpdate; + /** When true, this entry is transient and will be purged on next startup. */ + sessionOnly?: boolean; +}; + +type SerializedData = { + animations: StoredAnimation[]; +}; + +const AnimationSettingDescriptor: SettingDescriptor = { + key: "AnimRetargeting/Animations", + defaultValue: { animations: [] }, +}; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function GenerateId(): string { + return `${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; +} + +async function BlobToBase64(blob: Blob): Promise { + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => { + const result = reader.result as string; + resolve(result.substring(result.indexOf(",") + 1)); + }; + reader.onerror = () => reject(new Error("Failed to read blob")); + reader.readAsDataURL(blob); + }); +} + +function Base64ToBlob(base64: string): Blob { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return new Blob([bytes]); +} + +// ─── IndexedDB helpers ──────────────────────────────────────────────────────── + +async function OpenIDB(): Promise { + return await new Promise((resolve, reject) => { + const request = indexedDB.open(IDBName, 2); + request.onupgradeneeded = () => { + const db = request.result; + if (!db.objectStoreNames.contains("avatarFiles")) { + db.createObjectStore("avatarFiles"); + } + if (!db.objectStoreNames.contains(IDBStore)) { + db.createObjectStore(IDBStore); + } + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error ?? new Error("IDB open failed")); + }); +} + +async function IdbPut(key: string, value: Blob): Promise { + const db = await OpenIDB(); + return await new Promise((resolve, reject) => { + const tx = db.transaction(IDBStore, "readwrite"); + tx.objectStore(IDBStore).put(value, key); + tx.oncomplete = () => { + db.close(); + resolve(); + }; + tx.onerror = () => { + db.close(); + reject(tx.error ?? new Error("IDB transaction failed")); + }; + }); +} + +async function IdbGet(key: string): Promise { + const db = await OpenIDB(); + return await new Promise((resolve, reject) => { + const tx = db.transaction(IDBStore, "readonly"); + const req = tx.objectStore(IDBStore).get(key); + req.onsuccess = () => { + db.close(); + resolve(req.result as Blob | undefined); + }; + req.onerror = () => { + db.close(); + reject(req.error ?? new Error("IDB get failed")); + }; + }); +} + +async function IdbDeletePrefix(prefix: string): Promise { + const db = await OpenIDB(); + return await new Promise((resolve, reject) => { + const tx = db.transaction(IDBStore, "readwrite"); + const store = tx.objectStore(IDBStore); + const cursorReq = store.openCursor(); + cursorReq.onsuccess = () => { + const cursor = cursorReq.result; + if (cursor) { + if (typeof cursor.key === "string" && cursor.key.startsWith(prefix)) { + cursor.delete(); + } + cursor.continue(); + } + }; + tx.oncomplete = () => { + db.close(); + resolve(); + }; + tx.onerror = () => { + db.close(); + reject(tx.error ?? new Error("IDB transaction failed")); + }; + }); +} + +// ─── AnimationManager ───────────────────────────────────────────────────────── + +/** + * + */ +export class AnimationManager { + private _animations: StoredAnimation[] = []; + + public constructor(private readonly _settingsStore: ISettingsStore) { + this._loadFromStorage(); + } + + // ─── Public API ─────────────────────────────────────────────────────────── + + public getAllAnimations(): readonly StoredAnimation[] { + return this._animations; + } + + /** Returns all non-empty display names across all stored animation files. + * @returns Array of display name strings. + */ + public getAllDisplayNames(): string[] { + const names: string[] = []; + for (const entry of this._animations) { + for (const mapping of entry.animations) { + if (mapping.displayName) { + names.push(mapping.displayName); + } + } + } + return names; + } + + /** Finds the stored animation file and specific mapping for a given display name. + * @param displayName - The display name to look up. + * @returns The stored animation entry and mapping, or undefined. + */ + public getByDisplayName(displayName: string): + | { + /** + * + */ + entry: StoredAnimation /** + * + */; + mapping: AnimationGroupMapping; + } + | undefined { + if (!displayName) { + return undefined; + } + for (const entry of this._animations) { + for (const mapping of entry.animations) { + if (mapping.displayName === displayName) { + return { entry, mapping }; + } + } + } + return undefined; + } + + public getAnimationById(id: string): StoredAnimation | undefined { + return this._animations.find((a) => a.id === id); + } + + /** Checks whether a display name is already used by any animation (optionally excluding a specific file id). + * @param displayName - The display name to check. + * @param excludeFileId - Optional file id to exclude from the check. + * @returns True if the display name is already in use. + */ + public isDisplayNameUsed(displayName: string, excludeFileId?: string): boolean { + for (const entry of this._animations) { + if (excludeFileId && entry.id === excludeFileId) { + continue; + } + for (const mapping of entry.animations) { + if (mapping.displayName === displayName) { + return true; + } + } + } + return false; + } + + /** + * Adds or replaces an animation. + * If the animation has no `id`, one is generated automatically. + * For file-based animations, the files must already be stored in IndexedDB via `storeFilesAsync`. + * @param animation - The animation to add or replace. + */ + public addAnimation(animation: StoredAnimation): void { + if (!animation.id) { + animation.id = GenerateId(); + } + const idx = this._animations.findIndex((a) => a.id === animation.id); + if (idx !== -1) { + this._animations[idx] = animation; + } else { + this._animations.push(animation); + } + this._saveToStorage(); + } + + /** + * Removes an animation by id and its associated files from IndexedDB. + * @param id - The animation id to remove. + */ + public async removeAnimationAsync(id: string): Promise { + const animation = this._animations.find((a) => a.id === id); + if (!animation) { + return; + } + if (animation.source === "file") { + await IdbDeletePrefix(`animation:${id}/`); + } + this._animations = this._animations.filter((a) => a.id !== id); + this._saveToStorage(); + } + + /** + * Stores files in IndexedDB for a file-based animation, keyed by its immutable id. + * @param animationId - The animation id. + * @param files - The files to store. + * @returns The list of file names stored. + */ + public async storeFilesAsync(animationId: string, files: File[]): Promise { + const fileNames: string[] = []; + await Promise.all( + files.map(async (file) => { + const key = `animation:${animationId}/${file.name}`; + await IdbPut(key, file); + fileNames.push(file.name); + }) + ); + return fileNames; + } + + /** + * Retrieves files from IndexedDB for a file-based animation and converts them to File objects. + * @param animationId - The animation id. + * @param fileNames - The file names to retrieve. + * @returns Array of File objects. + */ + public async getFilesAsync(animationId: string, fileNames: string[]): Promise { + const results = await Promise.all( + fileNames.map(async (fileName) => { + const key = `animation:${animationId}/${fileName}`; + const blob = await IdbGet(key); + if (blob) { + return new File([blob], fileName, { type: blob.type }); + } + return undefined; + }) + ); + return results.filter((f): f is File => f !== undefined); + } + + /** + * Returns all animations in a JSON-serializable format, including base64-encoded file data for file-based entries. + * Session-only entries are excluded from the export. + * @returns Array of stored animations with optional file data. + */ + public async exportDataAsync(): Promise< + Array< + StoredAnimation & { + /** + * + */ + fileData?: Record; + } + > + > { + return await Promise.all( + this._animations + .filter((animation) => !animation.sessionOnly) + .map(async (animation) => { + const entry: StoredAnimation & { + /** + * + */ + fileData?: Record; + } = { ...animation }; + if (animation.source === "file" && animation.fileNames?.length) { + const files = await this.getFilesAsync(animation.id, animation.fileNames); + const pairs = await Promise.all(files.map(async (file) => [file.name, await BlobToBase64(file)] as const)); + const fileData: Record = {}; + for (const [name, b64] of pairs) { + fileData[name] = b64; + } + entry.fileData = fileData; + } + return entry; + }) + ); + } + + /** + * Imports animations, including restoring file-based entries to IndexedDB from base64 data. + * @param animations - The animations to import with optional file data. + * @param mode - "replace" clears all existing data first; "append" skips duplicates. + * @returns List of skipped entry descriptions. + */ + public async importDataAsync( + animations: Array< + StoredAnimation & { + /** + * + */ + fileData?: Record; + } + >, + mode: "replace" | "append" + ): Promise { + const skipped: string[] = []; + + if (mode === "replace") { + this._animations = []; + } + + // Sequential processing needed: each entry may depend on prior state for duplicate detection + for (const animation of animations) { + if (mode === "append" && this._animations.some((a) => a.name === animation.name)) { + skipped.push(`animation "${animation.name}"`); + } else { + const newId = GenerateId(); + const { fileData, ...animData } = animation; + this._animations.push({ ...animData, id: newId }); + + if (animData.source === "file" && fileData) { + const fileEntries = Object.entries(fileData); + // eslint-disable-next-line no-await-in-loop + await Promise.all( + fileEntries.map(async ([fileName, base64]) => { + const blob = Base64ToBlob(base64); + await IdbPut(`animation:${newId}/${fileName}`, blob); + }) + ); + this._animations[this._animations.length - 1].fileNames = fileEntries.map(([n]) => n); + } + } + } + + this._saveToStorage(); + return skipped; + } + + /** + * Creates default animation entries if the list is empty. + */ + public createDefaults(): void { + if (this._animations.length > 0) { + return; + } + const baseUrl = "https://assets.babylonjs.com/mixamo/Animations/"; + const defaults: { + /** + * + */ + name: string /** + * + */; + file: string /** + * + */; + scheme: string /** + * + */; + displayName: string /** + * + */; + rootNode: string; + }[] = [ + { name: "Rumba Dancing", file: "Rumba Dancing.glb", scheme: "Mixamo", displayName: "Rumba Dancing", rootNode: "mixamorig:Hips" }, + { name: "Hip Hop Dancing", file: "Hip Hop Dancing.glb", scheme: "Mixamo", displayName: "Hip Hop Dancing", rootNode: "mixamorig:Hips" }, + { name: "Sitting Clap", file: "Sitting Clap.glb", scheme: "Mixamo", displayName: "Sitting Clap", rootNode: "mixamorig:Hips" }, + { name: "Walking", file: "Walking.glb", scheme: "Mixamo", displayName: "Walking", rootNode: "mixamorig:Hips" }, + { name: "Catwalk Walking", file: "Catwalk Walking.glb", scheme: "Mixamo", displayName: "Catwalk Walking", rootNode: "mixamorig:Hips" }, + { name: "Praying", file: "Praying.glb", scheme: "Mixamo", displayName: "Praying", rootNode: "mixamorig:Hips" }, + { name: "Mousey Walking", file: "Mousey_walking.glb", scheme: "Mixamo", displayName: "Mousey Walking", rootNode: "mixamorig:Hips" }, + ]; + for (const d of defaults) { + this.addAnimation({ + id: "", + name: d.name, + source: "url", + url: baseUrl + d.file, + namingScheme: d.scheme, + rootNodeName: d.rootNode, + animations: [{ index: 0, groupName: d.displayName, displayName: d.displayName }], + }); + } + } + + /** + * Removes all session-only entries and saves the updated list. + */ + public purgeSessionOnly(): void { + const before = this._animations.length; + this._animations = this._animations.filter((a) => !a.sessionOnly); + if (this._animations.length !== before) { + this._saveToStorage(); + } + } + + // ─── Private ────────────────────────────────────────────────────────────── + + private _loadFromStorage(): void { + try { + const data = this._settingsStore.readSetting(AnimationSettingDescriptor); + // Migrate old format: entries with `animationGroupName` but no `animations` array + this._animations = (data.animations ?? []).map( + ( + entry: StoredAnimation & { + /** + * + */ + animationGroupName?: string; + } + ) => { + if (!entry.animations) { + entry.animations = []; + if (entry.animationGroupName) { + entry.animations.push({ + index: 0, + groupName: entry.animationGroupName, + displayName: entry.name || entry.animationGroupName, + }); + } + if (!entry.name) { + entry.name = entry.animationGroupName ?? "Unnamed"; + } + delete entry.animationGroupName; + } + if (!entry.name) { + entry.name = "Unnamed"; + } + if (!entry.rootNodeName) { + entry.rootNodeName = ""; + } + return entry as StoredAnimation; + } + ); + } catch { + this._animations = []; + } + } + + private _saveToStorage(): void { + const data: SerializedData = { animations: this._animations }; + this._settingsStore.writeSetting(AnimationSettingDescriptor, data); + } +} diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingExtension.tsx b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingExtension.tsx new file mode 100644 index 00000000000..0efc2dccd30 --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingExtension.tsx @@ -0,0 +1,202 @@ +import { type IDisposable } from "core/index"; +import { type Engine } from "core/Engines/engine"; +import { type ServiceDefinition } from "../../modularity/serviceDefinition"; +import { type IShellService, ShellServiceIdentity } from "../../services/shellService"; +import { type ISettingsStore, SettingsStoreIdentity } from "../../services/settingsStore"; +import { type ISceneContext, SceneContextIdentity } from "../../services/sceneContext"; +import { type IPlaygroundBridge, PlaygroundBridgeIdentity } from "../../services/playgroundBridgeService"; + +import { type IObserver, Observable } from "core/Misc/observable"; + +import { PersonRunningRegular } from "@fluentui/react-icons"; + +import { AnimationRetargetingViewport } from "./animationRetargetingViewport"; +import { AnimationRetargetingPanel, DefaultPanelState, type PanelStateStore } from "./animationRetargetingPanel"; + +import { type RetargetingSceneManager } from "./retargetingSceneManager"; +import { NamingSchemeManager } from "./namingSchemeManager"; +import { AvatarManager } from "./avatarManager"; +import { AnimationManager } from "./animationManager"; + +// Module-level reference to the playground bridge. Set by the BridgeConsumer service +// when the inspector is opened from the Playground. Null otherwise. +let PlaygroundBridge: IPlaygroundBridge | null = null; + +/** + * Returns the playground bridge if available (inspector opened from PG). + * @returns The playground bridge instance or null. + */ +export function GetPlaygroundBridge(): IPlaygroundBridge | null { + return PlaygroundBridge; +} + +/** + * Service definition for the Animation Retargeting extension. + * - Registers a dedicated 3D viewport (central content, left+right cameras). + * - Registers a controls side pane with Avatar / Animation / Retarget sections. + * - Exposes an Enable / Disable toggle to restore the original inspector scene. + */ +export const AnimationRetargetingServiceDefinition: ServiceDefinition<[], [IShellService, ISettingsStore, ISceneContext]> = { + friendlyName: "Animation Retargeting", + consumes: [ShellServiceIdentity, SettingsStoreIdentity, SceneContextIdentity], + factory: (shellService, settingsStore, sceneContext) => { + // Observable that fires whenever a new scene manager is ready (on each enable) + const onManagerReadyObs = new Observable(); + // Observable that broadcasts enable/disable state changes to the panel + const isEnabledObs = new Observable(); + // Observable that fires when the config dialog closes — panel refreshes dropdowns + const onConfigChangedObs = new Observable(); + + // Naming scheme manager — persists across extension lifetime via ISettingsStore + const namingSchemeManager = new NamingSchemeManager(settingsStore); + + // Avatar manager — persists across extension lifetime via ISettingsStore + IndexedDB + const avatarManager = new AvatarManager(settingsStore); + + // Animation manager — persists across extension lifetime via ISettingsStore + IndexedDB + const animationManager = new AnimationManager(settingsStore); + + // Purge any session-only entries left over from a previous session + avatarManager.purgeSessionOnly(); + animationManager.purgeSessionOnly(); + + // Create default entries if both lists are empty (first-time use) + if (avatarManager.getAllAvatars().length === 0 && animationManager.getAllDisplayNames().length === 0) { + avatarManager.createDefaults(); + animationManager.createDefaults(); + } + + let isEnabled = false; + let viewportReg: IDisposable | null = null; + let currentManager: RetargetingSceneManager | null = null; + let sceneDisposeObserver: IObserver | null = null; + + // Persists all panel UI state across remounts (e.g. when the panel is docked elsewhere) + const allAvatars = avatarManager.getAllAvatars(); + const allDisplayNames = animationManager.getAllDisplayNames(); + const persistedPanelState: PanelStateStore = { + ...DefaultPanelState, + avatarName: allAvatars.length > 0 ? allAvatars[0].name : "", + animationName: allDisplayNames.length > 0 ? allDisplayNames[0] : "", + }; + + function cleanupViewport(): void { + sceneDisposeObserver?.remove(); + sceneDisposeObserver = null; + viewportReg?.dispose(); + viewportReg = null; + currentManager = null; + } + + function setEnabled(enabled: boolean): void { + if (enabled === isEnabled) { + return; + } + isEnabled = enabled; + + if (enabled) { + const scene = sceneContext.currentScene; + if (!scene) { + return; + } + const engine = scene.getEngine() as Engine; + + // When the PG scene is disposed (e.g. Play button), clean up immediately + // so the retargeting scene is torn down before the engine is fully disposed. + sceneDisposeObserver = scene.onDisposeObservable.add(() => { + cleanupViewport(); + isEnabled = false; + isEnabledObs.notifyObservers(false); + }); + + // Register the central-content viewport; the component uses the PG's engine + viewportReg = shellService.addCentralContent({ + key: "AnimationRetargetingViewport", + order: 10, + component: () => ( + { + currentManager = manager; + onManagerReadyObs.notifyObservers(manager); + }} + /> + ), + }); + } else { + cleanupViewport(); + } + + isEnabledObs.notifyObservers(enabled); + } + + // Start disabled so the user opts-in to the viewport + setEnabled(false); + + const panelReg = shellService.addSidePane({ + key: "AnimationRetargetingPanel", + title: "Animation Retargeting", + icon: PersonRunningRegular, + horizontalLocation: "left", + verticalLocation: "top", + content: () => ( + currentManager} + getCurrentScene={() => sceneContext.currentScene} + getPlaygroundBridge={GetPlaygroundBridge} + namingSchemeManager={namingSchemeManager} + avatarManager={avatarManager} + animationManager={animationManager} + stateStore={persistedPanelState} + onSetEnabled={setEnabled} + onToggleConsole={() => currentManager?.htmlConsole.toggle()} + /> + ), + }); + + // When the PG scene changes (e.g. Play button), auto-disable so the old + // engine/scene references are released and render loops are restored cleanly. + const sceneObserver = sceneContext.currentSceneObservable.add(() => { + if (isEnabled) { + setEnabled(false); + } + }); + + return { + dispose: () => { + setEnabled(false); + sceneObserver.remove(); + panelReg.dispose(); + onManagerReadyObs.clear(); + isEnabledObs.clear(); + onConfigChangedObs.clear(); + }, + }; + }, +}; + +/** + * Optional service that wires the Playground bridge when the inspector is opened from the PG. + * If the bridge service is not registered (inspector opened outside PG), this service is simply + * never created and the extension works without it. + */ +const BridgeConsumerServiceDefinition: ServiceDefinition<[], [IPlaygroundBridge]> = { + friendlyName: "Animation Retargeting Bridge Consumer", + consumes: [PlaygroundBridgeIdentity], + factory: (bridge) => { + PlaygroundBridge = bridge; + return { + dispose: () => { + PlaygroundBridge = null; + }, + }; + }, +}; + +export default { + serviceDefinitions: [AnimationRetargetingServiceDefinition, BridgeConsumerServiceDefinition], +} as const; diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingPanel.tsx b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingPanel.tsx new file mode 100644 index 00000000000..d03924bf365 --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingPanel.tsx @@ -0,0 +1,1434 @@ +import { type FunctionComponent, useCallback, useEffect, useRef, useState } from "react"; +import { type Observable } from "core/Misc/observable"; +import { type Nullable } from "core/types"; +import { type Scene } from "core/scene"; +import { type IPlaygroundBridge } from "../../services/playgroundBridgeService"; +import { FilesInputStore } from "core/Misc/filesInputStore"; +import { SceneLoader } from "core/Loading/sceneLoader"; +import { SceneSerializer } from "core/Misc/sceneSerializer"; +import { type Transform, TransformProperties } from "../../components/properties/transformProperties"; +import { makeStyles, tokens, Body1Strong, Caption1 } from "@fluentui/react-components"; +import { Button } from "shared-ui-components/fluent/primitives/button"; +import { ArrowClockwiseRegular, EyeRegular, EyeOffRegular, WindowConsoleRegular, SettingsRegular, InfoRegular } from "@fluentui/react-icons"; + +import { Accordion as BabylonAccordion, AccordionSection as BabylonAccordionSection } from "shared-ui-components/fluent/primitives/accordion"; +import { SwitchPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/switchPropertyLine"; +import { SyncedSliderPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/syncedSliderPropertyLine"; +import { StringDropdownPropertyLine } from "shared-ui-components/fluent/hoc/propertyLines/dropdownPropertyLine"; +import { BoneDropdown } from "./boneDropdown"; +import { ButtonLine } from "shared-ui-components/fluent/hoc/buttonLine"; + +import { type RetargetingSceneManager } from "./retargetingSceneManager"; +import { type NamingSchemeManager } from "./namingSchemeManager"; +import { type AvatarManager } from "./avatarManager"; +import { type AnimationManager } from "./animationManager"; +import { type GizmoType } from "./avatar"; +import { RetargetingConfigDialog } from "./retargetingConfigDialog"; +import { UXContextProvider } from "../../components/uxContextProvider"; + +/** + * Mirrors gui.ts _getSourceTransformNodeList: returns animation transform nodes filtered + * to only those that have a bone remapping entry for the current avatar/animation pair. + * Matches original PG behaviour for "Root node" and "Ground ref. node" dropdowns. + * @param names - List of all transform node names. + * @param animName - The animation display name. + * @param avatarName - The avatar name. + * @param namingSchemeManager - The naming scheme manager. + * @param avatarManager - The avatar manager. + * @param animationManager - The animation manager. + * @returns Filtered list of bone names. + */ +function BuildFilteredBoneList( + names: string[], + animName: string, + avatarName: string, + namingSchemeManager: NamingSchemeManager, + avatarManager: AvatarManager, + animationManager: AnimationManager +): string[] { + const sourceScheme = animationManager.getByDisplayName(animName)?.entry.namingScheme; + const targetScheme = avatarManager.getAvatar(avatarName)?.namingScheme; + if (!sourceScheme || !targetScheme) { + return names; + } + const remapping = namingSchemeManager.getRemapping(sourceScheme, targetScheme); + if (!remapping) { + return names; + } + const result: string[] = []; + for (const [key] of remapping) { + if (names.includes(key)) { + result.push(key); + } + } + return result; +} + +const useStyles = makeStyles({ + root: { + display: "flex", + flexDirection: "column", + height: "100%", + overflow: "hidden", + }, + toolbar: { + display: "flex", + gap: tokens.spacingHorizontalXXS, + padding: tokens.spacingVerticalXS, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + flexShrink: 0, + }, + actionRow: { + flexShrink: 0, + display: "flex", + flexDirection: "column", + gap: tokens.spacingVerticalXS, + padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalM}`, + backgroundColor: tokens.colorNeutralBackground2, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + }, + scrollContent: { + flex: 1, + overflowY: "auto", + overflowX: "hidden", + }, + disabledOverlay: { + flex: 1, + display: "flex", + alignItems: "center", + justifyContent: "center", + color: tokens.colorNeutralForeground3, + fontSize: tokens.fontSizeBase200, + fontStyle: "italic", + }, + boneDropdownLabel: { + flex: "1 1 0", + minWidth: "50px", + textAlign: "left", + whiteSpace: "nowrap", + overflow: "hidden", + textOverflow: "ellipsis", + }, + boneDropdownWrapper: { + display: "flex", + alignItems: "center", + width: "100%", + paddingTop: tokens.spacingVerticalXXS, + paddingBottom: tokens.spacingVerticalXXS, + minHeight: tokens.lineHeightHero700, + boxSizing: "border-box", + }, + boneDropdownControl: { + width: "225px", // 150px (standard) * 1.5 = 50% larger + boxSizing: "border-box", + flexShrink: 0, + }, + loadingText: { + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalM}`, + fontSize: tokens.fontSizeBase200, + color: tokens.colorNeutralForeground3, + }, + dropdownRow: { + display: "flex", + alignItems: "center", + gap: tokens.spacingHorizontalXS, + }, + flexOne: { + flex: 1, + }, + indented: { + paddingLeft: tokens.spacingHorizontalL, + }, + errorHint: { + color: tokens.colorPaletteRedForeground1, + fontSize: tokens.fontSizeBase200, + paddingLeft: tokens.spacingHorizontalL, + }, +}); + +const GizmoTypeOptions: { label: string; value: string }[] = [ + { label: "Position", value: "Position" }, + { label: "Rotation", value: "Rotation" }, + { label: "Scale", value: "Scale" }, +]; +const VerticalAxisOptions: { label: string; value: string }[] = [ + { label: "Auto", value: "" }, + { label: "X", value: "X" }, + { label: "Y", value: "Y" }, + { label: "Z", value: "Z" }, +]; + +export type PanelStateStore = { + /** + * + */ + avatarName: string; + /** + * + */ + avatarRescaleAvatar: boolean; + /** + * + */ + avatarAnimSpeed: number; + /** + * + */ + avatarShowSkeleton: boolean; + /** + * + */ + avatarShowSkeletonLocalAxes: boolean; + /** + * + */ + avatarGizmoEnabled: boolean; + /** + * + */ + avatarGizmoType: string; + /** + * + */ + avatarGizmoSelectedNode: string; + /** + * + */ + animationName: string; + /** + * + */ + animationSpeed: number; + /** + * + */ + animationShowSkeletonLocalAxes: boolean; + /** + * + */ + animationGizmoEnabled: boolean; + /** + * + */ + animationGizmoType: string; + /** + * + */ + animationGizmoSelectedNode: string; + /** + * + */ + fixAnimations: boolean; + /** + * + */ + checkHierarchy: boolean; + /** + * + */ + retargetAnimationKeys: boolean; + /** + * + */ + fixRootPosition: boolean; + /** + * + */ + fixGroundReference: boolean; + /** + * + */ + fixGroundReferenceDynamicRefNode: boolean; + /** + * + */ + rootNodeName: string; + /** + * + */ + groundReferenceNodeName: string; + /** + * + */ + groundReferenceVerticalAxis: string; +}; + +export const DefaultPanelState: PanelStateStore = { + avatarName: "", + avatarRescaleAvatar: true, + avatarAnimSpeed: 1, + avatarShowSkeleton: false, + avatarShowSkeletonLocalAxes: false, + avatarGizmoEnabled: false, + avatarGizmoType: "Rotation", + avatarGizmoSelectedNode: "", + animationName: "", + animationSpeed: 1, + animationShowSkeletonLocalAxes: false, + animationGizmoEnabled: false, + animationGizmoType: "Rotation", + animationGizmoSelectedNode: "", + fixAnimations: false, + checkHierarchy: false, + retargetAnimationKeys: true, + fixRootPosition: true, + fixGroundReference: false, + fixGroundReferenceDynamicRefNode: false, + rootNodeName: "Auto", + groundReferenceNodeName: "", + groundReferenceVerticalAxis: "", +}; + +export type AnimationRetargetingPanelProps = { + /** + * + */ + initialIsEnabled: boolean; + /** + * + */ + isEnabledObs: Observable; + /** + * + */ + onConfigChangedObs: Observable; + /** + * + */ + onManagerReadyObs: Observable; + /** + * + */ + getCurrentManager: () => RetargetingSceneManager | null; + /** + * + */ + getCurrentScene: () => Nullable; + /** + * + */ + getPlaygroundBridge: () => IPlaygroundBridge | null; + /** + * + */ + namingSchemeManager: NamingSchemeManager; + /** + * + */ + avatarManager: AvatarManager; + /** + * + */ + animationManager: AnimationManager; + /** Persisted across remounts (e.g. when the panel is docked elsewhere). Lives in the extension closure. */ + stateStore: PanelStateStore; + /** + * + */ + onSetEnabled: (enabled: boolean) => void; + /** + * + */ + onToggleConsole: () => void; +}; + +/** + * Main panel component for the Animation Retargeting extension. + * @returns The React element. + */ +export const AnimationRetargetingPanel: FunctionComponent = ({ + initialIsEnabled, + isEnabledObs, + onConfigChangedObs, + onManagerReadyObs, + getCurrentManager, + getCurrentScene, + getPlaygroundBridge, + namingSchemeManager, + avatarManager, + animationManager, + stateStore, + onSetEnabled, + onToggleConsole, +}) => { + const classes = useStyles(); + const managerRef = useRef(null); + const handleLoadAvatarRef = useRef<(name: string, rescale: boolean) => void | Promise>(() => {}); + const handleLoadAnimationRef = useRef<(name: string) => void | Promise>(() => {}); + const [isEnabled, setIsEnabled] = useState(initialIsEnabled); + const [, forceUpdate] = useState(0); + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [isConsoleVisible, setIsConsoleVisible] = useState(false); + + // Avatar state + const [avatarName, setAvatarName] = useState(() => stateStore.avatarName); + const [avatarRescaleAvatar, setAvatarRescaleAvatar] = useState(() => stateStore.avatarRescaleAvatar); + const [avatarAnimSpeed, setAvatarAnimSpeed] = useState(() => stateStore.avatarAnimSpeed); + const [avatarShowSkeleton, setAvatarShowSkeleton] = useState(() => stateStore.avatarShowSkeleton); + const [avatarShowSkeletonLocalAxes, setAvatarShowSkeletonLocalAxes] = useState(() => stateStore.avatarShowSkeletonLocalAxes); + const [avatarGizmoEnabled, setAvatarGizmoEnabled] = useState(() => stateStore.avatarGizmoEnabled); + const [avatarGizmoType, setAvatarGizmoType] = useState(() => stateStore.avatarGizmoType); + const [avatarGizmoSelectedNode, setAvatarGizmoSelectedNode] = useState(() => stateStore.avatarGizmoSelectedNode); + + // Animation state + const [animationName, setAnimationName] = useState(() => stateStore.animationName); + const [animationSpeed, setAnimationSpeed] = useState(() => stateStore.animationSpeed); + const [animationShowSkeletonLocalAxes, setAnimationShowSkeletonLocalAxes] = useState(() => stateStore.animationShowSkeletonLocalAxes); + const [animationGizmoEnabled, setAnimationGizmoEnabled] = useState(() => stateStore.animationGizmoEnabled); + const [animationGizmoType, setAnimationGizmoType] = useState(() => stateStore.animationGizmoType); + const [animationGizmoSelectedNode, setAnimationGizmoSelectedNode] = useState(() => stateStore.animationGizmoSelectedNode); + + // Retarget options + const [fixAnimations, setFixAnimations] = useState(() => stateStore.fixAnimations); + const [checkHierarchy, setCheckHierarchy] = useState(() => stateStore.checkHierarchy); + const [retargetAnimationKeys, setRetargetAnimationKeys] = useState(() => stateStore.retargetAnimationKeys); + const [fixRootPosition, setFixRootPosition] = useState(() => stateStore.fixRootPosition); + const [fixGroundReference, setFixGroundReference] = useState(() => stateStore.fixGroundReference); + const [fixGroundReferenceDynamicRefNode, setFixGroundReferenceDynamicRefNode] = useState(() => stateStore.fixGroundReferenceDynamicRefNode); + const [rootNodeName, setRootNodeName] = useState(() => stateStore.rootNodeName); + const [groundReferenceNodeName, setGroundReferenceNodeName] = useState(() => stateStore.groundReferenceNodeName); + const [groundReferenceVerticalAxis, setGroundReferenceVerticalAxis] = useState(() => stateStore.groundReferenceVerticalAxis); + + // Dynamic bone lists + const [avatarBoneOptions, setAvatarBoneOptions] = useState< + { + /** + * + */ + label: string /** + * + */; + value: string; + }[] + >([]); + const [rootNodeOptions, setRootNodeOptions] = useState([{ label: "Auto", value: "Auto" }]); + const [groundRefNodeOptions, setGroundRefNodeOptions] = useState< + { + /** + * + */ + label: string /** + * + */; + value: string; + }[] + >([]); + + // Dropdown options derived from managers — refreshed when config dialog closes + const [avatarOptions, setAvatarOptions] = useState(() => + avatarManager + .getAllAvatars() + .map((a) => ({ label: a.name, value: a.name })) + .sort((a, b) => a.label.localeCompare(b.label)) + ); + const [animationOptions, setAnimationOptions] = useState(() => + animationManager + .getAllDisplayNames() + .map((n) => ({ label: n, value: n })) + .sort((a, b) => a.label.localeCompare(b.label)) + ); + + // Selected bone/node transforms for the Properties section + const [avatarGizmoSelectedTransform, setAvatarGizmoSelectedTransform] = useState(null); + const [animGizmoSelectedTransform, setAnimGizmoSelectedTransform] = useState(null); + + // Loading / retargeted states + const [isAvatarLoaded, setIsAvatarLoaded] = useState(false); + const [isAnimLoaded, setIsAnimLoaded] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [isRetargeted, setIsRetargeted] = useState(false); + const [isExporting, setIsExporting] = useState(false); + const [isAvatarPlaying, setIsAvatarPlaying] = useState(false); + const [isAnimPlaying, setIsAnimPlaying] = useState(false); + + // Refs tracking current values for callbacks (always up-to-date even in stale closures). + // Updated synchronously during render so callbacks always read the latest values. + // Also used to save all state to stateStore on unmount. + const stateSnapshotRef = useRef({ + avatarName, + avatarRescaleAvatar, + avatarAnimSpeed, + avatarShowSkeleton, + avatarShowSkeletonLocalAxes, + avatarGizmoEnabled, + avatarGizmoType, + avatarGizmoSelectedNode, + animationName, + animationSpeed, + animationShowSkeletonLocalAxes, + animationGizmoEnabled, + animationGizmoType, + animationGizmoSelectedNode, + fixAnimations, + checkHierarchy, + retargetAnimationKeys, + fixRootPosition, + fixGroundReference, + fixGroundReferenceDynamicRefNode, + rootNodeName, + groundReferenceNodeName, + groundReferenceVerticalAxis, + }); + stateSnapshotRef.current = { + avatarName, + avatarRescaleAvatar, + avatarAnimSpeed, + avatarShowSkeleton, + avatarShowSkeletonLocalAxes, + avatarGizmoEnabled, + avatarGizmoType, + avatarGizmoSelectedNode, + animationName, + animationSpeed, + animationShowSkeletonLocalAxes, + animationGizmoEnabled, + animationGizmoType, + animationGizmoSelectedNode, + fixAnimations, + checkHierarchy, + retargetAnimationKeys, + fixRootPosition, + fixGroundReference, + fixGroundReferenceDynamicRefNode, + rootNodeName, + groundReferenceNodeName, + groundReferenceVerticalAxis, + }; + + // Save all UI state back to the persistent store on unmount so it survives re-docking. + useEffect(() => { + return () => { + Object.assign(stateStore, stateSnapshotRef.current); + }; + }, []); + + // Subscribe to isEnabled observable + useEffect(() => { + const obs = isEnabledObs.add((v) => setIsEnabled(v)); + return () => { + isEnabledObs.remove(obs); + }; + }, [isEnabledObs]); + + // Refresh dropdown options when the config dialog closes. + // If the current avatar/animation was removed, clear it and dispose the loaded model. + useEffect(() => { + const obs = onConfigChangedObs.add(() => { + const newAvatarOptions = avatarManager + .getAllAvatars() + .map((a) => ({ label: a.name, value: a.name })) + .sort((a, b) => a.label.localeCompare(b.label)); + const newAnimationOptions = animationManager + .getAllDisplayNames() + .map((n) => ({ label: n, value: n })) + .sort((a, b) => a.label.localeCompare(b.label)); + setAvatarOptions(newAvatarOptions); + setAnimationOptions(newAnimationOptions); + + const s = stateSnapshotRef.current; + if (s.avatarName && !avatarManager.getAvatar(s.avatarName)) { + const firstAvatar = newAvatarOptions.length > 0 ? newAvatarOptions[0].value : ""; + setAvatarName(firstAvatar); + if (firstAvatar) { + void handleLoadAvatarRef.current(firstAvatar, s.avatarRescaleAvatar); + } else { + managerRef.current?.avatar?.clearScene(); + managerRef.current?.avatar?.setGizmo(false, s.avatarGizmoType as GizmoType); + setAvatarGizmoEnabled(false); + setAvatarBoneOptions([]); + setAvatarGizmoSelectedTransform(null); + setIsAvatarLoaded(false); + setIsAvatarPlaying(false); + } + setIsRetargeted(false); + } + if (s.animationName && !animationManager.getByDisplayName(s.animationName)) { + const firstAnim = newAnimationOptions.length > 0 ? newAnimationOptions[0].value : ""; + setAnimationName(firstAnim); + if (firstAnim) { + void handleLoadAnimationRef.current(firstAnim); + } else { + managerRef.current?.animationSource?.clearScene(); + managerRef.current?.animationSource?.setGizmo(false, s.animationGizmoType as GizmoType); + setAnimationGizmoEnabled(false); + setRootNodeOptions([{ label: "Auto", value: "Auto" }]); + setGroundRefNodeOptions([]); + setAnimGizmoSelectedTransform(null); + setIsAnimLoaded(false); + setIsAnimPlaying(false); + } + setIsRetargeted(false); + } + }); + return () => { + onConfigChangedObs.remove(obs); + }; + }, [onConfigChangedObs, avatarManager, animationManager]); + + // Subscribe to manager ready observable -- re-subscribe each time a new manager is created + useEffect(() => { + let cleanup: (() => void) | null = null; + + /** + * Attaches observable listeners to `manager` and either triggers fresh loads + * (triggerLoads=true, first enable) or restores state from the existing scene + * (triggerLoads=false, panel remount after docking elsewhere). + * @param manager - The retargeting scene manager. + * @param triggerLoads - Whether to trigger fresh loads. + * @returns Void. + */ + const setupManager = (manager: RetargetingSceneManager, triggerLoads: boolean) => { + managerRef.current = manager; + + // Track console visibility + setIsConsoleVisible(manager.htmlConsole.isVisible); + const consoleObs = manager.htmlConsole.onVisibilityChangedObservable.add((v) => setIsConsoleVisible(v)); + + // Helper: rebuild "Root node" / "Ground ref. node" dropdowns using bone-remapping filtering. + // Mirrors original gui.ts _getSourceTransformNodeList + updateBoneList logic. + const rebuildBoneLists = ( + allNames: string[], + allOptions: { + /** + * + */ + label: string /** + * + */; + value: string; + }[], + animName: string, + avatarName: string, + currentGroundRef: string + ) => { + const filtered = BuildFilteredBoneList(allNames, animName, avatarName, namingSchemeManager, avatarManager, animationManager); + const filteredSet = new Set(filtered); + const filteredOptions = allOptions.filter((o) => filteredSet.has(o.value)); + setRootNodeOptions([{ label: "Auto", value: "Auto" }, ...filteredOptions]); + setGroundRefNodeOptions(filteredOptions); + const best = filtered.includes(currentGroundRef) + ? currentGroundRef + : (filtered.find((n) => n.toLowerCase().includes("lefttoe_end")) ?? filtered.find((n) => n.toLowerCase().includes("lefttoebase")) ?? filtered[0] ?? ""); + setGroundReferenceNodeName(best); + }; + + // Subscribe to Avatar observables + const avatarLoadObs = manager.avatar!.onLoadedObservable.add(() => { + const s = stateSnapshotRef.current; + setIsAvatarLoaded(true); + setIsRetargeted(false); + setIsLoading(false); + setIsAvatarPlaying(false); // new avatar starts in rest pose -- gizmo is usable + // Rebuild avatar bone list for the Avatar Gizmo "Selected node" dropdown + setAvatarBoneOptions(manager.avatar!.getBoneOptions()); + // Restore visual state + manager.avatar!.setSkeletonVisible(s.avatarShowSkeleton); + manager.avatar!.setSkeletonLocalAxes(s.avatarShowSkeletonLocalAxes); + manager.avatar!.setAnimSpeed(s.avatarAnimSpeed); + manager.avatar!.setGizmo(s.avatarGizmoEnabled, s.avatarGizmoType as GizmoType); + // Rebuild bone lists: avatar naming scheme affects the filtered list + if (manager.animationSource) { + rebuildBoneLists( + manager.animationSource.getTransformNodeNames(), + manager.animationSource.getTransformNodeOptions(), + s.animationName, + s.avatarName, + s.groundReferenceNodeName + ); + } + }); + + const avatarGizmoObs = manager.avatar!.onGizmoNodeSelectedObservable.add((name) => { + setAvatarGizmoSelectedNode(name); + setAvatarGizmoSelectedTransform(manager.avatar!.selectedBoneTransform as Transform | null); + }); + + const avatarPlayingObs = manager.avatar!.onPlayingObservable.add((playing) => { + setIsAvatarPlaying(playing); + if (playing) { + setAvatarGizmoEnabled(false); + } + const s = stateSnapshotRef.current; + manager.avatar!.setGizmo(playing ? false : s.avatarGizmoEnabled, s.avatarGizmoType as GizmoType); + }); + + // Subscribe to AnimationSource observables + const animLoadObs = manager.animationSource!.onLoadedObservable.add(() => { + const s = stateSnapshotRef.current; + setIsAnimLoaded(true); + setIsRetargeted(false); + setIsLoading(false); + setIsAnimPlaying(true); // animation always starts in "playing" state after load + // Return avatar to rest pose when a new animation is loaded + manager.avatar?.returnToRest(); + // Restore visual state + manager.animationSource!.setSkeletonVisible(true); + manager.animationSource!.setSkeletonLocalAxes(s.animationShowSkeletonLocalAxes); + manager.animationSource!.setGizmo(s.animationGizmoEnabled, s.animationGizmoType as GizmoType); + // Rebuild bone list dropdowns using filtered names (matching original updateBoneList logic) + const allNames = manager.animationSource!.getTransformNodeNames(); + rebuildBoneLists(allNames, manager.animationSource!.getTransformNodeOptions(), s.animationName, s.avatarName, stateSnapshotRef.current.groundReferenceNodeName); + }); + + const animGizmoObs = manager.animationSource!.onGizmoNodeSelectedObservable.add((name) => { + setAnimationGizmoSelectedNode(name); + setAnimGizmoSelectedTransform(manager.animationSource!.selectedTransformNode as Transform | null); + }); + + const animPlayingObs = manager.animationSource!.onPlayingObservable.add((playing) => { + setIsAnimPlaying(playing); + if (playing) { + setAnimationGizmoEnabled(false); + } + const s = stateSnapshotRef.current; + manager.animationSource!.setGizmo(playing ? false : s.animationGizmoEnabled, s.animationGizmoType as GizmoType); + }); + + // Subscribe to retarget done + const retargetObs = manager.onRetargetDoneObservable.add(() => { + setIsRetargeted(true); + setIsLoading(false); + }); + + if (triggerLoads) { + // Fresh enable: kick off loads from the current UI state + const s = stateSnapshotRef.current; + setIsLoading(true); + setIsAvatarLoaded(false); + setIsAnimLoaded(false); + setIsAvatarPlaying(false); + setIsAnimPlaying(false); + const storedAv = avatarManager.getAvatar(s.avatarName); + const animResult = animationManager.getByDisplayName(s.animationName); + const storedAn = animResult?.entry; + const animGroupIndex = animResult?.mapping.index ?? 0; + if (storedAv) { + if (storedAv.source === "scene") { + const pgScene = getCurrentScene(); + if (pgScene) { + const rootMesh = pgScene.getMeshByName(storedAv.rootNodeName); + if (rootMesh) { + void manager.avatar!.loadFromSceneAsync(pgScene, storedAv.rootNodeName, s.avatarRescaleAvatar, storedAv.restPoseUpdate); + } + } + } else if (storedAv.source === "url" && storedAv.url) { + void manager.avatar!.loadAsync(storedAv.url, s.avatarRescaleAvatar, storedAv.restPoseUpdate); + } else if (storedAv.source === "file" && storedAv.fileNames?.length) { + void (async () => { + const files = await avatarManager.getFilesAsync(storedAv.id, storedAv.fileNames!); + let sceneFile: File | undefined; + for (const file of files) { + const lowerName = file.name.toLowerCase(); + FilesInputStore.FilesToLoad[lowerName] = file; + const ext = lowerName.split(".").pop(); + if (ext && SceneLoader.IsPluginForExtensionAvailable("." + ext)) { + sceneFile = file; + } + } + if (sceneFile) { + void manager.avatar!.loadAsync("file:" + sceneFile.name, s.avatarRescaleAvatar, storedAv.restPoseUpdate); + } + })(); + } + } + if (storedAn) { + const loadAnimAsync = async (path: string) => { + await manager.animationSource!.loadAsync(path, storedAn.restPoseUpdate, storedAn.rootNodeName, animGroupIndex); + manager.animationSource?.setSkeletonVisible(true); + }; + if (storedAn.source === "scene") { + const pgScene = getCurrentScene(); + if (pgScene) { + const doSerializeAndLoadAsync = async () => { + const serialized = JSON.stringify(SceneSerializer.Serialize(pgScene)); + await loadAnimAsync("data:" + serialized); + }; + // If the avatar was transferred from the scene, temporarily restore it + // so the serializer captures the full scene + if (manager.avatar?.sceneTransfer.isActive) { + void manager.avatar.sceneTransfer.withRestoredSourceAsync(doSerializeAndLoadAsync); + } else { + void doSerializeAndLoadAsync(); + } + } + } else if (storedAn.source === "url" && storedAn.url) { + void loadAnimAsync(storedAn.url); + } else if (storedAn.source === "file" && storedAn.fileNames?.length) { + void (async () => { + const files = await animationManager.getFilesAsync(storedAn.id, storedAn.fileNames!); + let sceneFile: File | undefined; + for (const file of files) { + const lowerName = file.name.toLowerCase(); + FilesInputStore.FilesToLoad[lowerName] = file; + const ext = lowerName.split(".").pop(); + if (ext && SceneLoader.IsPluginForExtensionAvailable("." + ext)) { + sceneFile = file; + } + } + if (sceneFile) { + void loadAnimAsync("file:" + sceneFile.name); + } + })(); + } + } + } else { + // Remount: scene is already loaded -- restore UI state from the manager directly + const avatarLoaded = manager.avatar!.isLoaded; + const animLoaded = manager.animationSource!.isLoaded; + setIsAvatarLoaded(avatarLoaded); + setIsAnimLoaded(animLoaded); + setIsRetargeted(manager.isRetargeted); + setIsAvatarPlaying(manager.avatar!.isPlaying); + setIsAnimPlaying(manager.animationSource!.isPlaying); + setIsLoading(false); + if (avatarLoaded) { + setAvatarBoneOptions(manager.avatar!.getBoneOptions()); + setAvatarGizmoSelectedTransform(manager.avatar!.selectedBoneTransform as Transform | null); + } + if (animLoaded) { + const s = stateSnapshotRef.current; + rebuildBoneLists( + manager.animationSource!.getTransformNodeNames(), + manager.animationSource!.getTransformNodeOptions(), + s.animationName, + s.avatarName, + s.groundReferenceNodeName + ); + setAnimGizmoSelectedTransform(manager.animationSource!.selectedTransformNode as Transform | null); + } + } + + forceUpdate((n) => n + 1); + + return () => { + manager.avatar?.onLoadedObservable.remove(avatarLoadObs); + manager.avatar?.onGizmoNodeSelectedObservable.remove(avatarGizmoObs); + manager.avatar?.onPlayingObservable.remove(avatarPlayingObs); + manager.animationSource?.onLoadedObservable.remove(animLoadObs); + manager.animationSource?.onGizmoNodeSelectedObservable.remove(animGizmoObs); + manager.animationSource?.onPlayingObservable.remove(animPlayingObs); + manager.onRetargetDoneObservable.remove(retargetObs); + manager.htmlConsole.onVisibilityChangedObservable.remove(consoleObs); + }; + }; + + const obs = onManagerReadyObs.add((manager) => { + cleanup?.(); + cleanup = setupManager(manager, true); + }); + + // If a manager already exists (panel remounted after docking elsewhere), restore state now + const existingManager = getCurrentManager(); + if (existingManager) { + cleanup = setupManager(existingManager, false); + } + + return () => { + onManagerReadyObs.remove(obs); + cleanup?.(); + }; + }, [onManagerReadyObs]); + + const handleLoadAvatar = useCallback( + async (name: string, rescale: boolean) => { + const manager = managerRef.current; + if (!manager?.avatar) { + return; + } + const storedAvatar = avatarManager.getAvatar(name); + if (!storedAvatar) { + return; + } + setIsLoading(true); + setIsAvatarLoaded(false); + setIsRetargeted(false); + + if (storedAvatar.source === "scene") { + const pgScene = getCurrentScene(); + if (pgScene) { + void manager.avatar.loadFromSceneAsync(pgScene, storedAvatar.rootNodeName, rescale, storedAvatar.restPoseUpdate); + } + } else if (storedAvatar.source === "url" && storedAvatar.url) { + void manager.avatar.loadAsync(storedAvatar.url, rescale, storedAvatar.restPoseUpdate); + } else if (storedAvatar.source === "file" && storedAvatar.fileNames?.length) { + const files = await avatarManager.getFilesAsync(storedAvatar.id, storedAvatar.fileNames); + let sceneFile: File | undefined; + for (const file of files) { + const lowerName = file.name.toLowerCase(); + FilesInputStore.FilesToLoad[lowerName] = file; + const ext = lowerName.split(".").pop(); + if (ext && SceneLoader.IsPluginForExtensionAvailable("." + ext)) { + sceneFile = file; + } + } + if (sceneFile) { + void manager.avatar.loadAsync("file:" + sceneFile.name, rescale, storedAvatar.restPoseUpdate); + } + } + }, + [avatarManager, getCurrentScene] + ); + + const handleLoadAnimation = useCallback( + async (name: string) => { + const manager = managerRef.current; + if (!manager?.animationSource) { + return; + } + const animResult = animationManager.getByDisplayName(name); + const storedAnimation = animResult?.entry; + const animGroupIdx = animResult?.mapping.index ?? 0; + if (!storedAnimation) { + return; + } + setIsLoading(true); + setIsAnimLoaded(false); + setIsRetargeted(false); + + let loadPath: string | undefined; + if (storedAnimation.source === "scene") { + const pgScene = getCurrentScene(); + if (pgScene) { + const serializeFn = () => JSON.stringify(SceneSerializer.Serialize(pgScene)); + // If the avatar was transferred from the scene, temporarily restore it + const serialized = manager.avatar?.sceneTransfer.isActive ? await manager.avatar.sceneTransfer.withRestoredSourceAsync(serializeFn) : serializeFn(); + loadPath = "data:" + serialized; + } + } else if (storedAnimation.source === "url" && storedAnimation.url) { + loadPath = storedAnimation.url; + } else if (storedAnimation.source === "file" && storedAnimation.fileNames?.length) { + const files = await animationManager.getFilesAsync(storedAnimation.id, storedAnimation.fileNames); + let sceneFile: File | undefined; + for (const file of files) { + const lowerName = file.name.toLowerCase(); + FilesInputStore.FilesToLoad[lowerName] = file; + const ext = lowerName.split(".").pop(); + if (ext && SceneLoader.IsPluginForExtensionAvailable("." + ext)) { + sceneFile = file; + } + } + if (sceneFile) { + loadPath = "file:" + sceneFile.name; + } + } + + if (loadPath) { + await manager.animationSource.loadAsync(loadPath, storedAnimation.restPoseUpdate, storedAnimation.rootNodeName, animGroupIdx); + manager.animationSource!.setSkeletonVisible(true); + } + }, + [animationManager, getCurrentScene] + ); + + // Keep refs up to date so the config-changed handler can call the latest versions + handleLoadAvatarRef.current = handleLoadAvatar; + handleLoadAnimationRef.current = handleLoadAnimation; + + const handleRetarget = useCallback(() => { + const manager = managerRef.current; + if (!manager) { + return; + } + setIsLoading(true); + manager.htmlConsole.clear(); + manager.retarget( + { + avatarName, + avatarRescaleAvatar, + avatarAnimSpeed, + animationName, + animationSpeed, + fixAnimations, + checkHierarchy, + retargetAnimationKeys, + fixRootPosition, + fixGroundReference, + fixGroundReferenceDynamicRefNode, + rootNodeName, + groundReferenceNodeName, + groundReferenceVerticalAxis: groundReferenceVerticalAxis as "" | "X" | "Y" | "Z", + }, + namingSchemeManager, + avatarManager, + animationManager + ); + }, [ + avatarName, + avatarRescaleAvatar, + avatarAnimSpeed, + animationName, + animationSpeed, + fixAnimations, + checkHierarchy, + retargetAnimationKeys, + fixRootPosition, + fixGroundReference, + fixGroundReferenceDynamicRefNode, + rootNodeName, + groundReferenceNodeName, + groundReferenceVerticalAxis, + ]); + + const loadingText = isLoading && !isAvatarLoaded && !isAnimLoaded ? "Loading..." : isLoading ? "Loading..." : null; + + return ( +
+
+
+ {!isEnabled ? ( +
Extension is disabled -- original scene is shown
+ ) : ( + +
+ + { + setIsExporting(true); + void (async () => { + try { + await managerRef.current?.exportToPlaygroundAsync(avatarManager, animationManager, () => setIsExporting(false)); + } catch { + setIsExporting(false); + } + })(); + }} + disabled={!isRetargeted || isExporting} + /> + {getPlaygroundBridge() && ( + { + const bridge = getPlaygroundBridge(); + if (bridge) { + void (async () => { + // Read existing retargeting.ts to find existing function names + const existingContent = bridge.getFileContent("retargeting.ts"); + const existingFunctions = new Set(); + if (existingContent) { + const regex = /export\s+async\s+function\s+(retargetAnimation\d*)\s*\(/g; + let match; + while ((match = regex.exec(existingContent)) !== null) { + existingFunctions.add(match[1]); + } + } + + // Find the next available function name + let functionName = "retargetAnimation"; + if (existingFunctions.has(functionName)) { + let counter = 2; + while (existingFunctions.has(`retargetAnimation${counter}`)) { + counter++; + } + functionName = `retargetAnimation${counter}`; + } + + const result = await managerRef.current?.generateRetargetingCodeAsync(avatarManager, animationManager, functionName); + if (result) { + // Always write/update the helpers file + bridge.addFileTab("retarget.helpers.ts", result.helpersCode); + + // Create or append to retargeting.ts + let newContent: string; + if (existingContent) { + newContent = existingContent + result.functionCode; + } else { + newContent = result.headerCode + result.functionCode; + } + bridge.addFileTab("retargeting.ts", newContent); + + // Collect all function names now in the file + existingFunctions.add(functionName); + + // Add/update the import in the entry file + const entryPath = bridge.getEntryFilePath(); + const entryContent = bridge.getFileContent(entryPath) ?? ""; + const importLine = `import { ${[...existingFunctions].join(", ")} } from "./retargeting";`; + const importRegex = /import\s*\{[^}]*\}\s*from\s*["']\.\/retargeting["'];?\s*\n?/; + let updatedEntry: string; + if (importRegex.test(entryContent)) { + updatedEntry = entryContent.replace(importRegex, importLine + "\n"); + } else { + updatedEntry = importLine + "\n" + entryContent; + } + bridge.addFileTab(entryPath, updatedEntry); + } + })(); + } + }} + disabled={!isRetargeted} + /> + )} +
+
+ {loadingText &&
{loadingText}
} + + {/* Avatar */} + +
+
+ { + setAvatarName(name); + void handleLoadAvatar(name, avatarRescaleAvatar); + }} + /> +
+
+ { + setAvatarRescaleAvatar(v); + void handleLoadAvatar(avatarName, v); + }} + /> + { + setAvatarShowSkeleton(v); + managerRef.current?.avatar?.setSkeletonVisible(v); + }} + /> + { + setAvatarShowSkeletonLocalAxes(v); + managerRef.current?.avatar?.setSkeletonLocalAxes(v); + }} + /> + { + setAvatarAnimSpeed(v); + managerRef.current?.avatar?.setAnimSpeed(v); + }} + /> + managerRef.current?.avatar?.returnToRest()} + disabled={!isAvatarLoaded} + /> + { + const restPose = managerRef.current?.avatar?.saveAsRestPose(); + if (restPose && avatarName) { + const stored = avatarManager.getAvatar(avatarName); + if (stored) { + avatarManager.addAvatar({ ...stored, restPoseUpdate: restPose }); + } + } + }} + /> + managerRef.current?.avatar?.play(avatarAnimSpeed)} + disabled={!isRetargeted} + /> +
+ {/* Avatar Gizmo */} + + { + setAvatarGizmoEnabled(v); + managerRef.current?.avatar?.setGizmo(v, avatarGizmoType as GizmoType); + }} + /> + { + setAvatarGizmoType(v); + managerRef.current?.avatar?.setGizmo(avatarGizmoEnabled, v as GizmoType); + }} + /> +
+ Selected node +
+ { + setAvatarGizmoSelectedNode(name); + managerRef.current?.avatar?.attachGizmoToBone(name); + }} + /> +
+
+ {avatarGizmoSelectedTransform && ( + + + + )} +
+ {/* Animation */} + +
+
+ { + setAnimationName(name); + void handleLoadAnimation(name); + }} + /> +
+
+ { + setAnimationShowSkeletonLocalAxes(v); + managerRef.current?.animationSource?.setSkeletonLocalAxes(v); + }} + /> + { + setAnimationSpeed(v); + managerRef.current?.animationSource?.play(v); + }} + /> + managerRef.current?.animationSource?.returnToRest()} + disabled={!isAnimLoaded} + /> + { + const restPose = managerRef.current?.animationSource?.saveAsRestPose(); + if (restPose && animationName) { + const stored = animationManager.getByDisplayName(animationName)?.entry; + if (stored) { + animationManager.addAnimation({ ...stored, restPoseUpdate: restPose }); + } + } + }} + /> + managerRef.current?.animationSource?.play(animationSpeed)} + disabled={!isAnimLoaded} + /> +
+ {/* Animation Gizmo */} + + { + setAnimationGizmoEnabled(v); + managerRef.current?.animationSource?.setGizmo(v, animationGizmoType as GizmoType); + }} + /> + { + setAnimationGizmoType(v); + managerRef.current?.animationSource?.setGizmo(animationGizmoEnabled, v as GizmoType); + }} + /> +
+ Selected node +
+ { + setAnimationGizmoSelectedNode(name); + managerRef.current?.animationSource?.attachGizmoToTransformNode(name); + }} + /> +
+
+ {animGizmoSelectedTransform && ( + + + + )} +
+ {/* Retarget Options */} + + + + + + +
+ +
+
+ Root node +
+ +
+
+ {rootNodeName !== "Auto" && rootNodeName === groundReferenceNodeName && ( + Root node and Ground ref. node must be different. + )} +
+ Ground ref. node +
+ +
+
+
+ +
+
+
+
+
+ )} +
+ ); +}; diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingViewport.tsx b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingViewport.tsx new file mode 100644 index 00000000000..7e961582105 --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationRetargetingViewport.tsx @@ -0,0 +1,81 @@ +import { type FunctionComponent, useEffect, useRef } from "react"; + +import { type Engine } from "core/Engines/engine"; +import { makeStyles, tokens, Body1 } from "@fluentui/react-components"; + +import { RetargetingSceneManager } from "./retargetingSceneManager"; + +const useStyles = makeStyles({ + container: { + position: "absolute", + inset: "0", + overflow: "hidden", + pointerEvents: "none", + }, + labels: { + position: "absolute", + top: "8px", + left: 0, + width: "100%", + display: "flex", + justifyContent: "space-around", + pointerEvents: "none", + color: "rgba(255,255,255,0.7)", + fontSize: tokens.fontSizeBase200, + fontFamily: "sans-serif", + }, + consoleContainer: { + pointerEvents: "auto", + }, +}); + +type AnimationRetargetingViewportProps = { + engine: Engine; + onManagerReady: (manager: RetargetingSceneManager) => void; +}; + +/** + * Overlay component for the animation retargeting viewport. + * @returns The React element. + */ +export const AnimationRetargetingViewport: FunctionComponent = ({ engine, onManagerReady }) => { + const classes = useStyles(); + const containerRef = useRef(null); + const managerRef = useRef(null); + + useEffect(() => { + if (!containerRef.current) { + return; + } + + const manager = new RetargetingSceneManager(); + managerRef.current = manager; + manager.initialize(engine); + manager.htmlConsole.attachToContainer(containerRef.current); + onManagerReady(manager); + + // Resize the engine when the central content area changes size + const resizeObserver = new ResizeObserver(() => engine.resize()); + resizeObserver.observe(containerRef.current); + + return () => { + resizeObserver.disconnect(); + try { + manager.dispose(); + } catch { + // Ignore errors during dispose — the engine/scene may already be partially disposed + } + managerRef.current = null; + }; + // onManagerReady is intentionally not in deps - it's a stable callback from the service + }, [engine]); + + return ( +
+
+ ◀ Avatar + Animation ▶ +
+
+ ); +}; diff --git a/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationsPanel.tsx b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationsPanel.tsx new file mode 100644 index 00000000000..50d9124988e --- /dev/null +++ b/packages/dev/inspector-v2/src/extensions/animationRetargeting/animationsPanel.tsx @@ -0,0 +1,1032 @@ +import { type FunctionComponent, useState, useCallback, useRef, useEffect } from "react"; +import { type AnimationManager, type StoredAnimation, type AnimationGroupMapping } from "./animationManager"; +import { type NamingSchemeManager } from "./namingSchemeManager"; +import { type RestPoseDataUpdate } from "./avatarManager"; +import { type Nullable } from "core/types"; + +import { + Input, + Label, + makeStyles, + mergeClasses, + tokens, + Body1Strong, + Caption1, + Dialog, + DialogSurface, + DialogBody, + DialogTitle, + DialogContent, + DialogActions, + Spinner, + DataGrid, + DataGridHeader, + DataGridBody, + DataGridRow, + DataGridHeaderCell, + DataGridCell, + createTableColumn, + type TableColumnDefinition, + type TableColumnSizingOptions, +} from "@fluentui/react-components"; + +import { Button } from "shared-ui-components/fluent/primitives/button"; +import { TextInput } from "shared-ui-components/fluent/primitives/textInput"; +import { StringDropdown } from "shared-ui-components/fluent/primitives/dropdown"; +import { AddRegular, DeleteRegular, EditRegular, ArrowUploadRegular, DocumentArrowLeftRegular } from "@fluentui/react-icons"; +import { Textarea } from "shared-ui-components/fluent/primitives/textarea"; +import { NullEngine } from "core/Engines/nullEngine"; +import { Scene } from "core/scene"; +import { ImportMeshAsync, SceneLoader } from "core/Loading/sceneLoader"; +import { FilesInputStore } from "core/Misc/filesInputStore"; +import { type Node } from "core/node"; + +// ─── Styles ─────────────────────────────────────────────────────────────────── + +const useStyles = makeStyles({ + panel: { + flex: 1, + overflow: "hidden", + display: "flex", + flexDirection: "column", + gap: tokens.spacingVerticalS, + }, + listHeader: { + display: "flex", + justifyContent: "space-between", + alignItems: "center", + flexShrink: 0, + paddingBottom: tokens.spacingVerticalXS, + }, + listButtons: { + display: "flex", + gap: tokens.spacingHorizontalXS, + }, + editSectionFlex: { + flex: 1, + overflowY: "auto", + borderTop: `1px solid ${tokens.colorNeutralStroke2}`, + paddingTop: tokens.spacingVerticalS, + display: "flex", + flexDirection: "column", + gap: tokens.spacingVerticalS, + }, + formRow: { + display: "flex", + alignItems: "center", + gap: tokens.spacingHorizontalS, + flexShrink: 0, + }, + formLabel: { + flexShrink: 0, + width: "110px", + }, + formControl: { + flex: 1, + textAlign: "left", + "& span": { textAlign: "left" }, + "& input": { textAlign: "left" }, + }, + actionRow: { + display: "flex", + gap: tokens.spacingHorizontalS, + justifyContent: "flex-end", + flexShrink: 0, + }, + errorText: { + color: tokens.colorPaletteRedForeground1, + fontSize: tokens.fontSizeBase200, + flexShrink: 0, + }, + warningText: { + color: tokens.colorPaletteYellowForeground1, + fontSize: tokens.fontSizeBase200, + flexShrink: 0, + }, + emptyMsg: { + padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalS}`, + color: tokens.colorNeutralForeground3, + fontSize: tokens.fontSizeBase200, + }, + confirmSurface: { + width: "380px", + maxWidth: "90vw", + }, + restPoseTextarea: { + resize: "vertical", + width: "100%", + minHeight: "150px", + maxHeight: "300px", + boxSizing: "border-box", + border: `1px solid ${tokens.colorNeutralStroke1}`, + borderRadius: tokens.borderRadiusMedium, + padding: `${tokens.spacingVerticalSNudge} ${tokens.spacingHorizontalS}`, + fontFamily: "monospace", + fontSize: tokens.fontSizeBase100, + backgroundColor: tokens.colorNeutralBackground1, + color: tokens.colorNeutralForeground1, + }, + dropZone: { + border: `2px dashed ${tokens.colorNeutralStroke1}`, + borderRadius: tokens.borderRadiusMedium, + padding: tokens.spacingVerticalM, + textAlign: "center", + cursor: "pointer", + color: tokens.colorNeutralForeground3, + fontSize: tokens.fontSizeBase200, + }, + dropZoneActive: { + border: `2px dashed ${tokens.colorCompoundBrandStroke}`, + backgroundColor: tokens.colorBrandBackground2, + }, + animList: { + flex: 1, + overflowY: "auto", + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + minHeight: "60px", + }, + animRow: { + display: "flex", + alignItems: "center", + padding: `${tokens.spacingVerticalXXS} ${tokens.spacingHorizontalS}`, + gap: tokens.spacingHorizontalS, + fontSize: tokens.fontSizeBase200, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + ":last-child": { borderBottom: "none" }, + }, + animRowName: { + flex: "0 0 40%", + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", + color: tokens.colorNeutralForeground3, + }, + animRowInput: { + flex: 1, + }, + animRowError: { + color: tokens.colorPaletteRedForeground1, + fontSize: tokens.fontSizeBase100, + flexShrink: 0, + }, + nodeTree: { + flex: 1, + overflowY: "auto", + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusMedium, + minHeight: "60px", + maxHeight: "150px", + }, + nodeRow: { + padding: `0 ${tokens.spacingHorizontalS}`, + fontSize: tokens.fontSizeBase200, + lineHeight: "13px", + fontFamily: "monospace", + cursor: "pointer", + whiteSpace: "pre", + }, + nodeRowSelected: { + backgroundColor: tokens.colorBrandBackground2, + fontWeight: "bold", + }, + dataGridFlex: { + flex: 1, + overflowY: "auto", + }, + dataGridCompact: { + maxHeight: "140px", + overflowY: "auto", + }, + hiddenInput: { + display: "none", + }, + subtleText: { + color: tokens.colorNeutralForeground3, + }, + spinnerInline: { + display: "inline-block", + }, + formRowAlignStart: { + alignItems: "flex-start", + }, + formLabelPadTop: { + paddingTop: "6px", + }, +}); + +// ─── Types ──────────────────────────────────────────────────────────────────── + +type NodeInfo = { + name: string; + depth: number; + prefix: string; +}; + +type AnimationEdit = { + id: string | null; + name: string; + sourceType: "url" | "file" | "scene"; + url: string; + files: File[]; + /** One row per animation group found in the file. */ + mappings: AnimationGroupMapping[]; + /** Index into the nodeList array — used for unique selection in the dialog. */ + rootNodeIndex: number; + /** Display name of the root node. */ + rootNodeName: string; + namingScheme: string; + restPoseJson: string; + sessionOnly?: boolean; +}; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function BuildNodeList(node: Node, depth: number, list: NodeInfo[], ancestorContinues: boolean[]): void { + let prefix = ""; + if (depth > 0) { + for (let i = 1; i < depth; i++) { + prefix += ancestorContinues[i] ? "│ " : " "; + } + const isLast = !ancestorContinues[depth]; + prefix += isLast ? "└─ " : "├─ "; + } + list.push({ name: node.name, depth, prefix }); + const children = node.getChildren(); + for (let i = 0; i < children.length; i++) { + const isLastChild = i === children.length - 1; + ancestorContinues[depth + 1] = !isLastChild; + BuildNodeList(children[i], depth + 1, list, ancestorContinues); + } +} + +/** + * Detects the best matching naming scheme by checking how many target names + * appear in each scheme. Returns the scheme with the most matches + * (minimum 10 required), or null if none qualify. + * @param targetNames - Set of animation target names. + * @param namingSchemeManager - The naming scheme manager. + * @returns The best matching scheme name, or null. + */ +function DetectNamingScheme(targetNames: Set, namingSchemeManager: NamingSchemeManager): string | null { + const schemeNames = namingSchemeManager.getAllSchemeNames(); + let bestScheme: string | null = null; + let bestCount = 0; + + for (const schemeName of schemeNames) { + const entries = namingSchemeManager.getNamingScheme(schemeName); + if (!entries) { + continue; + } + const boneSet = new Set(entries.map((e) => e.name)); + let matches = 0; + for (const name of targetNames) { + if (boneSet.has(name)) { + matches++; + } + } + if (matches > bestCount) { + bestCount = matches; + bestScheme = schemeName; + } + } + + return bestCount >= 10 ? bestScheme : null; +} + +// ─── Component ──────────────────────────────────────────────────────────────── + +/** + * Animations tab panel for the retargeting config dialog. + * @returns The React element. + */ +export const AnimationsPanel: FunctionComponent<{ + animationManager: AnimationManager; + namingSchemeManager: NamingSchemeManager; + getCurrentScene: () => Nullable; + onMutate: () => void; + onEditingChange: (editing: boolean) => void; +}> = ({ animationManager, namingSchemeManager, getCurrentScene, onMutate, onEditingChange }) => { + const classes = useStyles(); + const [editing, setEditing] = useState(null); + const [error, setError] = useState(null); + const [warning, setWarning] = useState(null); + const [confirmDelete, setConfirmDelete] = useState<{ + id: string /** + * + */; + label: string; + } | null>(null); + const [isLoading, setIsLoading] = useState(false); + const [isDragOver, setIsDragOver] = useState(false); + const [nodeList, setNodeList] = useState([]); + + const tempEngineRef = useRef(null); + const tempSceneRef = useRef(null); + const fileInputRef = useRef(null); + + useEffect(() => { + return () => { + tempSceneRef.current?.dispose(); + tempEngineRef.current?.dispose(); + }; + }, []); + + const setEditingWithNotify = useCallback( + (value: AnimationEdit | null) => { + setEditing(value); + onEditingChange(value !== null); + }, + [onEditingChange] + ); + + const allAnimations = [...animationManager.getAllAnimations()].sort((a, b) => a.name.localeCompare(b.name)); + const schemeNames = namingSchemeManager.getAllSchemeNames(); + + const animationColumnSizing: TableColumnSizingOptions = { + name: { defaultWidth: 150 }, + source: { defaultWidth: 60 }, + scheme: { defaultWidth: 150 }, + actions: { defaultWidth: 80 }, + }; + + const animationColumns: TableColumnDefinition[] = [ + createTableColumn({ columnId: "name", renderHeaderCell: () => "Name", renderCell: (item) => item.name }), + createTableColumn({ + columnId: "source", + renderHeaderCell: () => "Source", + renderCell: (item) => + item.sessionOnly + ? `${item.source === "scene" ? "Scene" : item.source === "url" ? "URL" : "File"} (session)` + : item.source === "scene" + ? "Scene" + : item.source === "url" + ? "URL" + : "File", + }), + createTableColumn({ columnId: "scheme", renderHeaderCell: () => "Scheme", renderCell: (item) => item.namingScheme }), + createTableColumn({ + columnId: "actions", + renderHeaderCell: () => "", + renderCell: (item) => ( + <> +