Build on modern macOS (Apple Silicon) + Metal depth backend

CMakeLists.txt

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12.1)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
 
 SET(PROJECT_VER_MAJOR 0)
 SET(PROJECT_VER_MINOR 2)
@@ -41,6 +41,11 @@ OPTION(ENABLE_OPENGL "Enable OpenGL support" ON)
 OPTION(ENABLE_VAAPI "Enable VA-API support" ON)
 OPTION(ENABLE_TEGRAJPEG "Enable Tegra HW JPEG support" ON)
 OPTION(ENABLE_PROFILING "Collect profiling stats (memory consuming)" OFF)
+IF(APPLE)
+  OPTION(ENABLE_METAL "Enable Metal GPU depth processing (Apple platforms)" ON)
+ELSE()
+  OPTION(ENABLE_METAL "Enable Metal GPU depth processing (Apple platforms)" OFF)
+ENDIF()
 
 IF(ENABLE_PROFILING)
   SET(LIBFREENECT2_WITH_PROFILING 1)
@@ -165,9 +170,22 @@ SET(LIBFREENECT2_DLLS
   ${LibUSB_DLL}
 )
 
+# VideoToolbox's VTRgbPacketProcessor causes a hard crash (heap corruption in the
+# threading model) on Apple Silicon, so default it OFF there and fall back to
+# TurboJPEG for RGB decoding. It remains ON for Intel Macs where it works.
+IF(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
+  OPTION(ENABLE_VIDEOTOOLBOX "Enable VideoToolbox RGB decoder (crashes on Apple Silicon / M1+, use OFF to force TurboJPEG)" OFF)
+ELSE()
+  OPTION(ENABLE_VIDEOTOOLBOX "Enable VideoToolbox RGB decoder (crashes on Apple Silicon / M1+, use OFF to force TurboJPEG)" ON)
+ENDIF()
+
 SET(HAVE_VideoToolbox "no (Apple only)")
 IF(APPLE)
-  FIND_LIBRARY(VIDEOTOOLBOX_LIBRARY VideoToolbox)
+  IF(ENABLE_VIDEOTOOLBOX)
+    FIND_LIBRARY(VIDEOTOOLBOX_LIBRARY VideoToolbox)
+  ELSE()
+    MESSAGE(STATUS "VideoToolbox disabled by ENABLE_VIDEOTOOLBOX=OFF — using TurboJPEG for RGB (required on Apple Silicon / M1+)")
+  ENDIF()
 
   SET(HAVE_VideoToolbox no)
   IF(VIDEOTOOLBOX_LIBRARY)
@@ -337,6 +355,57 @@ IF(ENABLE_OPENCL)
   ENDIF(OpenCL_FOUND)
 ENDIF(ENABLE_OPENCL)
 
+SET(HAVE_Metal "disabled (non-Apple)")
+IF(APPLE AND ENABLE_METAL)
+  SET(HAVE_Metal no)
+
+  SET(METAL_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/metal_depth_packet_processor.metal")
+  SET(METAL_AIR    "${CMAKE_CURRENT_BINARY_DIR}/metal_depth_packet_processor.air")
+  # Emit the metallib into the library output dir so it sits next to
+  # libfreenect2.dylib — the runtime loader locates it relative to the dylib
+  # (via dladdr), so this makes both the build tree and the install work.
+  SET(METAL_LIB    "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/default.metallib")
+
+  ADD_CUSTOM_COMMAND(
+    OUTPUT ${METAL_AIR}
+    COMMAND xcrun -sdk macosx metal -c ${METAL_SOURCE} -o ${METAL_AIR}
+    DEPENDS ${METAL_SOURCE}
+    COMMENT "Compiling Metal shader to AIR"
+    VERBATIM
+  )
+
+  ADD_CUSTOM_COMMAND(
+    OUTPUT ${METAL_LIB}
+    COMMAND xcrun -sdk macosx metallib ${METAL_AIR} -o ${METAL_LIB}
+    DEPENDS ${METAL_AIR}
+    COMMENT "Linking Metal library"
+    VERBATIM
+  )
+
+  ADD_CUSTOM_TARGET(MetalShaders ALL DEPENDS ${METAL_LIB})
+
+  SET(LIBFREENECT2_WITH_METAL_SUPPORT 1)
+  SET(HAVE_Metal yes)
+
+  LIST(APPEND SOURCES
+    src/metal_depth_packet_processor.mm
+  )
+
+  LIST(APPEND LIBRARIES
+    "-framework Metal"
+    "-framework Foundation"
+  )
+
+  # Mark the .mm file for Objective-C++ compilation (CMake handles this via the
+  # extension, but the explicit property guards against edge cases).
+  SET_SOURCE_FILES_PROPERTIES(src/metal_depth_packet_processor.mm
+    PROPERTIES COMPILE_FLAGS "-x objective-c++"
+  )
+
+  # Install the compiled metallib alongside the dylib.
+  INSTALL(FILES ${METAL_LIB} DESTINATION lib)
+ENDIF(APPLE AND ENABLE_METAL)
+
 SET(HAVE_CUDA disabled)
 IF(ENABLE_CUDA)
   FIND_PACKAGE(CUDA)
@@ -417,6 +486,11 @@ GENERATE_RESOURCES(${RESOURCES_INC_FILE} ${MY_DIR} ${RESOURCES})
 
 ADD_DEFINITIONS(-DRESOURCES_INC)
 ADD_LIBRARY(freenect2 ${SOURCES})
+
+IF(APPLE AND ENABLE_METAL AND LIBFREENECT2_WITH_METAL_SUPPORT)
+  ADD_DEPENDENCIES(freenect2 MetalShaders)
+ENDIF()
+
 SET_TARGET_PROPERTIES(freenect2 PROPERTIES
   CXX_VISIBILITY_PRESET hidden
   VISIBILITY_INLINES_HIDDEN 1

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12.1)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
 
 if(WIN32 AND NOT MINGW)
   if(NOT DEFINED CMAKE_DEBUG_POSTFIX)

examples/Protonect.cpp

@@ -113,7 +113,7 @@ int main(int argc, char *argv[])
   std::string program_path(argv[0]);
   std::cerr << "Version: " << LIBFREENECT2_VERSION << std::endl;
   std::cerr << "Environment variables: LOGFILE=<protonect.log>" << std::endl;
-  std::cerr << "Usage: " << program_path << " [-gpu=<id>] [gl | cl | clkde | cuda | cudakde | cpu] [<device serial>]" << std::endl;
+  std::cerr << "Usage: " << program_path << " [-gpu=<id>] [gl | cl | clkde | cuda | cudakde | metal | cpu] [<device serial>]" << std::endl;
   std::cerr << "        [-noviewer] [-norgb | -nodepth] [-help] [-version]" << std::endl;
   std::cerr << "        [-frames <number of frames to process>]" << std::endl;
   std::cerr << "To pause and unpause: pkill -USR1 Protonect" << std::endl;
@@ -225,6 +225,15 @@ int main(int argc, char *argv[])
         pipeline = new libfreenect2::CudaKdePacketPipeline(deviceId);
 #else
       std::cout << "CUDA pipeline is not supported!" << std::endl;
+#endif
+    }
+    else if(arg == "metal")
+    {
+#ifdef LIBFREENECT2_WITH_METAL_SUPPORT
+      if(!pipeline)
+        pipeline = new libfreenect2::MetalPacketPipeline(deviceId);
+#else
+      std::cout << "Metal pipeline is not supported!" << std::endl;
 #endif
     }
     else if(arg.find_first_not_of("0123456789") == std::string::npos) //check if parameter could be a serial number

include/internal/libfreenect2/depth_packet_processor.h

@@ -280,6 +280,31 @@ class CudaKdeDepthPacketProcessor : public DepthPacketProcessor
 };
 #endif // LIBFREENECT2_WITH_CUDA_SUPPORT
 
+#ifdef LIBFREENECT2_WITH_METAL_SUPPORT
+class MetalDepthPacketProcessorImpl;
+
+/** Depth packet processor using Apple Metal GPU compute. */
+class MetalDepthPacketProcessor : public DepthPacketProcessor
+{
+public:
+  MetalDepthPacketProcessor(const int deviceIndex = -1);
+  virtual ~MetalDepthPacketProcessor();
+  virtual void setConfiguration(const libfreenect2::DepthPacketProcessor::Config &config);
+
+  virtual void loadP0TablesFromCommandResponse(unsigned char *buffer, size_t buffer_length);
+
+  virtual void loadXZTables(const float *xtable, const float *ztable);
+  virtual void loadLookupTable(const short *lut);
+
+  virtual bool good();
+  virtual const char *name() { return "Metal"; }
+
+  virtual void process(const DepthPacket &packet);
+private:
+  MetalDepthPacketProcessorImpl *impl_;
+};
+#endif // LIBFREENECT2_WITH_METAL_SUPPORT
+
 class DumpDepthPacketProcessor : public DepthPacketProcessor
 {
  public:

include/libfreenect2/config.h.in

@@ -44,6 +44,8 @@
 #cmakedefine LIBFREENECT2_WITH_OPENCL_SUPPORT
 #cmakedefine LIBFREENECT2_OPENCL_ICD_LOADER_IS_OLD
 
+#cmakedefine LIBFREENECT2_WITH_METAL_SUPPORT
+
 #cmakedefine LIBFREENECT2_WITH_CUDA_SUPPORT
 
 #cmakedefine LIBFREENECT2_WITH_VT_SUPPORT

include/libfreenect2/packet_pipeline.h

@@ -156,6 +156,18 @@ class LIBFREENECT2_API CudaKdePacketPipeline : public PacketPipeline
 };
 #endif // LIBFREENECT2_WITH_CUDA_SUPPORT
 
+#ifdef LIBFREENECT2_WITH_METAL_SUPPORT
+/** Pipeline with Metal GPU depth processing (Apple platforms). */
+class LIBFREENECT2_API MetalPacketPipeline : public PacketPipeline
+{
+protected:
+  const int deviceId;
+public:
+  MetalPacketPipeline(const int deviceId = -1);
+  virtual ~MetalPacketPipeline();
+};
+#endif // LIBFREENECT2_WITH_METAL_SUPPORT
+
 ///@}
 } /* namespace libfreenect2 */
 #endif /* PACKET_PIPELINE_H_ */

src/libfreenect2.cpp

@@ -1101,6 +1101,10 @@ PacketPipeline *createPacketPipelineByName(std::string name)
 #if defined(LIBFREENECT2_WITH_OPENCL_SUPPORT)
   if (name == "cl")
     return new OpenCLPacketPipeline();
+#endif
+#if defined(LIBFREENECT2_WITH_METAL_SUPPORT)
+  if (name == "metal")
+    return new MetalPacketPipeline();
 #endif
   if (name == "cpu")
     return new CpuPacketPipeline();
@@ -1123,6 +1127,8 @@ PacketPipeline *createDefaultPacketPipeline()
   return new OpenGLPacketPipeline();
 #elif defined(LIBFREENECT2_WITH_CUDA_SUPPORT)
   return new CudaPacketPipeline();
+#elif defined(LIBFREENECT2_WITH_METAL_SUPPORT)
+  return new MetalPacketPipeline();
 #elif defined(LIBFREENECT2_WITH_OPENCL_SUPPORT)
   return new OpenCLPacketPipeline();
 #else