Fix small-marker detection under downsampling (AKAZE) — #53

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp

@@ -1,7 +1,8 @@
 #include <WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h>
 
-extern const double DEFAULT_NN_MATCH_RATIO = 0.7f;
-extern const double TEBLID_NN_MATCH_RATIO = 0.8f;
+extern const double DEFAULT_NN_MATCH_RATIO = 0.7;
+extern const double TEBLID_NN_MATCH_RATIO = 0.8;
+extern const double AKAZE_NN_MATCH_RATIO = 0.8; ///< artoolkitX OCVT parity (OCVConfig nn_match_ratio).
 extern const int DEFAULT_MAX_FEATURES = 800;
 extern const int TEBLID_MAX_FEATURES = 1000;
 extern const int N = 10;

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp

@@ -60,8 +60,11 @@ class WebARKitTracker::WebARKitTrackerImpl {
         if (trackerType == webarkit::TEBLID_TRACKER) {
             _nn_match_ratio = TEBLID_NN_MATCH_RATIO;
         } else if (trackerType == webarkit::AKAZE_TRACKER) {
-            _nn_match_ratio = DEFAULT_NN_MATCH_RATIO;
-            minNumMatches = 40;
+            // WebARKitLib#53: align with artoolkitX OCVT -- nn_match_ratio 0.8 (not 0.7)
+            // and a lower minNumMatches floor. A small marker yields few matches after
+            // downsampling; the previous values (0.7 / 40) rejected nearly all of them.
+            _nn_match_ratio = AKAZE_NN_MATCH_RATIO;
+            minNumMatches = 15;
         } else {
             _nn_match_ratio = DEFAULT_NN_MATCH_RATIO;
             minNumMatches = 15;
@@ -368,24 +371,49 @@ class WebARKitTracker::WebARKitTrackerImpl {
             // (frame <= featureImageMinSize, e.g. 640x480) detectionFrame == frame, so the
             // path is identical to full-res detection.
             cv::Mat detectionFrame;
+            cv::Vec2f detectionScaleFactor;
             if (_featureDetectPyrLevel < 1) {
                 detectionFrame = frame;
+                detectionScaleFactor = cv::Vec2f(1.0f, 1.0f);
             } else {
                 cv::Mat srcFrame = frame;
                 for (int pyrLevel = 1; pyrLevel <= _featureDetectPyrLevel; pyrLevel++) {
                     cv::pyrDown(srcFrame, detectionFrame, cv::Size(0, 0));
                     srcFrame = detectionFrame;
                 }
+                detectionScaleFactor = _featureDetectScaleFactor;
             }
 
             cv::Mat featureMask = createFeatureMask(detectionFrame);
 
             if (!extractFeatures(detectionFrame, featureMask, frameKeyPts, frameDescr)) {
                 WEBARKIT_LOGe("No features detected in extractFeatures!\n");
             }
-            WEBARKIT_LOGd("frame KeyPoints size: %d\n", frameKeyPts.size());
+            WEBARKIT_LOGd("frame KeyPoints size: %d (pyrLevel %d)\n", frameKeyPts.size(),
+                          (int)_featureDetectPyrLevel);
+
+            // WebARKitLib#53: a small marker in a large frame can fall below the detector
+            // threshold after pyrDown (too few keypoints to attempt a match). When that
+            // happens and the frame was downsampled, retry once on the full-resolution
+            // frame before giving up. This preserves the #44 fast path for markers that
+            // survive downsampling while restoring small-marker detection parity with
+            // WebARKitLib-rs / jsartoolkitNFT (which never downsample).
+            if (static_cast<int>(frameKeyPts.size()) <= minRequiredDetectedFeatures && _featureDetectPyrLevel > 0) {
+                WEBARKIT_LOGd("Too few keypoints after pyrDown (%d <= %d); retrying at full resolution.\n",
+                              frameKeyPts.size(), (int)minRequiredDetectedFeatures);
+                detectionFrame = frame;
+                detectionScaleFactor = cv::Vec2f(1.0f, 1.0f);
+                featureMask = createFeatureMask(detectionFrame);
+                frameKeyPts.clear();
+                frameDescr.release();
+                if (!extractFeatures(detectionFrame, featureMask, frameKeyPts, frameDescr)) {
+                    WEBARKIT_LOGe("No features detected in full-resolution extractFeatures!\n");
+                }
+                WEBARKIT_LOGd("frame KeyPoints size (full-res retry): %d\n", frameKeyPts.size());
+            }
+
             if (static_cast<int>(frameKeyPts.size()) > minRequiredDetectedFeatures) {
-                MatchFeatures(frameKeyPts, frameDescr);
+                MatchFeatures(frameKeyPts, frameDescr, detectionScaleFactor);
             }
         }
         int i = 0;
@@ -480,7 +508,8 @@ class WebARKitTracker::WebARKitTrackerImpl {
 
     void swapImagePyramid() { _pyramid.swap(_prevPyramid); }
 
-    void MatchFeatures(const std::vector<cv::KeyPoint>& newFrameFeatures, cv::Mat newFrameDescriptors) {
+    void MatchFeatures(const std::vector<cv::KeyPoint>& newFrameFeatures, cv::Mat newFrameDescriptors,
+                       const cv::Vec2f& scaleFactor) {
         int maxMatches = 0;
         int bestMatchIndex = -1;
         std::vector<cv::KeyPoint> finalMatched1, finalMatched2;
@@ -518,14 +547,14 @@ class WebARKitTracker::WebARKitTrackerImpl {
         // } // end for cycle
 
         if (maxMatches > 0) {
-            // WebARKitLib#44: detection ran on the downsampled detectionFrame, so the
-            // matched FRAME keypoints (finalMatched1) are in downsampled coordinates --
+            // WebARKitLib#44: detection may run on the downsampled detectionFrame, so the
+            // matched FRAME keypoints (finalMatched1) are in that frame's coordinates --
             // scale them back up to full-frame coordinates before fitting the
-            // homography. Level 0 => factor 1.0 => no-op. The reference keypoints
-            // (finalMatched2) stay in reference coordinates.
+            // homography. Identity factor (level 0, or the #53 full-res retry) => no-op.
+            // The reference keypoints (finalMatched2) stay in reference coordinates.
             for (size_t i = 0; i < finalMatched1.size(); i++) {
-                finalMatched1[i].pt.x *= _featureDetectScaleFactor[0];
-                finalMatched1[i].pt.y *= _featureDetectScaleFactor[1];
+                finalMatched1[i].pt.x *= scaleFactor[0];
+                finalMatched1[i].pt.y *= scaleFactor[1];
             }
             homography::WebARKitHomographyInfo homoInfo =
                 getHomographyInliers(Points(finalMatched2), Points(finalMatched1));
@@ -814,7 +843,12 @@ class WebARKitTracker::WebARKitTrackerImpl {
     void setDetectorType(webarkit::TRACKER_TYPE trackerType) {
         _trackerType = trackerType;
         if (trackerType == webarkit::TRACKER_TYPE::AKAZE_TRACKER) {
-            const double akaze_thresh = 3e-4; // AKAZE detection threshold set to locate about 1000 keypoints
+            // WebARKitLib#53: use the artoolkitX OCVT default threshold (0.001) instead of
+            // 3e-4. This is the minimum detector response a keypoint must have to be
+            // accepted, so it is stricter than 3e-4, not looser -- but it matches the
+            // value artoolkitX uses in production and was validated to restore detection
+            // on small markers after downsampling.
+            const double akaze_thresh = 1e-3; // AKAZE detection threshold (artoolkitX OCVT default)
             cv::Ptr<cv::AKAZE> akaze = cv::AKAZE::create();
             akaze->setThreshold(akaze_thresh);
             this->_featureDetector = akaze;

WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h

@@ -10,6 +10,7 @@
 
 extern const double DEFAULT_NN_MATCH_RATIO;
 extern const double TEBLID_NN_MATCH_RATIO;
+extern const double AKAZE_NN_MATCH_RATIO;
 extern const int DEFAULT_MAX_FEATURES;
 extern const int TEBLID_MAX_FEATURES;
 extern const int N;

WebARKit/WebARKitVideoLuma.cpp

@@ -0,0 +1,118 @@
+#include <WebARKitVideoLuma.h>
+
+namespace webarkit {
+
+WebARKitLumaInfo *webarkitVideoLumaInit(int xsize, int ysize, bool simd128) {
+  WebARKitLumaInfo *vli = new WebARKitLumaInfo;
+
+  if (!vli) {
+    webarkitLOGe("Out of memory!!");
+    return nullptr;
+  }
+  vli->xsize = xsize;
+  vli->ysize = ysize;
+  vli->buffSize = xsize * ysize;
+  vli->simd128 = simd128;
+  vli->buff = std::make_unique<uint8_t[]>(vli->buffSize);
+  if (!vli->buff) {
+    webarkitLOGe("Out of memory!!\n");
+    delete vli;
+    return nullptr;
+  }
+
+  return vli;
+}
+
+uint8_t *__restrict webarkitVideoLuma(WebARKitLumaInfo *vli,
+                                const uint8_t *__restrict dataPtr) {
+  //unsigned int p, q;
+
+  if (vli->simd128 == true) {
+#ifdef __wasm_simd128__
+    webarkitVideoLumaRGBAtoLuma_Emscripten_simd128(
+        vli->buff.get(), (unsigned char *__restrict)dataPtr, vli->buffSize);
+    return vli->buff.get();
+#else
+    webarkitVideoLuma_default(vli->buff.get(), (unsigned char *__restrict)dataPtr, vli->buffSize);
+    return vli->buff.get();
+#endif
+  } else {
+    webarkitVideoLuma_default(vli->buff.get(), (unsigned char *__restrict)dataPtr, vli->buffSize);
+    return vli->buff.get();
+  }
+}
+
+int webarkitVideoLumaFinal(WebARKitLumaInfo **vli_p) {
+  if (!vli_p)
+    return -1;
+  if (!*vli_p)
+    return 0;
+
+  delete *vli_p;
+  *vli_p = nullptr;
+
+  return 0;
+}
+
+static void webarkitVideoLuma_default(uint8_t *__restrict dest,
+                                 uint8_t *__restrict src, int32_t numPixels) {
+  unsigned int p, q;
+  webarkitLOGd("Using webarkitVideoLuma_default for luma conversion!!!");
+  q = 0;
+  for (p = 0; p < numPixels; p++) {
+    dest[p] = (R8_CCIR601 * src[q + 0] + G8_CCIR601 * src[q + 1] +
+                    B8_CCIR601 * src[q + 2]) >>
+                   8;
+    q += 4;
+  }
+}
+
+#ifdef __wasm_simd128__
+static void webarkitVideoLumaRGBAtoLuma_Emscripten_simd128(uint8_t *__restrict dest,
+                                                       uint8_t *__restrict src,
+                                                       int32_t numPixels) {
+
+  webarkitLOGd("Using webarkitVideoLumaRGBAtoLuma_Emscripten_simd128 for Luma conversion !!!");
+
+  v128_t *pin = (v128_t *)src;
+  int64_t *pout = (int64_t *)dest;
+  int numPixelsDiv8 = numPixels / 8;
+
+  v128_t maskRedBlue = wasm_i32x4_splat(0x00FF00FF);
+  v128_t scaleRedBlue =
+      wasm_i32x4_splat((uint32_t)B8_CCIR601 << 16 | R8_CCIR601);
+  v128_t scaleGreen = wasm_i32x4_splat(G8_CCIR601);
+  do {
+    v128_t pixels1 = wasm_v128_load(pin); // Load 16 bytes (4 pixels) from src
+    v128_t pixels2 = wasm_v128_load(pin + 1);
+    pin += 2;
+
+    v128_t g1 = wasm_u16x8_shr(pixels1, 8);
+    v128_t g2 = wasm_u16x8_shr(pixels2, 8);
+
+    v128_t rb1 = wasm_v128_and(pixels1, maskRedBlue);
+    v128_t rb2 = wasm_v128_and(pixels2, maskRedBlue);
+
+    g1 = wasm_i32x4_dot_i16x8(g1, scaleGreen);
+    g2 = wasm_i32x4_dot_i16x8(g2, scaleGreen);
+    rb1 = wasm_i32x4_dot_i16x8(rb1, scaleRedBlue);
+    rb2 = wasm_i32x4_dot_i16x8(rb2, scaleRedBlue);
+
+    v128_t y1 = wasm_i32x4_add(g1, rb1);
+    v128_t y2 = wasm_i32x4_add(g2, rb2);
+
+    y1 = wasm_u32x4_shr(y1, 8);
+    y2 = wasm_u32x4_shr(y2, 8);
+
+    v128_t y = wasm_i16x8_narrow_i32x4(y1, y2);
+    y = wasm_u8x16_narrow_i16x8(y, y);
+
+    *pout = wasm_i64x2_extract_lane(y, 0);
+
+    pout++;
+    numPixelsDiv8--;
+  } while (numPixelsDiv8);
+}
+#endif
+
+} // namespace webarkit

WebARKit/include/WebARKitVideoLuma.h

@@ -0,0 +1,47 @@
+#ifndef WEBARKITVIDEOLUMA_H
+#define WEBARKITVIDEOLUMA_H
+
+#include <cstdint>
+#include <cstdbool>
+#include <memory>
+#include <WebARKit/WebARKitLog.h>
+
+#ifdef __wasm_simd128__
+#include <wasm_simd128.h> // For SIMD operations
+#endif
+
+namespace webarkit {
+
+// CCIR 601 recommended values. See
+// http://www.poynton.com/notes/colour_and_gamma/ColorFAQ.html#RTFToC11 .
+const uint8_t R8_CCIR601 = 77;
+const uint8_t G8_CCIR601 = 150;
+const uint8_t B8_CCIR601 = 29;
+
+struct WebARKitLumaInfo {
+  int xsize;
+  int ysize;
+  int buffSize;
+  bool simd128;
+  std::unique_ptr<uint8_t[]> buff;
+};
+
+#ifdef __wasm_simd128__
+static void webarkitVideoLumaRGBAtoLuma_Emscripten_simd128(uint8_t *__restrict dest,
+    uint8_t *__restrict src,
+    int32_t numPixels);
+#endif
+
+static void webarkitVideoLuma_default(uint8_t *__restrict dest, uint8_t *__restrict src,
+    int32_t numPixels);
+
+WebARKitLumaInfo *webarkitVideoLumaInit(int xsize, int ysize, bool simd128);
+
+uint8_t *__restrict webarkitVideoLuma(WebARKitLumaInfo *vli,
+    const uint8_t *__restrict dataPtr);
+
+int webarkitVideoLumaFinal(WebARKitLumaInfo **vli_p);
+
+} // namespace webarkit
+
+#endif // WEBARKITVIDEOLUMA_H

include/AR/config.h

@@ -276,7 +276,7 @@ typedef enum {
 #undef  ARVIDEO_INPUT_DEFAULT_1394
 #undef  ARVIDEO_INPUT_DEFAULT_GSTREAMER
 #undef  ARVIDEO_INPUT_DEFAULT_IMAGE
-#undef  ARVIDEO_INPUT_DEFAULT_DUMMY
+#define  ARVIDEO_INPUT_DEFAULT_DUMMY
 
 // Other Linux-only configuration.
 #define HAVE_LIBJPEG 1
@@ -358,7 +358,7 @@ typedef enum {
 #undef  ARVIDEO_INPUT_WINDOWS_MEDIA_CAPTURE
 
 // Default input module. This is edited by the configure script.
-#undef  ARVIDEO_INPUT_DEFAULT_DUMMY
+#define  ARVIDEO_INPUT_DEFAULT_DUMMY
 #undef  ARVIDEO_INPUT_DEFAULT_IMAGE
 #undef  ARVIDEO_INPUT_DEFAULT_WINDOWS_MEDIA_FOUNDATION
 #undef  ARVIDEO_INPUT_DEFAULT_WINDOWS_MEDIA_CAPTURE
@@ -408,7 +408,7 @@ typedef enum {
 #undef  ARVIDEO_INPUT_DUMMY
 #define ARVIDEO_INPUT_ANDROID
 #undef  ARVIDEO_INPUT_IMAGE
-#undef  ARVIDEO_INPUT_DEFAULT_DUMMY
+#define  ARVIDEO_INPUT_DEFAULT_DUMMY
 #define ARVIDEO_INPUT_DEFAULT_ANDROID
 #undef  ARVIDEO_INPUT_DEFAULT_IMAGE
 
@@ -459,7 +459,7 @@ typedef enum {
 #undef  ARVIDEO_INPUT_DUMMY
 #define ARVIDEO_INPUT_IMAGE
 #define ARVIDEO_INPUT_DEFAULT_AVFOUNDATION
-#undef  ARVIDEO_INPUT_DEFAULT_DUMMY
+#define  ARVIDEO_INPUT_DEFAULT_DUMMY
 #undef  ARVIDEO_INPUT_DEFAULT_IMAGE
 #define HAVE_LIBJPEG 1
 #define USE_OPENGL_ES 1
@@ -476,7 +476,7 @@ typedef enum {
 #define ARVIDEO_INPUT_DUMMY
 #define ARVIDEO_INPUT_IMAGE
 #define ARVIDEO_INPUT_DEFAULT_AVFOUNDATION
-#undef  ARVIDEO_INPUT_DEFAULT_DUMMY
+#define  ARVIDEO_INPUT_DEFAULT_DUMMY
 #undef  ARVIDEO_INPUT_DEFAULT_IMAGE
 #define HAVE_LIBJPEG 1
 #define HAVE_INTEL_SIMD 1

include/WebARKit/WebARKitLog.h

@@ -40,5 +40,6 @@ void webarkitLOGw(const std::string &message, const char * format);
 
 void webarkitLOGw(const std::string &message, int  format);
 
+void webarkitLOGd(const std::string &message);
 
 #endif // #ifndef WEBARKIT_LOG_H

lib/SRC/AR2/featureMap.c

@@ -200,7 +200,7 @@ AR2FeatureMapT *ar2GenFeatureMap( AR2ImageT *image,
         fp2++;
     }
     for( j = 1; j < ysize-1; j++ ) {
-        ARLOGi("\r%4d/%4d.", j+1, ysize); fflush(stdout);
+        ARLOGd("\r%4d/%4d.", j+1, ysize); fflush(stdout);
         *(fp++) = 1.0f;
         fp2++;
         for( i = 1; i < xsize-1; i++ ) {

lib/SRC/ARUtil/log.c

@@ -47,6 +47,10 @@
 #  define snprintf _snprintf
 #endif
 
+#ifdef __EMSCRIPTEN__
+#include <emscripten/console.h>
+#endif
+
 //
 // Global required for logging functions.
 //
@@ -201,7 +205,16 @@ void arLogv(const char *tag, const int logLevel, const char *format, va_list ap)
             os_log_with_type(OS_LOG_DEFAULT, type, "%{public}s", buf);
         }
 #else
+
+#ifdef __EMSCRIPTEN__
+        if(logLevel == AR_LOG_LEVEL_ERROR)
+            emscripten_console_error(buf);
+        else
+            emscripten_console_warn(buf);
+#else
         fprintf(stderr, "%s", buf);
+#endif
+
 #endif
     }
     free(buf);