Vignette/visualization utils

DESCRIPTION

@@ -118,7 +118,7 @@ Collate:
     'transforms-magick.R'
     'transforms-tensor.R'
     'utils.R'
-    'vision_utils.R'
+    'vision-utils.R'
 Depends: 
     R (>= 3.5)
 LazyData: true

NEWS.md

@@ -36,9 +36,11 @@
 * Detection datasets (`coco_detection_dataset()`, `pascal_detection_dataset()` and the `rf100_*_collection()`s) now inherit the `object_detection_dataset` class and their item target `y` the `object_detection_target` class.
   Segmentation datasets (`coco_segmentation_dataset()`, `pascal_segmentation_dataset()`, `cityscapes_dataset()`, `oxfordiiitpet_segmentation_dataset()` and `rf100_peixos_segmentation_dataset()`) now inherit the `segmentation_dataset` and `segmentation_target` classes. 
   Target transforms now dispatch on those classes instead of inspecting the target fields: `target_transform_resize()`, `target_transform_rotate()`, `target_transform_affine()` and `target_transform_sahi_crop()` take an `object_detection_target`, and `target_transform_coco_masks()` and `target_transform_trimap_masks()` a `segmentation_target`. A bare list is no longer accepted as a target, so a hand-built one needs its class set (@srishtiii28, #391).
+* Added a "Visualization utilities" article covering `vision_make_grid()`, `draw_bounding_boxes()`, `draw_segmentation_masks()` and `draw_keypoints()` on the output of `model_rfdetr_base()` and `model_fcn_resnet50()` (@srishtiii28, #400).
 
 ## Bug fixes and improvements
 
+* `draw_bounding_boxes()` now accepts degenerated bounding-box by default with the `lazy = TRUE` parameter (#400).
 * `vision_make_grid()` now accepts multiple 3D tensors with mixed uint8 and float dtype (#398).
 * `transform_random_affine()` now accepts a bare number for `shear`. It used to widen `degrees`
   instead of `shear`, which left the shear range incomplete and made the sampling fail (#390).

R/vision_utils.R → R/vision-utils.R

@@ -145,6 +145,28 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8
 }
 
 
+check_bbox_is_xyxy <- function(boxes, lazy = TRUE) {
+  valid <- (boxes[, 1] < boxes[, 3])$logical_and(boxes[, 2] < boxes[, 4])
+
+  if (lazy) {
+    if ((!valid)$any()$item()) {
+      boxes <- boxes[valid, ]
+    }
+  } else {
+    if ((!valid)$any()$item()) {
+      invalid_indices <- which(as.logical(!valid))
+      first_idx <- invalid_indices[1]
+      first_box <- as.numeric(boxes[first_idx, ])
+      cli_abort(c(
+        "Bounding box {.val {first_idx}} is not in valid xyxy format.",
+        "x" = "xmin ({.val {first_box[1]}}) must be < xmax ({.val {first_box[3]}}), and ymin ({.val {first_box[2]}}) must be < ymax ({.val {first_box[4]}})."
+      ))
+    }
+  }
+
+  boxes
+}
+
 #' Draws bounding boxes on image.
 #'
 #' Draws bounding boxes on top of one image tensor
@@ -166,6 +188,8 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8
 #' @param width  Width of text shift to the bounding box.
 #' @param font NULL for the current font family, or a character vector of length 2 for Hershey vector fonts.
 #' @param font_size The requested font size in points.
+#' @param lazy if `TRUE`, silently filter out degenerate bounding boxes (xmin >= xmax or ymin >= ymax).
+#'    If `FALSE`, error on the first non-conforming box.
 #' @param ... Additional arguments passed to methods.
 #'
 #' @return  torch_tensor of size (C, H, W) of dtype uint8: Image Tensor with bounding boxes plotted.
@@ -177,7 +201,7 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8
 #' x <- torch::torch_randint(low = 1, high = 160, size = c(12,1))
 #' y <- torch::torch_randint(low = 1, high = 260, size = c(12,1))
 #' boxes <- torch::torch_cat(c(x, y, x + 20, y +  10), dim = 2)
-#' bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", fill = TRUE)
+#' bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", label = "label", fill = TRUE)
 #' tensor_image_browse(bboxed)
 #' }
 #' }
@@ -202,7 +226,8 @@ draw_bounding_boxes.torch_tensor <- function(x,
                                              fill = FALSE,
                                              width = 1,
                                              font = c("serif", "plain"),
-                                             font_size = 10, ...) {
+                                             font_size = 10,
+                                             lazy = TRUE, ...) {
   rlang::check_installed("magick")
 
   if (x$ndim == 4 && x$size(1) == 1) x <- x$squeeze(1)
@@ -215,14 +240,13 @@ draw_bounding_boxes.torch_tensor <- function(x,
     x$permute(c(2, 3, 1))$to(device = "cpu") %>% as.array()
   } else type_error("`x` should be torch_uint8 or torch_float")
 
-  if (boxes$shape[2] == 4 &&
-      ((boxes[, 1] >= boxes[, 3])$any() %>% as.logical() ||
-       (boxes[, 2] >= boxes[, 4])$any() %>% as.logical())) {
-    value_error("Boxes must be in c(xmin, ymin, xmax, ymax) format")
-  }
+  boxes <- check_bbox_is_xyxy(boxes, lazy = lazy)
   num_boxes <- boxes$shape[1]
   if (num_boxes == 0) {
-    cli_warn("boxes doesn't contain any box. No box was drawn")
+    cli_warn(if (lazy)
+      "No valid bounding box to draw after filtering degenerate boxes."
+    else
+      "boxes doesn't contain any box. No box was drawn.")
     return(x)
   }
   if (!is.null(labels) && inherits(labels, "torch_tensor")) {
@@ -352,10 +376,12 @@ draw_bounding_boxes.image_with_rotated_box <- function(x,
                                                        fill = FALSE,
                                                        width = 1,
                                                        font = c("serif", "plain"),
-                                                       font_size = 10, ...) {
+                                                       font_size = 10,
+                                                       lazy = TRUE, ...) {
   rlang::check_installed("magick")
 
-  boxes <- x$y$boxes
+  boxes <- check_bbox_is_xyxy(x$y$boxes, lazy = lazy)
+
 
   img_to_draw <- if (x$x$dtype == torch_uint8()) {
     x$x$div(255)$permute(c(2, 3, 1))$to(device = "cpu") %>% as.array()
@@ -627,6 +653,7 @@ draw_segmentation_masks.torch_tensor <- function(x,
   if (num_masks %% length(colors) != 0) {
     cli_abort("colors vector of size {.value {length(colors)}} cannot be broadcasted on {.value {num_masks}} masks")
   }
+  colors <- rep_len(colors, num_masks)
 
   color_tt <- colors %>%
     grDevices::col2rgb() %>%

_pkgdown.yml

@@ -31,6 +31,8 @@ navbar:
         href: articles/examples/fcnresnet.html
       - text: keypoints
         href: articles/examples/keypoints.html
+      - text: visualization-utilities
+        href: articles/examples/visualization-utilities.html
 
 reference:
 - title: Image Transforms

tests/testthat/test-vision-utils.R

@@ -93,6 +93,49 @@ test_that("draw_bounding_boxes works", {
   expect_no_error(bboxed_image <- draw_bounding_boxes(image_uint, boxes, colors = "black", fill = TRUE))
 })
 
+test_that("draw_bounding_boxes lazy=TRUE silently filters degenerate boxes", {
+  image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8())
+  valid_boxes <- torch::torch_tensor(rbind(c(10, 10, 50, 50), c(20, 20, 80, 80)))
+  bad_box     <- torch::torch_tensor(rbind(c(50, 10, 10, 50)))  # xmin > xmax
+  mixed_boxes <- torch::torch_cat(list(valid_boxes, bad_box), dim = 1)
+
+  expect_no_warning(result <- draw_bounding_boxes(image, mixed_boxes, lazy = TRUE))
+  expect_tensor_shape(result, c(3L, 100L, 100L))
+  expect_tensor_dtype(result, torch::torch_uint8())
+})
+
+test_that("draw_bounding_boxes lazy=FALSE errors on first degenerate box", {
+  image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8())
+  # first box valid, second degenerate (index 2 should be reported)
+  boxes <- torch::torch_tensor(rbind(
+    c(10, 10, 50, 50),
+    c(50, 10, 10, 50)   # xmin > xmax
+  ))
+
+  expect_error(
+    draw_bounding_boxes(image, boxes, lazy = FALSE),
+    regexp = "xyxy format"
+  )
+  expect_error(
+    draw_bounding_boxes(image, boxes, lazy = FALSE),
+    regexp = "2"  # first invalid box index
+  )
+})
+
+test_that("draw_bounding_boxes lazy=TRUE warns when all boxes are degenerate", {
+  image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8())
+  bad_boxes <- torch::torch_tensor(rbind(
+    c(50, 10, 10, 50),  # xmin > xmax
+    c(20, 80, 60, 20)   # ymin > ymax
+  ))
+
+  expect_warning(
+    result <- draw_bounding_boxes(image, bad_boxes, lazy = TRUE),
+    regexp = "No valid bounding box"
+  )
+  expect_tensor_shape(result, c(3L, 100L, 100L))
+})
+
 test_that("draw_bounding_boxes correctly mask a complete image", {
 
   image_float <- 1 - (torch::torch_randn(c(3, 360, 360)) / 20)