diff --git a/DESCRIPTION b/DESCRIPTION index ead8fbd6..16041ee4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -118,7 +118,7 @@ Collate: 'transforms-magick.R' 'transforms-tensor.R' 'utils.R' - 'vision_utils.R' + 'vision-utils.R' Depends: R (>= 3.5) LazyData: true diff --git a/NEWS.md b/NEWS.md index 2a1f7c25..21e8c408 100644 --- a/NEWS.md +++ b/NEWS.md @@ -36,9 +36,11 @@ * Detection datasets (`coco_detection_dataset()`, `pascal_detection_dataset()` and the `rf100_*_collection()`s) now inherit the `object_detection_dataset` class and their item target `y` the `object_detection_target` class. Segmentation datasets (`coco_segmentation_dataset()`, `pascal_segmentation_dataset()`, `cityscapes_dataset()`, `oxfordiiitpet_segmentation_dataset()` and `rf100_peixos_segmentation_dataset()`) now inherit the `segmentation_dataset` and `segmentation_target` classes. Target transforms now dispatch on those classes instead of inspecting the target fields: `target_transform_resize()`, `target_transform_rotate()`, `target_transform_affine()` and `target_transform_sahi_crop()` take an `object_detection_target`, and `target_transform_coco_masks()` and `target_transform_trimap_masks()` a `segmentation_target`. A bare list is no longer accepted as a target, so a hand-built one needs its class set (@srishtiii28, #391). +* Added a "Visualization utilities" article covering `vision_make_grid()`, `draw_bounding_boxes()`, `draw_segmentation_masks()` and `draw_keypoints()` on the output of `model_rfdetr_base()` and `model_fcn_resnet50()` (@srishtiii28, #400). ## Bug fixes and improvements +* `draw_bounding_boxes()` now accepts degenerated bounding-box by default with the `lazy = TRUE` parameter (#400). * `vision_make_grid()` now accepts multiple 3D tensors with mixed uint8 and float dtype (#398). * `transform_random_affine()` now accepts a bare number for `shear`. It used to widen `degrees` instead of `shear`, which left the shear range incomplete and made the sampling fail (#390). diff --git a/R/vision_utils.R b/R/vision-utils.R similarity index 95% rename from R/vision_utils.R rename to R/vision-utils.R index 0a926992..aa1be4d5 100644 --- a/R/vision_utils.R +++ b/R/vision-utils.R @@ -145,6 +145,28 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8 } +check_bbox_is_xyxy <- function(boxes, lazy = TRUE) { + valid <- (boxes[, 1] < boxes[, 3])$logical_and(boxes[, 2] < boxes[, 4]) + + if (lazy) { + if ((!valid)$any()$item()) { + boxes <- boxes[valid, ] + } + } else { + if ((!valid)$any()$item()) { + invalid_indices <- which(as.logical(!valid)) + first_idx <- invalid_indices[1] + first_box <- as.numeric(boxes[first_idx, ]) + cli_abort(c( + "Bounding box {.val {first_idx}} is not in valid xyxy format.", + "x" = "xmin ({.val {first_box[1]}}) must be < xmax ({.val {first_box[3]}}), and ymin ({.val {first_box[2]}}) must be < ymax ({.val {first_box[4]}})." + )) + } + } + + boxes +} + #' Draws bounding boxes on image. #' #' Draws bounding boxes on top of one image tensor @@ -166,6 +188,8 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8 #' @param width Width of text shift to the bounding box. #' @param font NULL for the current font family, or a character vector of length 2 for Hershey vector fonts. #' @param font_size The requested font size in points. +#' @param lazy if `TRUE`, silently filter out degenerate bounding boxes (xmin >= xmax or ymin >= ymax). +#' If `FALSE`, error on the first non-conforming box. #' @param ... Additional arguments passed to methods. #' #' @return torch_tensor of size (C, H, W) of dtype uint8: Image Tensor with bounding boxes plotted. @@ -177,7 +201,7 @@ vision_make_grid.torch_tensor <- function(tensor, ..., scale = TRUE, per_row = 8 #' x <- torch::torch_randint(low = 1, high = 160, size = c(12,1)) #' y <- torch::torch_randint(low = 1, high = 260, size = c(12,1)) #' boxes <- torch::torch_cat(c(x, y, x + 20, y + 10), dim = 2) -#' bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", fill = TRUE) +#' bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", label = "label", fill = TRUE) #' tensor_image_browse(bboxed) #' } #' } @@ -202,7 +226,8 @@ draw_bounding_boxes.torch_tensor <- function(x, fill = FALSE, width = 1, font = c("serif", "plain"), - font_size = 10, ...) { + font_size = 10, + lazy = TRUE, ...) { rlang::check_installed("magick") if (x$ndim == 4 && x$size(1) == 1) x <- x$squeeze(1) @@ -215,14 +240,13 @@ draw_bounding_boxes.torch_tensor <- function(x, x$permute(c(2, 3, 1))$to(device = "cpu") %>% as.array() } else type_error("`x` should be torch_uint8 or torch_float") - if (boxes$shape[2] == 4 && - ((boxes[, 1] >= boxes[, 3])$any() %>% as.logical() || - (boxes[, 2] >= boxes[, 4])$any() %>% as.logical())) { - value_error("Boxes must be in c(xmin, ymin, xmax, ymax) format") - } + boxes <- check_bbox_is_xyxy(boxes, lazy = lazy) num_boxes <- boxes$shape[1] if (num_boxes == 0) { - cli_warn("boxes doesn't contain any box. No box was drawn") + cli_warn(if (lazy) + "No valid bounding box to draw after filtering degenerate boxes." + else + "boxes doesn't contain any box. No box was drawn.") return(x) } if (!is.null(labels) && inherits(labels, "torch_tensor")) { @@ -352,10 +376,12 @@ draw_bounding_boxes.image_with_rotated_box <- function(x, fill = FALSE, width = 1, font = c("serif", "plain"), - font_size = 10, ...) { + font_size = 10, + lazy = TRUE, ...) { rlang::check_installed("magick") - boxes <- x$y$boxes + boxes <- check_bbox_is_xyxy(x$y$boxes, lazy = lazy) + img_to_draw <- if (x$x$dtype == torch_uint8()) { x$x$div(255)$permute(c(2, 3, 1))$to(device = "cpu") %>% as.array() @@ -627,6 +653,7 @@ draw_segmentation_masks.torch_tensor <- function(x, if (num_masks %% length(colors) != 0) { cli_abort("colors vector of size {.value {length(colors)}} cannot be broadcasted on {.value {num_masks}} masks") } + colors <- rep_len(colors, num_masks) color_tt <- colors %>% grDevices::col2rgb() %>% diff --git a/_pkgdown.yml b/_pkgdown.yml index d359d280..fecd1650 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -31,6 +31,8 @@ navbar: href: articles/examples/fcnresnet.html - text: keypoints href: articles/examples/keypoints.html + - text: visualization-utilities + href: articles/examples/visualization-utilities.html reference: - title: Image Transforms diff --git a/man/coco_polygon_to_mask.Rd b/man/coco_polygon_to_mask.Rd index 6d81b11e..62e77f08 100644 --- a/man/coco_polygon_to_mask.Rd +++ b/man/coco_polygon_to_mask.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{coco_polygon_to_mask} \alias{coco_polygon_to_mask} \title{Convert COCO polygon to mask tensor (Robust Version)} diff --git a/man/draw_bounding_boxes.Rd b/man/draw_bounding_boxes.Rd index fbae4f1f..b2c3396e 100644 --- a/man/draw_bounding_boxes.Rd +++ b/man/draw_bounding_boxes.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{draw_bounding_boxes} \alias{draw_bounding_boxes} \alias{draw_bounding_boxes.default} @@ -21,6 +21,7 @@ draw_bounding_boxes(x, ...) width = 1, font = c("serif", "plain"), font_size = 10, + lazy = TRUE, ... ) @@ -34,6 +35,7 @@ draw_bounding_boxes(x, ...) width = 1, font = c("serif", "plain"), font_size = 10, + lazy = TRUE, ... ) } @@ -64,6 +66,9 @@ strings e.g. "red" or "#FF00FF". By default, viridis colors are generated for bo \item{font}{NULL for the current font family, or a character vector of length 2 for Hershey vector fonts.} \item{font_size}{The requested font size in points.} + +\item{lazy}{if \code{TRUE}, silently filter out degenerate bounding boxes (xmin >= xmax or ymin >= ymax). +If \code{FALSE}, error on the first non-conforming box.} } \value{ torch_tensor of size (C, H, W) of dtype uint8: Image Tensor with bounding boxes plotted. @@ -78,7 +83,7 @@ image_tensor <- torch::torch_randint(170, 250, size = c(3, 360, 360))$to(torch:: x <- torch::torch_randint(low = 1, high = 160, size = c(12,1)) y <- torch::torch_randint(low = 1, high = 260, size = c(12,1)) boxes <- torch::torch_cat(c(x, y, x + 20, y + 10), dim = 2) -bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", fill = TRUE) +bboxed <- draw_bounding_boxes(image_tensor, boxes, colors = "black", label = "label", fill = TRUE) tensor_image_browse(bboxed) } } diff --git a/man/draw_keypoints.Rd b/man/draw_keypoints.Rd index 03658131..d99683bd 100644 --- a/man/draw_keypoints.Rd +++ b/man/draw_keypoints.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{draw_keypoints} \alias{draw_keypoints} \title{Draws Keypoints} diff --git a/man/draw_segmentation_masks.Rd b/man/draw_segmentation_masks.Rd index 0b6ccb12..9670398a 100644 --- a/man/draw_segmentation_masks.Rd +++ b/man/draw_segmentation_masks.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{draw_segmentation_masks} \alias{draw_segmentation_masks} \alias{draw_segmentation_masks.default} diff --git a/man/tensor_image_browse.Rd b/man/tensor_image_browse.Rd index 3f226ea8..2bfef048 100644 --- a/man/tensor_image_browse.Rd +++ b/man/tensor_image_browse.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{tensor_image_browse} \alias{tensor_image_browse} \title{Display image tensor} diff --git a/man/tensor_image_display.Rd b/man/tensor_image_display.Rd index 856cc3d0..98b24ddb 100644 --- a/man/tensor_image_display.Rd +++ b/man/tensor_image_display.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{tensor_image_display} \alias{tensor_image_display} \title{Display image tensor} diff --git a/man/vision_make_grid.Rd b/man/vision_make_grid.Rd index 8fa91cf8..a113e20a 100644 --- a/man/vision_make_grid.Rd +++ b/man/vision_make_grid.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/vision_utils.R +% Please edit documentation in R/vision-utils.R \name{vision_make_grid} \alias{vision_make_grid} \alias{vision_make_grid.default} diff --git a/tests/testthat/test-vision-utils.R b/tests/testthat/test-vision-utils.R index 29d41e4f..aeb4d848 100644 --- a/tests/testthat/test-vision-utils.R +++ b/tests/testthat/test-vision-utils.R @@ -93,6 +93,49 @@ test_that("draw_bounding_boxes works", { expect_no_error(bboxed_image <- draw_bounding_boxes(image_uint, boxes, colors = "black", fill = TRUE)) }) +test_that("draw_bounding_boxes lazy=TRUE silently filters degenerate boxes", { + image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8()) + valid_boxes <- torch::torch_tensor(rbind(c(10, 10, 50, 50), c(20, 20, 80, 80))) + bad_box <- torch::torch_tensor(rbind(c(50, 10, 10, 50))) # xmin > xmax + mixed_boxes <- torch::torch_cat(list(valid_boxes, bad_box), dim = 1) + + expect_no_warning(result <- draw_bounding_boxes(image, mixed_boxes, lazy = TRUE)) + expect_tensor_shape(result, c(3L, 100L, 100L)) + expect_tensor_dtype(result, torch::torch_uint8()) +}) + +test_that("draw_bounding_boxes lazy=FALSE errors on first degenerate box", { + image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8()) + # first box valid, second degenerate (index 2 should be reported) + boxes <- torch::torch_tensor(rbind( + c(10, 10, 50, 50), + c(50, 10, 10, 50) # xmin > xmax + )) + + expect_error( + draw_bounding_boxes(image, boxes, lazy = FALSE), + regexp = "xyxy format" + ) + expect_error( + draw_bounding_boxes(image, boxes, lazy = FALSE), + regexp = "2" # first invalid box index + ) +}) + +test_that("draw_bounding_boxes lazy=TRUE warns when all boxes are degenerate", { + image <- torch::torch_randint(1L, 200L, c(3L, 100L, 100L))$to(torch::torch_uint8()) + bad_boxes <- torch::torch_tensor(rbind( + c(50, 10, 10, 50), # xmin > xmax + c(20, 80, 60, 20) # ymin > ymax + )) + + expect_warning( + result <- draw_bounding_boxes(image, bad_boxes, lazy = TRUE), + regexp = "No valid bounding box" + ) + expect_tensor_shape(result, c(3L, 100L, 100L)) +}) + test_that("draw_bounding_boxes correctly mask a complete image", { image_float <- 1 - (torch::torch_randn(c(3, 360, 360)) / 20) diff --git a/vignettes/examples/assets/dog_with_two_bbox.png b/vignettes/examples/assets/dog_with_two_bbox.png deleted file mode 100644 index dab38255..00000000 Binary files a/vignettes/examples/assets/dog_with_two_bbox.png and /dev/null differ diff --git a/vignettes/examples/assets/file84fb43ce0fa5.png b/vignettes/examples/assets/file84fb43ce0fa5.png deleted file mode 100644 index 6ea0d76e..00000000 Binary files a/vignettes/examples/assets/file84fb43ce0fa5.png and /dev/null differ diff --git a/vignettes/examples/assets/viz-boxes.jpg b/vignettes/examples/assets/viz-boxes.jpg new file mode 100644 index 00000000..b602d787 Binary files /dev/null and b/vignettes/examples/assets/viz-boxes.jpg differ diff --git a/vignettes/examples/assets/viz-fcn-classes.jpg b/vignettes/examples/assets/viz-fcn-classes.jpg new file mode 100644 index 00000000..2971bee8 Binary files /dev/null and b/vignettes/examples/assets/viz-fcn-classes.jpg differ diff --git a/vignettes/examples/assets/viz-fcn-dogs.jpg b/vignettes/examples/assets/viz-fcn-dogs.jpg new file mode 100644 index 00000000..c5966fac Binary files /dev/null and b/vignettes/examples/assets/viz-fcn-dogs.jpg differ diff --git a/vignettes/examples/assets/viz-grid.jpg b/vignettes/examples/assets/viz-grid.jpg new file mode 100644 index 00000000..837d2e4e Binary files /dev/null and b/vignettes/examples/assets/viz-grid.jpg differ diff --git a/vignettes/examples/assets/viz-keypoints.jpg b/vignettes/examples/assets/viz-keypoints.jpg new file mode 100644 index 00000000..41cb7f50 Binary files /dev/null and b/vignettes/examples/assets/viz-keypoints.jpg differ diff --git a/vignettes/examples/assets/viz-rfdetr-filtered.jpg b/vignettes/examples/assets/viz-rfdetr-filtered.jpg new file mode 100644 index 00000000..6d1401eb Binary files /dev/null and b/vignettes/examples/assets/viz-rfdetr-filtered.jpg differ diff --git a/vignettes/examples/assets/viz-rfdetr-raw.jpg b/vignettes/examples/assets/viz-rfdetr-raw.jpg new file mode 100644 index 00000000..121ec5f9 Binary files /dev/null and b/vignettes/examples/assets/viz-rfdetr-raw.jpg differ diff --git a/vignettes/examples/visualization-utilities.Rmd b/vignettes/examples/visualization-utilities.Rmd new file mode 100644 index 00000000..0b3ae0da --- /dev/null +++ b/vignettes/examples/visualization-utilities.Rmd @@ -0,0 +1,197 @@ +--- +title: "Visualization utilities" +type: docs +--- + +```{r, echo = FALSE} +knitr::opts_chunk$set(eval = FALSE) +``` + +This article walks through the drawing helpers shipped with torchvision +(`vision_make_grid()`, `draw_bounding_boxes()`, `draw_segmentation_masks()` and +`draw_keypoints()`) and shows how to feed them the output of a pretrained +detection or segmentation model. It follows the +[Visualization utilities](https://docs.pytorch.org/vision/stable/auto_examples/others/plot_visualization_utils.html) +tutorial from PyTorch. + +The code below is not evaluated when the website is built, because it downloads +around 250 MB of model weights. Every figure was produced by running it. + +```{r setup} +library(torchvision) +library(torch) + +gallery <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/" +dog1 <- base_loader(paste0(gallery, "dog1.jpg")) %>% transform_to_tensor() +dog2 <- base_loader(paste0(gallery, "dog2.jpg")) %>% transform_to_tensor() +``` + +## A grid of images + +`vision_make_grid()` tiles a batch of images into a single tensor, which is a +quick way to eyeball a batch coming out of a `dataloader()`. + +```{r} +grid <- vision_make_grid(dog1, dog2, per_row = 2) +tensor_image_browse(grid) +``` + +![The two photographs tiled side by side into a single tensor](assets/viz-grid.jpg) + +## Bounding boxes + +`draw_bounding_boxes()` expects an `(N, 4)` tensor of boxes given as +`(xmin, ymin, xmax, ymax)` in absolute pixel coordinates. + +```{r} +boxes <- torch_tensor(rbind( + c(50, 50, 100, 200), + c(210, 150, 350, 430) +), dtype = torch_float()) + +boxed <- draw_bounding_boxes(dog1, boxes, + labels = c("tree", "partial corgi"), font_size = 18, + colors = c("darkorange", "yellow"), width = 5) +tensor_image_browse(boxed) +``` + +![Hand-written boxes: orange over the trees, yellow around the corgi](assets/viz-boxes.jpg) + +## Object detection with RF-DETR + +Boxes are usually not hand written but predicted. We will use the RF-DETR model to do so. +RF-DETR expects ImageNet-normalized inputs, so we assemble two images in a batch and then normalize. +Note: The model is able to detect the dogs on unnormalized input, but with far lower confidence: the corgi confidence scores in the previous image drops from 0.67 with normalization down 0.28 without. + +```{r} +norm_mean <- c(0.485, 0.456, 0.406) +norm_std <- c(0.229, 0.224, 0.225) + +batch <- list(dog1, dog2) %>% + torch_stack() %>% + transform_normalize(norm_mean, norm_std) + +model <- model_rfdetr_base(pretrained = TRUE, num_queries = 40) +model$eval() +detections <- with_no_grad(model(batch))$detections +``` + +RF-DETR resizes the batch to its own working resolution internally and maps the +boxes back to the coordinates of the images you passed in, so images can be drawn +straight onto the original tensors. Each element of `detections` holds `boxes`, +`labels` and `scores` sorted by decreasing score. + +Let's first draw the objects detected with raw model output. + +```{r} +tensor_image_browse(vision_make_grid( + draw_bounding_boxes(dog1, detections[[1]]$boxes), + draw_bounding_boxes(dog2, detections[[2]]$boxes) +)) +``` +![RF-DETR raw output has many boxes](assets/viz-rfdetr-raw.jpg) + +We can see that too many boxes are detected. RF-DETR Model default to 300 object detection, but we reduced it via the `num_queries` parameter. + +Let's filter them based on their associated confidence score. + +We will create a convenience function that keeps the confident ones, and +turn the label indices into readable names out of `coco_classes()`. + +```{r} +draw_high_confidence_bboxes <- function(image, detection, width = 4, font_size = 25, ...) { + keep <- as.logical(as.array(detection$scores > 0.2)) + draw_bounding_boxes( + image, + detection$boxes[keep, , drop = FALSE], + labels = coco_classes(as.integer(detection$labels[keep])), + width = width, + font_size = font_size, + ... + ) +} +``` + +Now let's draw again +```{r} +tensor_image_browse(vision_make_grid( + draw_high_confidence_bboxes(dog1, detections[[1]], color = "yellow"), + draw_high_confidence_bboxes(dog2, detections[[2]], color = "yellow") +)) + +``` + + ![RF-DETR confidence filtered output](assets/viz-rfdetr-filtered.jpg) + +## Semantic segmentation masks + +`model_fcn_resnet50()` scores every pixel against the 21 PASCAL VOC classes. Its +pretrained weights are evaluated at 520 pixels, so resize before normalizing, +and keep the resized images around to draw on: masks and image must share their +height and width. + +```{r} +resized <- list(dog1, dog2) %>% + torch_stack() %>% + transform_resize(size = c(520, 520)) + +batch <- resized %>% + transform_normalize(norm_mean, norm_std) + +model <- model_fcn_resnet50(pretrained = TRUE) +model$eval() +scores <- with_no_grad(model(batch))$out +``` + +`scores` is a `CPUFloatType{2,21,520,520}` tensor. Handeling such a tensor, +`draw_segmentation_masks()` takes the `argmax()` over the class dimension and +colors each class that appears. + +```{r} +segmented <- draw_segmentation_masks(resized[1,..], scores[1, ..], alpha = 0.6) +tensor_image_browse(segmented) +``` + +![Every class the model assigns: the dog in yellow, the background in pink](assets/viz-fcn-classes.jpg) + +To highlight a single class instead, build the boolean mask yourself. `dog` is +the 13th of the 21 classes, and softmax turns the scores into per-pixel +probabilities that can be thresholded. + +```{r} +tensor_image_browse(vision_make_grid( + draw_segmentation_masks(resized[1,..], scores[1, ..], alpha = 0.6), + draw_segmentation_masks(resized[2,..], scores[2, ..], alpha = 0.6) + ) +) +``` + +![The dog class on its own](assets/viz-fcn-dogs.jpg) + +## Keypoints + +`draw_keypoints()` takes an `(N, K, 2)` tensor holding the `(x, y)` location of +`K` keypoints for each of `N` instances. torchvision has no pose estimation +model to feed it (`model_mtcnn()` predicts five landmarks, but only on human +faces), so the keypoints here are annotated by hand: the two ear tips, the two +eyes and the nose of the corgi. `connectivity` lists the pairs of keypoint +indices to join with a line. Each line is drawn in the color of the keypoint it +starts from, so leave `colors` at its default or pass one color per keypoint. + +```{r} +keypoints <- torch_tensor(rbind( + c(220, 171), + c(296, 176), + c(239, 222), + c(262, 226), + c(242, 250) +), dtype = torch_float())$reshape(c(1, 5, 2)) + +connectivity <- list(c(1, 3), c(2, 4), c(3, 4), c(3, 5), c(4, 5)) + +posed <- draw_keypoints(dog1, keypoints, connectivity = connectivity, + radius = 6, width = 3) +tensor_image_browse(posed) +``` + +![Five hand-annotated keypoints joined by the connectivity pairs](assets/viz-keypoints.jpg)