Feat: add item_transform_random_perspective for detection and segmentation items

NAMESPACE

@@ -58,6 +58,11 @@ S3method(item_transform_random_horizontal_flip,default)
 S3method(item_transform_random_horizontal_flip,image_with_bounding_box)
 S3method(item_transform_random_horizontal_flip,image_with_rotated_box)
 S3method(item_transform_random_horizontal_flip,image_with_segmentation_mask)
+S3method(item_transform_random_perspective,dataset)
+S3method(item_transform_random_perspective,default)
+S3method(item_transform_random_perspective,image_with_bounding_box)
+S3method(item_transform_random_perspective,image_with_rotated_box)
+S3method(item_transform_random_perspective,image_with_segmentation_mask)
 S3method(item_transform_random_resize_crop,dataset)
 S3method(item_transform_random_resize_crop,default)
 S3method(item_transform_random_resize_crop,image_with_bounding_box)
@@ -219,6 +224,7 @@ export(item_transform_random_affine)
 export(item_transform_random_crop)
 export(item_transform_random_erasing)
 export(item_transform_random_horizontal_flip)
+export(item_transform_random_perspective)
 export(item_transform_random_resize_crop)
 export(item_transform_random_rotation)
 export(item_transform_random_vertical_flip)
@@ -389,91 +395,97 @@ importFrom(grDevices,dev.off)
 importFrom(graphics,polygon)
 importFrom(jsonlite,fromJSON)
 importFrom(magrittr,"%>%")
-importFrom(stats,runif)
-importFrom(stats,setNames)
-importFrom(torch,as_array)
-importFrom(torch,dataset)
-importFrom(torch,load_state_dict)
-importFrom(torch,nn_adaptive_avg_pool2d)
-importFrom(torch,nn_batch_norm1d)
-importFrom(torch,nn_batch_norm2d)
-importFrom(torch,nn_conv2d)
-importFrom(torch,nn_conv_transpose2d)
-importFrom(torch,nn_dropout)
-importFrom(torch,nn_embedding)
-importFrom(torch,nn_gelu)
-importFrom(torch,nn_hardsigmoid)
-importFrom(torch,nn_hardswish)
-importFrom(torch,nn_identity)
-importFrom(torch,nn_init_constant_)
-importFrom(torch,nn_init_ones_)
-importFrom(torch,nn_init_trunc_normal_)
-importFrom(torch,nn_init_xavier_uniform_)
-importFrom(torch,nn_init_zeros_)
-importFrom(torch,nn_layer_norm)
-importFrom(torch,nn_linear)
-importFrom(torch,nn_max_pool2d)
-importFrom(torch,nn_module)
-importFrom(torch,nn_module_dict)
-importFrom(torch,nn_module_list)
-importFrom(torch,nn_multihead_attention)
-importFrom(torch,nn_parameter)
-importFrom(torch,nn_prelu)
-importFrom(torch,nn_relu)
-importFrom(torch,nn_sequential)
-importFrom(torch,nn_silu)
-importFrom(torch,nn_softmax)
-importFrom(torch,nnf_affine_grid)
-importFrom(torch,nnf_gelu)
-importFrom(torch,nnf_grid_sample)
-importFrom(torch,nnf_interpolate)
-importFrom(torch,nnf_layer_norm)
-importFrom(torch,nnf_normalize)
-importFrom(torch,nnf_relu)
-importFrom(torch,nnf_silu)
-importFrom(torch,nnf_softmax)
-importFrom(torch,torch_abs)
-importFrom(torch,torch_arange)
-importFrom(torch,torch_atan2)
-importFrom(torch,torch_bool)
-importFrom(torch,torch_cat)
-importFrom(torch,torch_chunk)
-importFrom(torch,torch_clamp)
-importFrom(torch,torch_cos)
-importFrom(torch,torch_cumsum)
-importFrom(torch,torch_div)
-importFrom(torch,torch_empty)
-importFrom(torch,torch_flatten)
-importFrom(torch,torch_float)
-importFrom(torch,torch_float32)
-importFrom(torch,torch_gather)
-importFrom(torch,torch_int32)
-importFrom(torch,torch_linspace)
-importFrom(torch,torch_log)
-importFrom(torch,torch_log2)
-importFrom(torch,torch_long)
-importFrom(torch,torch_matmul)
-importFrom(torch,torch_max)
-importFrom(torch,torch_meshgrid)
-importFrom(torch,torch_min)
-importFrom(torch,torch_nonzero)
-importFrom(torch,torch_ones)
-importFrom(torch,torch_ones_like)
-importFrom(torch,torch_rand)
-importFrom(torch,torch_randn)
-importFrom(torch,torch_sigmoid)
-importFrom(torch,torch_sign)
-importFrom(torch,torch_sin)
-importFrom(torch,torch_stack)
-importFrom(torch,torch_sum)
-importFrom(torch,torch_tensor)
-importFrom(torch,torch_topk)
-importFrom(torch,torch_uint8)
-importFrom(torch,torch_zeros)
-importFrom(torch,torch_zeros_like)
-importFrom(torch,with_no_grad)
-importFrom(utils,head)
-importFrom(utils,read.csv)
-importFrom(utils,read.delim)
-importFrom(utils,tail)
+importFrom(stats,
+  runif,
+  setNames
+)
+importFrom(torch,
+  as_array,
+  dataset,
+  load_state_dict,
+  nn_adaptive_avg_pool2d,
+  nn_batch_norm1d,
+  nn_batch_norm2d,
+  nn_conv2d,
+  nn_conv_transpose2d,
+  nn_dropout,
+  nn_embedding,
+  nn_gelu,
+  nn_hardsigmoid,
+  nn_hardswish,
+  nn_identity,
+  nn_init_constant_,
+  nn_init_ones_,
+  nn_init_trunc_normal_,
+  nn_init_xavier_uniform_,
+  nn_init_zeros_,
+  nn_layer_norm,
+  nn_linear,
+  nn_max_pool2d,
+  nn_module,
+  nn_module_dict,
+  nn_module_list,
+  nn_multihead_attention,
+  nn_parameter,
+  nn_prelu,
+  nn_relu,
+  nn_sequential,
+  nn_silu,
+  nn_softmax,
+  nnf_affine_grid,
+  nnf_gelu,
+  nnf_grid_sample,
+  nnf_interpolate,
+  nnf_layer_norm,
+  nnf_normalize,
+  nnf_relu,
+  nnf_silu,
+  nnf_softmax,
+  torch_abs,
+  torch_arange,
+  torch_atan2,
+  torch_bool,
+  torch_cat,
+  torch_chunk,
+  torch_clamp,
+  torch_cos,
+  torch_cumsum,
+  torch_div,
+  torch_empty,
+  torch_flatten,
+  torch_float,
+  torch_float32,
+  torch_gather,
+  torch_int32,
+  torch_linspace,
+  torch_log,
+  torch_log2,
+  torch_long,
+  torch_matmul,
+  torch_max,
+  torch_meshgrid,
+  torch_min,
+  torch_nonzero,
+  torch_ones,
+  torch_ones_like,
+  torch_rand,
+  torch_randn,
+  torch_sigmoid,
+  torch_sign,
+  torch_sin,
+  torch_stack,
+  torch_sum,
+  torch_tensor,
+  torch_topk,
+  torch_uint8,
+  torch_zeros,
+  torch_zeros_like,
+  with_no_grad
+)
+importFrom(utils,
+  head,
+  read.csv,
+  read.delim,
+  tail
+)
 importFrom(zeallot,"%<-%")

NEWS.md

@@ -32,6 +32,7 @@
 * Added `item_transform_random_crop()` for cropping dataset items at a random location with optional padding, with support for detection and segmentation item types and datasets (@DerrickUnleashed, #387).
 * Added `item_transform_random_erasing()` for randomly erasing a rectangular region of dataset items with probability `p`, with support for detection, segmentation and rotated-box item types and datasets (#389).
 * Added `item_transform_random_affine()` for applying an affine transformation drawn from the given ranges to dataset items, with support for detection, segmentation and rotated-box item types and datasets (@srishtiii28, #390).
+* Added `item_transform_random_perspective()` for applying a random perspective transformation to dataset items with probability `p`, with support for detection, segmentation and rotated-box item types and datasets (@DerrickUnleashed, #393).
 * Detection datasets (`coco_detection_dataset()`, `pascal_detection_dataset()` and the `rf100_*_collection()`s) now inherit the `object_detection_dataset` class and their item target `y` the `object_detection_target` class.
   Segmentation datasets (`coco_segmentation_dataset()`, `pascal_segmentation_dataset()`, `cityscapes_dataset()`, `oxfordiiitpet_segmentation_dataset()` and `rf100_peixos_segmentation_dataset()`) now inherit the `segmentation_dataset` and `segmentation_target` classes. 
   Target transforms now dispatch on those classes instead of inspecting the target fields: `target_transform_resize()`, `target_transform_rotate()`, `target_transform_affine()` and `target_transform_sahi_crop()` take an `object_detection_target`, and `target_transform_coco_masks()` and `target_transform_trimap_masks()` a `segmentation_target`. A bare list is no longer accepted as a target, so a hand-built one needs its class set (@srishtiii28, #391).

R/item-transforms-random-geometry.R

@@ -344,7 +344,7 @@ rescale_box_angle <- function(angle_deg, scale_w, scale_h) {
 #' @param pad_if_needed (logical) It will pad the image if smaller than the
 #'   desired size to avoid raising an exception. Since cropping is done
 #'   after padding, the padding seems to be done at a random offset.
-#' @param fill (number or tuple) Pixel fill value for constant fill. Default is
+#' @param fill (numeric or vector) Pixel fill value for constant fill. Default is
 #'   0. If a tuple of length 3, it is used to fill R, G, B channels
 #'   respectively. This value is only used when the padding_mode is constant.
 #' @param padding_mode (str) Type of padding. Should be: constant, edge,
@@ -822,7 +822,6 @@ item_transform_random_erasing.image_with_bounding_box <- function(x, p = 0.5, sc
     img_size <- get_image_size(x$x)
     c(top, left, height, width) %<-% get_random_erasing_params(img_size[2], img_size[1], scale, ratio)
     if (!is.null(top)) {
-      # x$x <- erase_tensor_region(x$x, top, left, height, width, value, inplace)
       img_c <- x$x$size(1)
 
       if (!inplace) {
@@ -837,7 +836,6 @@ item_transform_random_erasing.image_with_bounding_box <- function(x, p = 0.5, sc
       w <- min(width, img_w - left)
 
       if (!(h <= 0L || w <= 0L)) {
-        # inplace region replacement
         region <- x$x$narrow(2, top + 1L, h)$narrow(3, left + 1L, w)
 
         if (is.character(value)) {
@@ -860,6 +858,128 @@ item_transform_random_erasing.image_with_segmentation_mask <- item_transform_ran
 #' @export
 item_transform_random_erasing.image_with_rotated_box <- item_transform_random_erasing.image_with_bounding_box
 
+#' Randomly apply a perspective transform to a dataset item
+#'
+#' Applies a random perspective transformation to the image inside a dataset
+#' item with a given probability. The degree of distortion is controlled by
+#' \code{distortion_scale}. When a transform occurs, the same perspective
+#' mapping is applied to bounding boxes and masks so that the targets stay
+#' aligned with the image.
+#'
+#' The transformation is drawn again for every item, so that a dataset wrapped
+#' with this transform yields a different perspective on each access.
+#'
+#' @param x A dataset item, typically an \code{image_with_bounding_box} or
+#'   \code{image_with_segmentation_mask} object containing an image tensor
+#'   and associated target data.
+#' @inheritParams transform_random_perspective
+#'
+#' @return A dataset item of the same class. With probability \code{p}, the
+#'   image and targets are perspective transformed; otherwise they are
+#'   returned unchanged.
+#'
+#' @examples
+#' \dontrun{
+#' url <- "https://upload.wikimedia.org/wikipedia/commons/b/b6/Felis_catus-cat_on_snow.jpg"
+#' img <- base_loader(url) |> transform_to_tensor()
+#'
+#' boxes <- torch_tensor(matrix(c(600, 200, 2880, 1860), ncol = 4), dtype = torch_float32())
+#'
+#' before <- list(x = img, y = list(boxes = boxes, labels = "cat"))
+#' class(before) <- c("image_with_bounding_box", "list")
+#'
+#' after <- item_transform_random_perspective(before)
+#'
+#' before_plot <- draw_bounding_boxes(before, colors = "blue", width = 10)
+#' after_plot <- draw_bounding_boxes(after, colors = "red", width = 10)
+#' tensor_image_browse(before_plot)
+#' tensor_image_browse(after_plot)
+#' }
+#'
+#' @family item_random_transforms
+#'
+#' @export
+item_transform_random_perspective <- function(x, distortion_scale = 0.5, p = 0.5,
+                                              interpolation = 2, fill = 0) {
+  UseMethod("item_transform_random_perspective", x)
+}
+
+#' @export
+item_transform_random_perspective.default <- function(x, distortion_scale = 0.5, p = 0.5,
+                                                      interpolation = 2, fill = 0) {
+  cli_abort(
+    "{.fn item_transform_random_perspective} requires a dataset item (a list with {.var x} and {.var y} fields), not {.obj_type_friendly {x}}.
+    To transform a raw image tensor, use {.fn transform_random_perspective} instead."
+  )
+}
+
+#' @export
+item_transform_random_perspective.dataset <- function(x, distortion_scale = 0.5, p = 0.5,
+                                                      interpolation = 2, fill = 0) {
+  original_getitem <- x$.getitem
+  unlockBinding(".getitem", as.environment(x))
+  x$.getitem <- function(index) {
+    item <- original_getitem(index)
+    item_transform_random_perspective(item, distortion_scale = distortion_scale, p = p,
+                                      interpolation = interpolation, fill = fill)
+  }
+  x
+}
+
+#' @export
+item_transform_random_perspective.image_with_bounding_box <- function(x, distortion_scale = 0.5,
+                                                                      p = 0.5, interpolation = 2,
+                                                                      fill = 0) {
+  if (stats::runif(1) < p) {
+    c(width, height) %<-% get_image_size(x$x)
+    c(startpoints, endpoints) %<-% get_random_perspective_params(width, height, distortion_scale)
+    x <- item_transform_perspective(x, startpoints = startpoints, endpoints = endpoints,
+                                    interpolation = interpolation, fill = fill)
+  }
+  x
+}
+
+#' @export
+item_transform_random_perspective.image_with_segmentation_mask <- item_transform_random_perspective.image_with_bounding_box
+
+#' @export
+item_transform_random_perspective.image_with_rotated_box <- function(x, distortion_scale = 0.5,
+                                                                      p = 0.5, interpolation = 2,
+                                                                      fill = 0) {
+  if (stats::runif(1) < p) {
+    c(width, height) %<-% get_image_size(x$x)
+    c(startpoints, endpoints) %<-% get_random_perspective_params(width, height, distortion_scale)
+
+    x$x <- transform_perspective(x$x, startpoints = startpoints, endpoints = endpoints,
+                                  interpolation = interpolation, fill = fill)
+
+    boxes <- x$y$boxes$clone()
+    if (boxes$size(1) > 0) {
+      angle <- boxes[, 5]$clone()
+      boxes_4 <- boxes[, 1:4]$clone()
+      boxes_4 <- perspective_boxes(boxes_4, startpoints = startpoints, endpoints = endpoints)
+      img_size <- get_image_size(x$x)
+      img_w <- img_size[1]
+      img_h <- img_size[2]
+      boxes_4[, 1] <- torch_clamp(boxes_4[, 1], 0, img_w)
+      boxes_4[, 3] <- torch_clamp(boxes_4[, 3], 0, img_w)
+      boxes_4[, 2] <- torch_clamp(boxes_4[, 2], 0, img_h)
+      boxes_4[, 4] <- torch_clamp(boxes_4[, 4], 0, img_h)
+      keep <- as.logical((boxes_4[, 3] > boxes_4[, 1]) & (boxes_4[, 4] > boxes_4[, 2]))
+      if (!all(keep)) {
+        boxes_4 <- boxes_4[keep, ]
+        angle <- angle[keep]
+        x$y$labels <- x$y$labels[keep]
+        if (!is.null(x$y$area)) {
+          x$y$area <- x$y$area[keep]
+        }
+      }
+      x$y$boxes <- torch_cat(list(boxes_4, angle$unsqueeze(2)), dim = 2)
+    }
+  }
+  x
+}
+
 # Sample the location and size of a random erasing rectangle.
 #
 # Returns a list with 0-indexed `top` and `left` and positive `height` and