diff --git a/NAMESPACE b/NAMESPACE index 88bb95e1..a9eb906e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -29,6 +29,11 @@ S3method(item_transform_hflip,default) S3method(item_transform_hflip,image_with_bounding_box) S3method(item_transform_hflip,image_with_rotated_box) S3method(item_transform_hflip,image_with_segmentation_mask) +S3method(item_transform_random_affine,dataset) +S3method(item_transform_random_affine,default) +S3method(item_transform_random_affine,image_with_bounding_box) +S3method(item_transform_random_affine,image_with_rotated_box) +S3method(item_transform_random_affine,image_with_segmentation_mask) S3method(item_transform_random_horizontal_flip,dataset) S3method(item_transform_random_horizontal_flip,default) S3method(item_transform_random_horizontal_flip,image_with_bounding_box) @@ -171,6 +176,7 @@ export(item_transform_affine) export(item_transform_center_crop) export(item_transform_crop) export(item_transform_hflip) +export(item_transform_random_affine) export(item_transform_random_horizontal_flip) export(item_transform_random_vertical_flip) export(item_transform_pad) diff --git a/NEWS.md b/NEWS.md index ac5ec17c..1a3007da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,9 +26,12 @@ * `item_transform_rotate()` now supports segmentation items and datasets, rotating the masks alongside the image (@srishtiii28, #379). * Added `item_transform_crop()` for cropping dataset items at a specified location and size, with support for detection and segmentation item types and datasets (@DerrickUnleashed, #371). * Added `item_transform_pad()` for padding dataset items on all sides, with support for detection, segmentation and rotated-box item types and datasets (@DerrickUnleashed, #373). +* Added `item_transform_random_affine()` for applying an affine transformation drawn from the given ranges to dataset items, with support for detection, segmentation and rotated-box item types and datasets (@srishtiii28, #354). ## Bug fixes and improvements +* `transform_random_affine()` now accepts a bare number for `shear`. It used to widen `degrees` + instead of `shear`, which left the shear range incomplete and made the sampling fail (#354). * `transform_crop()` now pads the result with zeros when the crop leaves the image, so that the output always has the requested size. It previously returned only the part of the image the crop covered, which could even be empty. diff --git a/R/item-transforms-random-geometry.R b/R/item-transforms-random-geometry.R index ffc871a2..39732ec8 100644 --- a/R/item-transforms-random-geometry.R +++ b/R/item-transforms-random-geometry.R @@ -165,3 +165,115 @@ item_transform_random_vertical_flip.image_with_rotated_box <- function(x, p = 0. } x } + +#' Randomly apply an affine transformation on a dataset item +#' +#' Draws a random rotation, translation, scale and shear inside the given ranges +#' and applies the resulting affine transformation to the dataset item with +#' \code{\link{item_transform_affine}}, keeping the image size unchanged. Image +#' and target share the same draw, so that they stay aligned. +#' +#' The transformation is drawn again for every item, so that a dataset wrapped +#' with this transform yields a different transformation on each access. +#' +#' @param x A dataset item, typically an \code{image_with_bounding_box}, +#' \code{image_with_rotated_box} or \code{image_with_segmentation_mask} object +#' containing an image tensor and associated target data. +#' @inheritParams transform_random_affine +#' @param fill Fill color for the area outside the transform. Default is +#' \code{NULL}. +#' @param center (numeric vector of length 2, optional): Optional center of +#' rotation, \code{c(x, y)}. Default is image center. +#' +#' @return A dataset item with the image and target transformed. Detection items +#' are returned as \code{image_with_rotated_box}; segmentation items keep their +#' class. +#' +#' @examples +#' \dontrun{ +#' url <- "https://upload.wikimedia.org/wikipedia/commons/b/b6/Felis_catus-cat_on_snow.jpg" +#' img <- base_loader(url) |> transform_to_tensor() +#' +#' boxes <- torch_tensor(matrix(c(600, 200, 2880, 1860), ncol = 4), dtype = torch_float32()) +#' +#' before <- list(x = img, y = list(boxes = boxes, labels = "cat", +#' image_height = img$shape[2], image_width = img$shape[3])) +#' class(before) <- c("image_with_bounding_box", "list") +#' +#' after <- item_transform_random_affine(before, degrees = 30, translate = c(0.1, 0.1), +#' scale = c(0.8, 1.2), shear = 10) +#' +#' before_plot <- draw_bounding_boxes(before, colors = "blue", width = 10) +#' after_plot <- draw_bounding_boxes(after, colors = "red", width = 10) +#' tensor_image_browse(before_plot) +#' tensor_image_browse(after_plot) +#' } +#' +#' @family item_random_transforms +#' +#' @export +item_transform_random_affine <- function(x, degrees, translate = NULL, scale = NULL, + shear = NULL, interpolation = 0, fill = NULL, + center = NULL) { + UseMethod("item_transform_random_affine", x) +} + +#' @export +item_transform_random_affine.default <- function(x, degrees, translate = NULL, scale = NULL, + shear = NULL, interpolation = 0, fill = NULL, + center = NULL) { + cli_abort( + "{.fn item_transform_random_affine} requires a dataset item (a list with {.var x} and {.var y} fields), not {.obj_type_friendly {x}}. + To transform a raw image tensor, use {.fn transform_random_affine} instead." + ) +} + +#' @export +item_transform_random_affine.dataset <- function(x, degrees, translate = NULL, scale = NULL, + shear = NULL, interpolation = 0, fill = NULL, + center = NULL) { + original_getitem <- x$.getitem + unlockBinding(".getitem", as.environment(x)) + x$.getitem <- function(index) { + item <- original_getitem(index) + item_transform_random_affine(item, degrees = degrees, translate = translate, + scale = scale, shear = shear, + interpolation = interpolation, fill = fill, + center = center) + } + x +} + +#' @export +item_transform_random_affine.image_with_bounding_box <- function(x, degrees, translate = NULL, + scale = NULL, shear = NULL, + interpolation = 0, fill = NULL, + center = NULL) { + random_affine_item(x, degrees, translate, scale, shear, interpolation, fill, center) +} + +#' @export +item_transform_random_affine.image_with_rotated_box <- function(x, degrees, translate = NULL, + scale = NULL, shear = NULL, + interpolation = 0, fill = NULL, + center = NULL) { + random_affine_item(x, degrees, translate, scale, shear, interpolation, fill, center) +} + +#' @export +item_transform_random_affine.image_with_segmentation_mask <- function(x, degrees, translate = NULL, + scale = NULL, shear = NULL, + interpolation = 0, fill = NULL, + center = NULL) { + random_affine_item(x, degrees, translate, scale, shear, interpolation, fill, center) +} + +random_affine_item <- function(x, degrees, translate, scale, shear, interpolation, fill, center) { + args <- check_random_affine_params(degrees, translate, scale, shear) + params <- get_random_affine_params(args$degrees, translate, scale, args$shear, + get_image_size(x$x)) + + item_transform_affine(x, angle = params[[1]], translate = params[[2]], + scale = params[[3]], shear = params[[4]], + interpolation = interpolation, fill = fill, center = center) +} diff --git a/R/transforms-defaults.R b/R/transforms-defaults.R index 335eb127..85f684a6 100644 --- a/R/transforms-defaults.R +++ b/R/transforms-defaults.R @@ -366,6 +366,57 @@ transform_random_rotation.default <- function(img, degrees, interpolation=0, } +check_random_affine_params <- function(degrees, translate, scale, shear) { + if (length(degrees) == 1) { + + if (degrees < 0) + value_error("degrees must be positive if it's a single value") + + degrees <- c(-degrees, degrees) + + } else if (length(degrees) != 2) { + value_error("degrees must be length 1 or 2") + } + + + if (!is.null(translate)) { + + if (length(translate) != 2) + value_error("translate must be length 2") + + if (any(translate > 1) || any(translate < 0)) + value_error("translate must be between 0 and 1") + + } + + if (!is.null(scale)) { + + if (length(scale) != 2) + value_error("scale must be length 2") + + if (any(scale < 0)) + value_error("scale must be positive") + + } + + if (!is.null(shear)) { + + if (length(shear) == 1) { + + if (shear < 0) + value_error("shear must be positive if it's a single value") + + shear <- c(-shear, shear) + + } else if (!length(shear) %in% c(2, 4)) { + value_error("shear's length must be 1, 2, or 4") + } + + } + + list(degrees = degrees, shear = shear) +} + get_random_affine_params <- function(degrees, translate, scale_ranges, @@ -415,57 +466,11 @@ transform_random_affine.default <- function(img, degrees, translate=NULL, scale= fill <- fillcolor } - if (length(degrees) == 1) { - - if (degrees < 0) - value_error("degrees must be positive if it's a single value") - - degrees <- c(-degrees, degrees) - - } else if (length(degrees) != 2) { - value_error("degrees must be length 1 or 2") - } - - - if (!is.null(translate)) { - - if (length(translate) != 2) - value_error("translate must be length 2") - - if (any(translate > 1) || any(translate < 0)) - value_error("translate must be between 0 and 1") - - } - - if (!is.null(scale)) { - - if (length(scale) != 2) - value_error("scale must be length 2") - - if (any(scale < 0)) - value_error("scale must be positive") - - } - - if (!is.null(shear)) { - - if (length(shear) == 1) { - - if (shear < 0) - value_error("shear must be positive if it's a single value") - - degrees <- c(-degrees, degrees) - - } else if (!length(shear) %in% c(2, 4)) { - value_error("shear's length must be 1, 2, or 4") - } - - } - + args <- check_random_affine_params(degrees, translate, scale, shear) img_size <- get_image_size(img) - ret <- get_random_affine_params(degrees, translate, scale, shear, img_size) + ret <- get_random_affine_params(args$degrees, translate, scale, args$shear, img_size) transform_affine(img, ret[[1]], ret[[2]], ret[[3]], ret[[4]], interpolation=interpolation, fill=fill) diff --git a/man/item_transform_random_affine.Rd b/man/item_transform_random_affine.Rd new file mode 100644 index 00000000..6d2a205f --- /dev/null +++ b/man/item_transform_random_affine.Rd @@ -0,0 +1,99 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/item-transforms-random-geometry.R +\name{item_transform_random_affine} +\alias{item_transform_random_affine} +\title{Randomly apply an affine transformation on a dataset item} +\usage{ +item_transform_random_affine( + x, + degrees, + translate = NULL, + scale = NULL, + shear = NULL, + interpolation = 0, + fill = NULL, + center = NULL +) +} +\arguments{ +\item{x}{A dataset item, typically an \code{image_with_bounding_box}, +\code{image_with_rotated_box} or \code{image_with_segmentation_mask} object +containing an image tensor and associated target data.} + +\item{degrees}{(numeric vector of length 2 or numeric): Range of degrees to +select from. If \code{degrees} is a bare number instead of a numeric vector of +length 2 like \code{c(min, max)}, the range of degrees will be +\verb{(-degrees, +degrees)}.} + +\item{translate}{(numeric vector of length 2, optional): Numeric vector of +maximum absolute fraction for horizontal and vertical translations. For +example \code{translate = c(a, b)}, then horizontal shift is randomly sampled +in the range \verb{-img_width * a < dx < img_width * a} and vertical shift is +randomly sampled in the range +\verb{-img_height * b < dy < img_height * b}. Will not translate by default.} + +\item{scale}{(numeric vector of length 2, optional): Scaling factor +interval, e.g. \code{c(a, b)}, then scale is randomly sampled from the range +\verb{a <= scale <= b}. Will keep original scale by default.} + +\item{shear}{(numeric vector or numeric, optional): Range of degrees to +select from. If \code{shear} is a bare number, a shear parallel to the x axis +in the range \verb{(-shear, +shear)} will be applied. Else if \code{shear} is a +numeric vector of length 2, a shear parallel to the x axis in the range +\verb{(shear[1], shear[2])} will be applied. Else if \code{shear} is a numeric +vector of length 4, a x-axis shear in \verb{(shear[1], shear[2])} and y-axis +shear in \verb{(shear[3], shear[4])} will be applied. Will not apply shear by +default.} + +\item{interpolation}{(integer or character, optional): Interpolation mode. +Supported values are \code{0} / \code{"nearest"} and \code{2} / \code{"bilinear"}. Default +is \code{0}.} + +\item{fill}{Fill color for the area outside the transform. Default is +\code{NULL}.} + +\item{center}{(numeric vector of length 2, optional): Optional center of +rotation, \code{c(x, y)}. Default is image center.} +} +\value{ +A dataset item with the image and target transformed. Detection items +are returned as \code{image_with_rotated_box}; segmentation items keep their +class. +} +\description{ +Draws a random rotation, translation, scale and shear inside the given ranges +and applies the resulting affine transformation to the dataset item with +\code{\link{item_transform_affine}}, keeping the image size unchanged. Image +and target share the same draw, so that they stay aligned. +} +\details{ +The transformation is drawn again for every item, so that a dataset wrapped +with this transform yields a different transformation on each access. +} +\examples{ +\dontrun{ +url <- "https://upload.wikimedia.org/wikipedia/commons/b/b6/Felis_catus-cat_on_snow.jpg" +img <- base_loader(url) |> transform_to_tensor() + +boxes <- torch_tensor(matrix(c(600, 200, 2880, 1860), ncol = 4), dtype = torch_float32()) + +before <- list(x = img, y = list(boxes = boxes, labels = "cat", + image_height = img$shape[2], image_width = img$shape[3])) +class(before) <- c("image_with_bounding_box", "list") + +after <- item_transform_random_affine(before, degrees = 30, translate = c(0.1, 0.1), + scale = c(0.8, 1.2), shear = 10) + +before_plot <- draw_bounding_boxes(before, colors = "blue", width = 10) +after_plot <- draw_bounding_boxes(after, colors = "red", width = 10) +tensor_image_browse(before_plot) +tensor_image_browse(after_plot) +} + +} +\seealso{ +Other item_random_transforms: +\code{\link[=item_transform_random_horizontal_flip]{item_transform_random_horizontal_flip()}}, +\code{\link[=item_transform_random_vertical_flip]{item_transform_random_vertical_flip()}} +} +\concept{item_random_transforms} diff --git a/man/item_transform_random_horizontal_flip.Rd b/man/item_transform_random_horizontal_flip.Rd index 1880840b..b61db145 100644 --- a/man/item_transform_random_horizontal_flip.Rd +++ b/man/item_transform_random_horizontal_flip.Rd @@ -45,6 +45,7 @@ tensor_image_browse(grid) } \seealso{ Other item_random_transforms: +\code{\link[=item_transform_random_affine]{item_transform_random_affine()}}, \code{\link[=item_transform_random_vertical_flip]{item_transform_random_vertical_flip()}} } \concept{item_random_transforms} diff --git a/man/item_transform_random_vertical_flip.Rd b/man/item_transform_random_vertical_flip.Rd index 2567f0f8..59986377 100644 --- a/man/item_transform_random_vertical_flip.Rd +++ b/man/item_transform_random_vertical_flip.Rd @@ -45,6 +45,7 @@ tensor_image_browse(grid) } \seealso{ Other item_random_transforms: +\code{\link[=item_transform_random_affine]{item_transform_random_affine()}}, \code{\link[=item_transform_random_horizontal_flip]{item_transform_random_horizontal_flip()}} } \concept{item_random_transforms} diff --git a/tests/testthat/test-item-transforms-random-geometry.R b/tests/testthat/test-item-transforms-random-geometry.R index 00965630..9dcd9487 100644 --- a/tests/testthat/test-item-transforms-random-geometry.R +++ b/tests/testthat/test-item-transforms-random-geometry.R @@ -145,3 +145,183 @@ test_that("item_transform_random_vertical_flip default p is 0.5", { fmls <- formals(item_transform_random_vertical_flip) expect_equal(fmls$p, 0.5) }) + +# --- item_transform_random_affine --- + +test_that("item_transform_random_affine rejects non-item inputs", { + img <- torch_randn(3, 100, 200) + expect_error( + item_transform_random_affine(img, degrees = 30), + "requires a dataset item" + ) + expect_error( + item_transform_random_affine(42, degrees = 30), + "requires a dataset item" + ) +}) + +test_that("item_transform_random_affine validates its ranges", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4)) + + expect_error(item_transform_random_affine(item, degrees = -10), "degrees must be positive") + expect_error(item_transform_random_affine(item, degrees = c(-10, 0, 10)), "degrees must be length 1 or 2") + expect_error(item_transform_random_affine(item, degrees = 10, translate = 0.1), "translate must be length 2") + expect_error(item_transform_random_affine(item, degrees = 10, translate = c(0.1, 2)), "translate must be between 0 and 1") + expect_error(item_transform_random_affine(item, degrees = 10, scale = 0.5), "scale must be length 2") + expect_error(item_transform_random_affine(item, degrees = 10, scale = c(-1, 1)), "scale must be positive") + expect_error(item_transform_random_affine(item, degrees = 10, shear = -5), "shear must be positive") + expect_error(item_transform_random_affine(item, degrees = 10, shear = c(1, 2, 3)), "shear's length must be 1, 2, or 4") +}) + +test_that("item_transform_random_affine with a zero range is the identity", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4), image_size = c(100L, 200L)) + original_img <- item$x$clone() + + result <- item_transform_random_affine(item, degrees = 0) + + expect_s3_class(result, "image_with_rotated_box") + expect_true(torch_equal(result$x, original_img)) + expect_equal_to_r(result$y$boxes, matrix(c(10, 20, 50, 60, 0), ncol = 5)) +}) + +test_that("item_transform_random_affine matches the affine it composes", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4), image_size = c(100L, 200L)) + + set.seed(1) + torch_manual_seed(1) + params <- get_random_affine_params(c(-30, 30), c(0.1, 0.2), c(0.8, 1.2), c(-10, 10), + get_image_size(item$x)) + + set.seed(1) + torch_manual_seed(1) + result <- item_transform_random_affine(item, degrees = 30, translate = c(0.1, 0.2), + scale = c(0.8, 1.2), shear = 10) + + expected <- item_transform_affine(item, angle = params[[1]], translate = params[[2]], + scale = params[[3]], shear = params[[4]]) + + expect_true(torch_equal(result$x, expected$x)) + expect_equal_to_r(result$y$boxes, as_array(expected$y$boxes)) +}) + +test_that("item_transform_random_affine draws the angle inside the given range", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4), image_size = c(100L, 200L)) + + angles <- vapply(1:20, function(i) { + as.numeric(item_transform_random_affine(item, degrees = 30)$y$boxes[1, 5]) + }, numeric(1)) + + expect_true(all(angles >= -30 & angles <= 30)) + expect_gt(length(unique(angles)), 1) +}) + +test_that("item_transform_random_affine draws the translation inside the given range", { + item <- make_detection_item(matrix(c(80, 40, 120, 60), ncol = 4), image_size = c(100L, 200L)) + + shifts <- vapply(1:20, function(i) { + boxes <- item_transform_random_affine(item, degrees = 0, translate = c(0.1, 0.2))$y$boxes + c(as.numeric(boxes[1, 1]) - 80, as.numeric(boxes[1, 2]) - 40) + }, numeric(2)) + + expect_true(all(abs(shifts[1, ]) <= 0.1 * 200)) + expect_true(all(abs(shifts[2, ]) <= 0.2 * 100)) +}) + +test_that("item_transform_random_affine keeps the image size and dtype for detection", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4), image_size = c(100L, 200L)) + + result <- item_transform_random_affine(item, degrees = 30, translate = c(0.1, 0.1), + scale = c(0.8, 1.2), shear = 10) + + expect_tensor_shape(result$x, c(3, 100, 200)) + expect_tensor_dtype(result$x, item$x$dtype) + expect_tensor_shape(result$y$boxes, c(1, 5)) +}) + +test_that("item_transform_random_affine preserves labels and handles empty boxes", { + labels <- torch_tensor(c(1L, 2L), dtype = torch_long()) + item <- make_detection_item( + matrix(c(10, 20, 50, 60, 5, 5, 15, 25), ncol = 4, byrow = TRUE), + labels = labels + ) + result <- item_transform_random_affine(item, degrees = 30) + + expect_true(result$y$labels$eq(labels)$all()$item()) + + item <- make_detection_item( + boxes = matrix(numeric(0), ncol = 4), + labels = torch_zeros(0L, dtype = torch_long()) + ) + result <- item_transform_random_affine(item, degrees = 30) + + expect_tensor_shape(result$y$boxes, c(0, 5)) +}) + +test_that("item_transform_random_affine does not mutate its input", { + item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4)) + original_img <- item$x$clone() + original_boxes <- as_array(item$y$boxes) + + item_transform_random_affine(item, degrees = 30, translate = c(0.1, 0.1)) + + expect_true(torch_equal(item$x, original_img)) + expect_equal_to_r(item$y$boxes, original_boxes) +}) + +test_that("item_transform_random_affine transforms segmentation masks", { + item <- make_segmentation_item(image_size = c(100L, 200L), num_masks = 2L) + original_masks <- item$y$masks$clone() + + result <- item_transform_random_affine(item, degrees = c(45, 45)) + + expect_s3_class(result, "image_with_segmentation_mask") + expect_tensor_shape(result$x, c(3, 100, 200)) + expect_tensor_shape(result$y$masks, original_masks$shape) + expect_tensor_dtype(result$y$masks, torch_bool()) + expect_false(result$y$masks$equal(original_masks)) +}) + +test_that("item_transform_random_affine keeps rotated boxes rotated", { + item <- make_detection_item(matrix(c(20, 30, 80, 90), ncol = 4), image_size = c(100L, 100L)) + rotated <- item_transform_rotate(item, angle = 30, expand = FALSE) + + result <- item_transform_random_affine(rotated, degrees = c(0, 0)) + + expect_s3_class(result, "image_with_rotated_box") + expect_tensor_shape(result$y$boxes, c(1, 5)) +}) + +test_that("item_transform_random_affine works on detection and segmentation datasets", { + detection_item <- make_detection_item(matrix(c(10, 20, 50, 60), ncol = 4), image_size = c(100L, 200L)) + ds <- dataset( + name = "toy_detection", + initialize = function() {}, + .getitem = function(index) detection_item, + .length = function() 1L + )() + + ds <- item_transform_random_affine(ds, degrees = 30, translate = c(0.1, 0.1)) + item <- ds$.getitem(1) + + expect_s3_class(item, "image_with_rotated_box") + expect_tensor_shape(item$x, c(3, 100, 200)) + + other <- ds$.getitem(1) + expect_false(torch_equal(item$x, other$x)) + expect_equal_to_r(detection_item$y$boxes, matrix(c(10, 20, 50, 60), ncol = 4)) + + ds <- dataset( + name = "toy_segmentation", + initialize = function() {}, + .getitem = function(index) { + make_segmentation_item(image_size = c(100L, 200L), num_masks = 2L) + }, + .length = function() 1L + )() + + ds <- item_transform_random_affine(ds, degrees = 30) + item <- ds$.getitem(1) + + expect_s3_class(item, "image_with_segmentation_mask") + expect_tensor_shape(item$y$masks, c(2, 100, 200)) +}) diff --git a/tests/testthat/test-transforms-tensor.R b/tests/testthat/test-transforms-tensor.R index c16b096c..044b3ed3 100644 --- a/tests/testthat/test-transforms-tensor.R +++ b/tests/testthat/test-transforms-tensor.R @@ -308,6 +308,9 @@ test_that("random_affine", { expect_lte(as.numeric(torch_sum(x) - 1), as.numeric(torch_sum(ob))) expect_gte(as.numeric(torch_sum(x)), as.numeric(torch_sum(ob))) + o <- transform_random_affine(x, 0, shear = 10) + expect_tensor_shape(o, c(1, 8, 8)) + }) test_that("affine", {