diff --git a/HISTORY.md b/HISTORY.md index e734fd9950..f5e3a608e5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -17,8 +17,11 @@ _????-??-??_ * Fix conversion of empty Armadillo objects to numpy in Python bindings (#3896). - * Added bootstrap strategies for RandomForest: IdentityBootstrap, - DefaultBootstrap, and SequentialBootstrap. (#3829) + * Added bootstrap strategies for `RandomForest`: `IdentityBootstrap`, + `DefaultBootstrap`, and `SequentialBootstrap` (#3829). + + * Add `ResizeCropImages()` for resize-and-crop image preprocessing + functionality (#3903). ## mlpack 4.5.1 diff --git a/doc/img/cat.jpg b/doc/img/cat.jpg new file mode 100644 index 0000000000..24b5b7a9ef Binary files /dev/null and b/doc/img/cat.jpg differ diff --git a/doc/img/cat_cropped.jpg b/doc/img/cat_cropped.jpg new file mode 100644 index 0000000000..e85654fe4d Binary files /dev/null and b/doc/img/cat_cropped.jpg differ diff --git a/doc/img/cat_rect.jpg b/doc/img/cat_rect.jpg new file mode 100644 index 0000000000..b07d134d0c Binary files /dev/null and b/doc/img/cat_rect.jpg differ diff --git a/doc/img/cat_scaled_rect.jpg b/doc/img/cat_scaled_rect.jpg new file mode 100644 index 0000000000..b7e24f62e5 Binary files /dev/null and b/doc/img/cat_scaled_rect.jpg differ diff --git a/doc/user/load_save.md b/doc/user/load_save.md index 358aec69c8..37f9e80ded 100644 --- a/doc/user/load_save.md +++ b/doc/user/load_save.md @@ -562,13 +562,15 @@ mlpack::data::ImageInfo info; // be used instead. std::vector files = {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", - "sheep_5.jpg", "sheep_6.jpg"}; + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; // The resized images will be saved locally. We are declaring the vector that // contains the names of the resized images. std::vector reSheeps = {"re_sheep_1.jpg", "re_sheep_2.jpg", "re_sheep_3.jpg", "re_sheep_4.jpg", - "re_sheep_5.jpg", "re_sheep_6.jpg"}; + "re_sheep_5.jpg", "re_sheep_6.jpg", "re_sheep_7.jpg", "re_sheep_8.jpg", + "re_sheep_9.jpg"}; // Load and Resize each one of them individually, because they do not have // the same dimensions. The `info` will contain the dimension for each one. @@ -593,7 +595,8 @@ mlpack::data::ImageInfo info; std::vector reSheeps = {"re_sheep_1.jpg", "re_sheep_2.jpg", "re_sheep_3.jpg", "re_sheep_4.jpg", - "re_sheep_5.jpg", "re_sheep_6.jpg"}; + "re_sheep_5.jpg", "re_sheep_6.jpg", "re_sheep_7.jpg", "re_sheep_8.jpg", + "re_sheep_9.jpg"}; mlpack::data::Load(reSheeps, images, info, false); @@ -604,11 +607,109 @@ mlpack::data::ResizeImages(images, info, 160, 160); // contains the names of the resized images. std::vector smSheeps = {"sm_sheep_1.jpg", "sm_sheep_2.jpg", "sm_sheep_3.jpg", "sm_sheep_4.jpg", - "sm_sheep_5.jpg", "sm_sheep_6.jpg"}; + "sm_sheep_5.jpg", "sm_sheep_6.jpg", "sm_sheep_7.jpg", "sm_sheep_8.jpg", + "sm_sheep_9.jpg"}; mlpack::data::Save(smSheeps, images, info, false); ``` +### Resize and crop images + +In addition to resizing images, mlpack also provides resize-and-crop +functionality. This is useful when the desired aspect ratio of an image differs +largely from the original image. + +The resize-and-crop operation, given a target size `outputWidth` x +`outputHeight`, first resizes the image while preserving the aspect ratio such +that the width and height of the image both no smaller than `outputWidth` and +`outputHeight`. Then, the image is cropped to have size `outputWidth` by +`outputHeight`, keeping the center pixels only. This process is shown below. + +*Original image:* + +

+ cat +

+ +*Original image with target size of* `220`x`220` *pixels:* + +

+ cat with rectangle overlaid +

+ +*First step: resize while preserving aspect ratio:* + +

+ scaled cat with rectangle overlaid +

+ +*Second step: crop to desired final size:* + +

+ cropped cat +

+ +- `ResizeCropImages(images, info, newWidth, newHeight)` + * `images` is a [column-major matrix](matrices.md) containing a set of + images; each image is represented as a flattened vector in one column. + + * `info` is a [`data::ImageInfo&`](#dataimageinfo) containing details about + the images in `images`. + + * `images` and `info` are modified in-place. + + * `newWidth` and `newHeight` (of type `size_t`) are the desired new + dimensions of the resized images. + - If the output size is larger than the input image size, the images will + be upscaled the minimum amount necessary before cropping. + - If the aspect ratio is not changed from the input aspect ratio, no + cropping is performed. + + * ***NOTE:*** if the element type of `images` is not `unsigned char` or + `float` (e.g. if `image` is not `arma::Mat` or + `arma::fmat`), the matrix will be temporarily converted during resizing; + therefore, using `unsigned char` or `float` as the element type is the most + efficient. + + * This function expects all the images to have identical dimensions. If this + is not the case, iteratively call `ResizeCropImages()` with a single + image/column in `images`. + +Example usage of the `ResizeCropImages()` function on a set of images with +different dimensions: + +```c++ +// See https://datasets.mlpack.org/sheep.tar.bz2. +arma::Mat image; +mlpack::data::ImageInfo info; + +// The images are located in our test/data directory. However, any image could +// be used instead. +std::vector files = + {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; + +// The resized images will be saved locally. We are declaring the vector that +// contains the names of the resized and cropped images. +std::vector cropSheeps = + {"crop_sheep_1.jpg", "crop_sheep_2.jpg", "crop_sheep_3.jpg", + "crop_sheep_4.jpg", "crop_sheep_5.jpg", "crop_sheep_6.jpg", + "crop_sheep_7.jpg", "crop_sheep_8.jpg", "crop_sheep_9.jpg"}; + +// Load and resize-and-crop each image individually, because they do not have +// the same dimensions. The `info` will contain the dimension for each one. +for (size_t i = 0; i < files.size(); i++) +{ + mlpack::data::Load(files.at(i), image, info, false); + mlpack::data::ResizeCropImages(image, info, 320, 320); + mlpack::data::Save(cropSheeps.at(i), image, info, false); + std::cout << "Resized and cropped " << files.at(i) << " to " + << cropSheeps.at(i) << " with output size 320x320." << std::endl; +} +``` + ## mlpack objects All mlpack objects can be saved with `data::Save()` and loaded with diff --git a/src/mlpack/core/data/image_resize_crop.hpp b/src/mlpack/core/data/image_resize_crop.hpp index 4be6bdecd6..dba5c5c5a2 100644 --- a/src/mlpack/core/data/image_resize_crop.hpp +++ b/src/mlpack/core/data/image_resize_crop.hpp @@ -128,6 +128,87 @@ inline void ResizeImages(arma::Mat& images, data::ImageInfo& info, info.Height() = newHeight; } +/** + * Resize & Crop one single image matrix or a set of images. + * + * This function should be used if the image is loaded as an armadillo matrix + * and the number of cols equal to the Width and the number of rows equal + * the Height of the image, or the total number of image pixels is equal to the + * number of element in an armadillo matrix. + * + * The same applies if a set of images is loaded, but all of them need to have + * identical dimension when loaded to this matrix. + * + * @param image The input matrix that contains the image to be resized. + * @param info Contains relevant input images information. + * @param newWidth The new requested width for the resized image. + * @param newHeight The new requested height for the resized image. + */ +template +inline void ResizeCropImages(arma::Mat& images, data::ImageInfo& info, + const size_t newWidth, const size_t newHeight) +{ + float ratioW = static_cast(newWidth) / + static_cast(info.Width()); + float ratioH = static_cast(newHeight) / + static_cast(info.Height()); + + float largestRatio = ratioW > ratioH ? ratioW : ratioH; + int midWidth = static_cast(largestRatio * info.Width()); + int midHeight = static_cast(largestRatio * info.Height()); + + // Edge cases, what if the width / height value is odd ? then increase the + // resize value to the closest pair number. + // We have to avoid touching the image, of the user ask for it. + // Add a condition to prevent cropping the image if the user did not ask for + // any modification. Because cropping depends on the aspect ratio. + if (ratioH != 1 || ratioW != 1) + { + if (midHeight % 2 != 0) + midHeight = midHeight + 1; + if (midWidth % 2 != 0) + midWidth = midWidth + 1; + + ResizeImages(images, info, midWidth, midHeight); + int nColsCrop = midWidth > midHeight ? (midWidth - midHeight) : 0; + int nRowsCrop = midHeight > midWidth ? (midHeight - midWidth) : 0; + + //temporary matrix to hold the images while being resized. + arma::Mat tmpImages(newHeight * newWidth * info.Channels(), + images.n_cols); + if (nRowsCrop != 0) + { + int cropUpDownEqually = (nRowsCrop / 2) * info.Channels() * midWidth; + tmpImages = images.rows(cropUpDownEqually, + images.n_rows - cropUpDownEqually - 1); + } + + #pragma omp parallel for + for (size_t u = 0; u < images.n_cols; ++u) + { + if (nColsCrop != 0) + { + // Saving some memory by avoiding copying the images. + // R into Row 1. + // G into Row 2. + // B into Row 3. + // Cols are the Width, no change + // Slices are the Height of the image instead of rows. + arma::Cube cube(images.colptr(u), info.Channels(), midWidth, + midHeight, false, false); + tmpImages.col(u) = vectorise(cube.cols((nColsCrop / 2), + (cube.n_cols - (nColsCrop / 2) - 1))); + } + } + if (nRowsCrop != 0 || nColsCrop != 0) + { + images = std::move(tmpImages); + } + } + info.Width() = newWidth; + info.Height() = newHeight; +} + } // namespace data } // namespace mlpack diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 834232fa32..d9efa8b8c2 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -260,6 +260,8 @@ add_custom_command(TARGET mlpack_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/data/ ${PROJECT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/doc/img/cat.jpg + ${PROJECT_BINARY_DIR}/ ) add_custom_command(TARGET mlpack_test diff --git a/src/mlpack/tests/data/sheep.tar.bz2 b/src/mlpack/tests/data/sheep.tar.bz2 index 77b08e8236..56f186b4a4 100644 Binary files a/src/mlpack/tests/data/sheep.tar.bz2 and b/src/mlpack/tests/data/sheep.tar.bz2 differ diff --git a/src/mlpack/tests/data/test_image.png b/src/mlpack/tests/data/test_image.png index 5a29595a1e..aec10baa93 100644 Binary files a/src/mlpack/tests/data/test_image.png and b/src/mlpack/tests/data/test_image.png differ diff --git a/src/mlpack/tests/image_test.cpp b/src/mlpack/tests/image_test.cpp index c3fd54b099..b728576338 100644 --- a/src/mlpack/tests/image_test.cpp +++ b/src/mlpack/tests/image_test.cpp @@ -163,13 +163,16 @@ TEMPLATE_TEST_CASE("ImagesResizeTest", "[ImageTest]", unsigned char, size_t, data::ImageInfo info, resizedInfo, resizedInfo2; std::vector files = {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", - "sheep_5.jpg", "sheep_6.jpg"}; + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; std::vector reSheeps = {"re_sheep_1.jpg", "re_sheep_2.jpg", "re_sheep_3.jpg", "re_sheep_4.jpg", - "re_sheep_5.jpg", "re_sheep_6.jpg"}; + "re_sheep_5.jpg", "re_sheep_6.jpg", "re_sheep_7.jpg", "re_sheep_8.jpg", + "re_sheep_9.jpg"}; std::vector smSheeps = {"sm_sheep_1.jpg", "sm_sheep_2.jpg", "sm_sheep_3.jpg", "sm_sheep_4.jpg", - "sm_sheep_5.jpg", "sm_sheep_6.jpg"}; + "sm_sheep_5.jpg", "sm_sheep_6.jpg", "sm_sheep_7.jpg", "sm_sheep_8.jpg", + "sm_sheep_9.jpg"}; // Load and Resize each one of them individually, because they do not have // the same sizes, and then the resized images, will be used in the next @@ -206,6 +209,65 @@ TEMPLATE_TEST_CASE("ImagesResizeTest", "[ImageTest]", unsigned char, size_t, } } +/** + * Test resize the image if this is done correctly. Try it with a few different + * types. + */ +TEMPLATE_TEST_CASE("ImagesResizeCropTest", "[ImageTest]", unsigned char, + size_t, float, double) +{ + typedef TestType eT; + + arma::Mat image, images; + data::ImageInfo info, resizedInfo, resizedInfo2; + std::vector files = + {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; + std::vector reSheeps = + {"re_sheep_1.jpg", "re_sheep_2.jpg", "re_sheep_3.jpg", "re_sheep_4.jpg", + "re_sheep_5.jpg", "re_sheep_6.jpg", "re_sheep_7.jpg", "re_sheep_8.jpg", + "re_sheep_9.jpg"}; + std::vector smSheeps = + {"sm_sheep_1.jpg", "sm_sheep_2.jpg", "sm_sheep_3.jpg", "sm_sheep_4.jpg", + "sm_sheep_5.jpg", "sm_sheep_6.jpg", "sm_sheep_7.jpg", "sm_sheep_8.jpg", + "sm_sheep_9.jpg"}; + + // Load and Resize each one of them individually, because they do not have + // the same sizes, and then the resized images, will be used in the next + // test. + for (size_t i = 0; i < files.size(); i++) + { + REQUIRE(data::Load(files.at(i), image, info, false) == true); + ResizeCropImages(image, info, 320, 320); + REQUIRE(data::Save(reSheeps.at(i), image, info, false) == true); + } + + // Since they are all resized, this should passes + REQUIRE(data::Load(reSheeps, images, resizedInfo, false) == true); + + REQUIRE(info.Width() == resizedInfo.Width()); + REQUIRE(info.Height() == resizedInfo.Height()); + + REQUIRE(data::Load(reSheeps, images, info, false) == true); + + ResizeCropImages(images, info, 160, 160); + + REQUIRE(data::Save(smSheeps, images, info, false) == true); + + REQUIRE(data::Load(smSheeps, images, resizedInfo2, false) == true); + + REQUIRE(info.Width() == resizedInfo2.Width()); + REQUIRE(info.Height() == resizedInfo2.Height()); + + // cleanup generated images. + for (size_t i = 0; i < reSheeps.size(); ++i) + { + remove(reSheeps.at(i).c_str()); + remove(smSheeps.at(i).c_str()); + } +} + /** * Test if we resize to the same original dimension we will get the same pixels * and no modification to the image. Try it with a few different types. @@ -219,7 +281,8 @@ TEMPLATE_TEST_CASE("IdenticalResizeTest", "[ImageTest]", unsigned char, size_t, data::ImageInfo info; std::vector files = {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", - "sheep_5.jpg", "sheep_6.jpg"}; + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; for (size_t i = 0; i < files.size(); i++) { @@ -236,3 +299,103 @@ TEMPLATE_TEST_CASE("IdenticalResizeTest", "[ImageTest]", unsigned char, size_t, } } +/** + * Test if we resize to the same original dimension we will get the same pixels + * and no modification to the image. Try it with a few different types. + */ +TEMPLATE_TEST_CASE("IdenticalResizeCropTest", "[ImageTest]", unsigned char, + size_t, float, double) +{ + typedef TestType eT; + + arma::Mat image; + data::ImageInfo info; + std::vector files = + {"sheep_1.jpg", "sheep_2.jpg", "sheep_3.jpg", "sheep_4.jpg", + "sheep_5.jpg", "sheep_6.jpg", "sheep_7.jpg", "sheep_8.jpg", + "sheep_9.jpg"}; + + for (size_t i = 0; i < files.size(); i++) + { + REQUIRE(data::Load(files.at(i), image, info, false) == true); + arma::Mat originalImage = image; + ResizeCropImages(image, info, info.Width(), info.Height()); + for (size_t i = 0; i < originalImage.n_rows; ++i) + { + for (size_t j = 0; j < originalImage.n_cols; ++j) + { + REQUIRE(originalImage.at(i, j) == image.at(i, j)); + } + } + } +} + +/** + * Test that if we resize an image, we get the pixels that we expect. + */ +TEMPLATE_TEST_CASE("ResizeCropPixelTest", "[ImageTest]", unsigned char, size_t, + float, double) +{ + typedef TestType eT; + + // Load cat.jpg, which has a strange aspect ratio. + arma::Mat image; + data::ImageInfo info; + REQUIRE(data::Load("cat.jpg", image, info, false) == true); + + // When we crop to match the height of the image, no resizing is needed and we + // can compare pixels directly. + const size_t inputWidth = info.Width(); + const size_t inputHeight = info.Height(); + const size_t inputChannels = info.Channels(); + const size_t leftOffset = (info.Width() - info.Height()) / 2; + arma::Mat oldImage(image); + ResizeCropImages(image, info, inputHeight, inputHeight); + + REQUIRE(info.Height() == inputHeight); + REQUIRE(info.Width() == inputHeight); + REQUIRE(info.Channels() == inputChannels); + REQUIRE(image.n_elem == info.Height() * info.Width() * info.Channels()); + + // Now make sure that all of the pixels are the same as from the center of the + // image. + for (size_t i = 0; i < image.n_elem; ++i) + { + const size_t channel = i % info.Channels(); + const size_t pixel = (i / info.Channels()); + const size_t x = pixel % info.Width(); + const size_t y = pixel / info.Width(); + + const size_t inputPixel = y * (inputWidth * inputChannels) + + (x + leftOffset) * inputChannels + channel; + const size_t outputPixel = y * (info.Width() * info.Channels()) + + x * info.Channels() + channel; + + REQUIRE(oldImage[inputPixel] == Approx(image[outputPixel])); + } +} + +/** + * Test that images can be upscaled if desired. + */ +TEMPLATE_TEST_CASE("ResizeCropUpscaleTest", "[ImageTest]", unsigned char, + size_t, float, double) +{ + typedef TestType eT; + + // Load cat.jpg, which has a strange aspect ratio. + arma::Mat image; + data::ImageInfo info; + REQUIRE(data::Load("cat.jpg", image, info, false) == true); + + // When we crop to match the height of the image, no resizing is needed and we + // can compare pixels directly. + const size_t inputChannels = info.Channels(); + ResizeCropImages(image, info, 1000, 1000); + + // Here we just check that the output image has the correct size. + REQUIRE(info.Height() == 1000); + REQUIRE(info.Width() == 1000); + REQUIRE(info.Channels() == inputChannels); + REQUIRE(image.n_elem == info.Height() * info.Width() * info.Channels()); +}