diff --git a/CHANGELOG.md b/CHANGELOG.md index 75069333..dc9adc84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## [v1.5.1] - 2026-04-01 + +### Fixed + +- Transitive dependencies now properly include sub-projects + +## [v1.5.0] - 2026-03-26 ### Fixed diff --git a/kirc-blocking/build.gradle.kts b/kirc-blocking/build.gradle.kts index e093276b..d14925c9 100644 --- a/kirc-blocking/build.gradle.kts +++ b/kirc-blocking/build.gradle.kts @@ -3,6 +3,8 @@ plugins { kotlin("libs.publisher") } +group = "com.github.cmdjulian.kirc" + dependencies { api(project(":kirc-core")) api(project(":kirc-image")) diff --git a/kirc-blocking/src/main/kotlin/de/cmdjulian/kirc/tar/BlockingImageExtractor.kt b/kirc-blocking/src/main/kotlin/de/cmdjulian/kirc/tar/BlockingImageExtractor.kt new file mode 100644 index 00000000..a4e143a8 --- /dev/null +++ b/kirc-blocking/src/main/kotlin/de/cmdjulian/kirc/tar/BlockingImageExtractor.kt @@ -0,0 +1,14 @@ +package de.cmdjulian.kirc.tar + +import kotlinx.coroutines.runBlocking +import java.nio.file.Path + +object BlockingImageExtractor { + /** + * Parses docker container image from [path] representing a tar file into an object Representation + * + * If [isGzipped] is true, the file will be treated as gzipped tar and decompressed before parsing + */ + fun parse(path: Path, isGzipped: Boolean = false): ContainerImageMetadata = + runBlocking { ImageExtractor.parse(path, isGzipped) } +} diff --git a/kirc-core/build.gradle.kts b/kirc-core/build.gradle.kts index f393f229..b2b5dde8 100644 --- a/kirc-core/build.gradle.kts +++ b/kirc-core/build.gradle.kts @@ -4,6 +4,8 @@ plugins { kotlin("kapt") } +group = "com.github.cmdjulian.kirc" + dependencies { implementation(project(":kirc-image")) diff --git a/kirc-core/src/main/kotlin/de/cmdjulian/kirc/tar/ContainerImageMetadata.kt b/kirc-core/src/main/kotlin/de/cmdjulian/kirc/tar/ContainerImageMetadata.kt new file mode 100644 index 00000000..ba783ae8 --- /dev/null +++ b/kirc-core/src/main/kotlin/de/cmdjulian/kirc/tar/ContainerImageMetadata.kt @@ -0,0 +1,40 @@ +package de.cmdjulian.kirc.tar + +import de.cmdjulian.kirc.image.Digest +import de.cmdjulian.kirc.spec.Platform +import de.cmdjulian.kirc.spec.image.ImageConfig +import de.cmdjulian.kirc.spec.manifest.LayerReference +import de.cmdjulian.kirc.spec.manifest.ManifestList +import de.cmdjulian.kirc.spec.manifest.ManifestSingle + +/** + * Represents the metadata of an image tar archive without spilled blobs. + * + * @param index content of index.json + * @param images collected manifests specified in index with their metadata + */ +data class ContainerImageMetadata(val index: ManifestList, val images: List) { + val imageForCurrentPlatform: ContainerImageSingleMetadata? + get() = images.firstOrNull { image -> + Platform(image.config.os, image.config.architecture) == Platform.current() + } +} + +/** + * Represents a single platform image's metadata — no blob paths, just descriptors and deserialized config. + * + * @param manifest manifest of platform image + * @param digest digest of platform image manifest + * @param layers layer descriptors from the manifest (mediaType, size, digest) + * @param config deserialized image config + * @param tags human-readable tags for this image (e.g. "ubuntu:22.04"); empty if untagged. + * Resolved from Docker-format manifest.json repoTags and OCI index annotations + * (org.opencontainers.image.ref.name). + */ +data class ContainerImageSingleMetadata( + val manifest: ManifestSingle, + val digest: Digest, + val layers: List, + val config: ImageConfig, + val tags: List, +) diff --git a/kirc-image/build.gradle.kts b/kirc-image/build.gradle.kts index 60aa1e9f..db599e7c 100644 --- a/kirc-image/build.gradle.kts +++ b/kirc-image/build.gradle.kts @@ -4,6 +4,8 @@ plugins { kotlin("kapt") } +group = "com.github.cmdjulian.kirc" + tasks.jar { manifest { attributes(mapOf("Implementation-Title" to project.name, "Implementation-Version" to project.version)) diff --git a/kirc-reactive/build.gradle.kts b/kirc-reactive/build.gradle.kts index da53f6e7..ed17d7fa 100644 --- a/kirc-reactive/build.gradle.kts +++ b/kirc-reactive/build.gradle.kts @@ -3,6 +3,8 @@ plugins { kotlin("libs.publisher") } +group = "com.github.cmdjulian.kirc" + dependencies { api(project(":kirc-core")) api(project(":kirc-image")) diff --git a/kirc-suspending/build.gradle.kts b/kirc-suspending/build.gradle.kts index d465f2bf..a5205e33 100644 --- a/kirc-suspending/build.gradle.kts +++ b/kirc-suspending/build.gradle.kts @@ -4,6 +4,8 @@ plugins { kotlin("kapt") } +group = "com.github.cmdjulian.kirc" + dependencies { api(project(":kirc-core")) api(project(":kirc-image")) diff --git a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageExtractor.kt b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageExtractor.kt deleted file mode 100644 index 962023e6..00000000 --- a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageExtractor.kt +++ /dev/null @@ -1,92 +0,0 @@ -package de.cmdjulian.kirc.impl.delegate - -import com.fasterxml.jackson.databind.exc.InvalidDefinitionException -import de.cmdjulian.kirc.image.Digest -import de.cmdjulian.kirc.spec.ManifestJson -import de.cmdjulian.kirc.spec.Repositories -import de.cmdjulian.kirc.spec.image.DockerImageConfigV1 -import de.cmdjulian.kirc.spec.image.ImageConfig -import de.cmdjulian.kirc.spec.image.OciImageConfigV1 -import de.cmdjulian.kirc.spec.manifest.ManifestList -import de.cmdjulian.kirc.spec.manifest.ManifestSingle -import kotlinx.coroutines.runBlocking -import kotlinx.io.Source -import kotlinx.io.asInputStream -import kotlinx.io.asSource -import kotlinx.io.buffered -import org.apache.commons.compress.archivers.tar.TarArchiveEntry -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import java.nio.file.Path -import java.util.zip.GZIPInputStream -import kotlin.io.path.inputStream - -/** - * Helper class to extract certain parts of a docker image, e.g. Index, Manifest, Config - * - * @param path location of docker image - * @param isGzipped optionally, to signalize tar archive is gzipped, and it needs to be unwrapped during extraction - */ -class ImageExtractor(private val path: Path, private val isGzipped: Boolean = false) { - - private val source: Source - get() = path.inputStream().asSource().buffered() - private val blobPath: String = "blobs/sha256/" - - private inline fun extract(path: String): T? = source.use { source -> - val unwrapStream = if (isGzipped) GZIPInputStream(source.asInputStream()) else source.asInputStream() - - unwrapStream.use { unwrapStream -> - TarArchiveInputStream(unwrapStream).use { stream -> - generateSequence(stream::getNextEntry) - .filterNot(TarArchiveEntry::isDirectory) - .firstOrNull { entry -> entry.name == path } - ?.let { entry -> runBlocking { stream.deserializeEntry(entry) } } - } - } - } - - // --- TOP LEVEL MODELS --- - - fun index(): ManifestList? = extract("index.json") - - fun repositoriesJson(): Repositories? = extract("repositories") - - fun manifestJson(): ManifestJson? = extract("manifest.json") - - // --- BLOB LEVEL MODELS --- - - /** Get manifest by extracting from [ManifestList] */ - fun manifest(index: ManifestList): ManifestSingle? = - index.manifests.firstOrNull()?.digest?.let { digest -> extract(blobPath + digest.hash) } - - /** Get manifest by [Digest] */ - fun manifest(digest: Digest): ManifestSingle? = blobAsType(digest) - - /** - * Get config by extracting from [ManifestSingle] - * - * Deserializes based on config media type defined in manifest - */ - fun config(manifest: ManifestSingle): ImageConfig? = when (manifest.config.mediaType) { - OciImageConfigV1.MediaType -> extract(blobPath + manifest.config.digest.hash) - DockerImageConfigV1.MediaType -> extract(blobPath + manifest.config.digest.hash) - else -> error("Unknown manifest single type encountered in manifest: ${manifest.mediaType}") - } - - /** - * Get config by [Digest] - * - * Since both Docker and OCI config have the same structure, we try to parse as OCI first, - */ - fun config(digest: Digest): ImageConfig? = try { - blobAsType(digest) - } catch (_: InvalidDefinitionException) { - blobAsType(digest) - } - - /** Be careful, as this loads the content of a potentially large blob into memory */ - fun blob(digest: Digest): ByteArray? = extract(blobPath + digest.hash) - - /** Retrieve Blob with certain [Digest] as type [T] */ - private inline fun blobAsType(digest: Digest): T? = extract(blobPath + digest.hash) -} diff --git a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageUploader.kt b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageUploader.kt index 8b12fce8..b830e284 100644 --- a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageUploader.kt +++ b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/ImageUploader.kt @@ -9,19 +9,8 @@ import de.cmdjulian.kirc.image.Reference import de.cmdjulian.kirc.image.Repository import de.cmdjulian.kirc.impl.auth.ScopeType import de.cmdjulian.kirc.impl.auth.withAuthSession -import de.cmdjulian.kirc.impl.serialization.JsonMapper -import de.cmdjulian.kirc.impl.serialization.deserialize -import de.cmdjulian.kirc.spec.ManifestJson -import de.cmdjulian.kirc.spec.OciLayout -import de.cmdjulian.kirc.spec.Repositories -import de.cmdjulian.kirc.spec.UploadBlobPath -import de.cmdjulian.kirc.spec.UploadContainerImage -import de.cmdjulian.kirc.spec.UploadSingleImage -import de.cmdjulian.kirc.spec.manifest.DockerManifestListV1 -import de.cmdjulian.kirc.spec.manifest.Manifest -import de.cmdjulian.kirc.spec.manifest.ManifestList -import de.cmdjulian.kirc.spec.manifest.ManifestSingle -import de.cmdjulian.kirc.spec.manifest.OciManifestListV1 +import de.cmdjulian.kirc.tar.BlobPath +import de.cmdjulian.kirc.tar.ImageExtractor import de.cmdjulian.kirc.utils.createSafePath import de.cmdjulian.kirc.utils.toKotlinPath import io.github.oshai.kotlinlogging.KotlinLogging @@ -33,9 +22,7 @@ import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import kotlinx.io.Source import kotlinx.io.asInputStream -import kotlinx.io.buffered import kotlinx.io.files.SystemFileSystem -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import java.nio.file.Path import kotlin.io.path.pathString @@ -63,7 +50,7 @@ internal class ImageUploader(private val client: SuspendingContainerImageRegistr try { // read from tar and deserialize (can throw; ensure cleanup in finally) - val uploadContainerImage = readFromTar(tar, tempDirectory) + val uploadContainerImage = ImageExtractor.parse(tar.asInputStream(), tempDirectory) // initialize auth for upload flow client.initializeAuth(repository, ScopeType.PULL_PUSH) @@ -74,7 +61,7 @@ internal class ImageUploader(private val client: SuspendingContainerImageRegistr coroutineScope { val semaphore = Semaphore(3) - for (blob in blobs.distinctBy(UploadBlobPath::digest)) { + for (blob in blobs.distinctBy(BlobPath::digest)) { launch { semaphore.withPermit { uploadBlob(repository, blob, mode) @@ -97,7 +84,7 @@ internal class ImageUploader(private val client: SuspendingContainerImageRegistr } } - private suspend fun uploadBlob(repository: Repository, blob: UploadBlobPath, mode: UploadMode) { + private suspend fun uploadBlob(repository: Repository, blob: BlobPath, mode: UploadMode) { if (!client.existsBlob(repository, blob.digest)) { val session = client.initiateBlobUpload(repository) @@ -113,6 +100,7 @@ internal class ImageUploader(private val client: SuspendingContainerImageRegistr private fun handleError(e: Exception): Nothing { val sanitizedError = when (e) { is KircException, is RegistryException -> e + else -> KircException.UnexpectedError( "Unexpected error, could not upload image to registry: ${e.cause}", e, @@ -129,135 +117,4 @@ internal class ImageUploader(private val client: SuspendingContainerImageRegistr SystemFileSystem.delete(tempDirectory.toKotlinPath()) } } - - // PARSE INPUT TAR - - private suspend fun readFromTar(tarSource: Source, tempDirectory: Path) = tarSource.use { source -> - TarArchiveInputStream(source.asInputStream()).use { stream -> - val blobs = mutableMapOf() - var indexFile: ManifestList? = null - var repositoriesFile: Repositories? = null - var manifestJsonFile: ManifestJson? = null - var ociLayoutFile: OciLayout? = null - - generateSequence(stream::getNextEntry).forEach { entry -> - when { - entry.isDirectory -> stream.skip(entry.size) - - entry.name.startsWith("blobs/sha256/") -> { - val digest = Digest.of("sha256:" + entry.name.removePrefix("blobs/sha256/")) - blobs[digest] = stream.processBlobEntry(entry, tempDirectory) - } - - "index.json" == entry.name -> indexFile = stream.deserializeEntry(entry) - - "repositories" == entry.name -> repositoriesFile = stream.deserializeEntry(entry) - - "manifest.json" == entry.name -> manifestJsonFile = stream.deserializeEntry(entry) - - "oci-layout" == entry.name -> ociLayoutFile = stream.deserializeEntry(entry) - - else -> stream.skip(entry.size) - } - } - - when { - indexFile == null -> throw KircException.CorruptArchiveError( - "index should be present inside provided docker image", - ) - - ociLayoutFile == null -> throw KircException.CorruptArchiveError( - "'oci-layout' file should be present inside the provided docker image", - ) - } - - val (processedIndex, resolvedManifests) = resolveManifestsAndBlobs(indexFile, blobs) - UploadContainerImage( - index = processedIndex, - images = resolvedManifests, - manifest = manifestJsonFile, - repositories = repositoriesFile, - layout = ociLayoutFile, - ) - } - } - - /** - * Recursively resolves all Manifests from blobs and associates manifest with its layer blobs and config blobs. - * Removes manifest attachments from manifests in the process if their platform contains UNKNOWN. - * - * Returns the resolved images as well as the index provided, stripped from all attachments (attestations, cache, etc.). - * - * [index] - the provided [ManifestList] for which manifests should be resolved - * [blobPaths] - a mapping of blob digests to their source path - */ - private suspend fun resolveManifestsAndBlobs( - index: ManifestList, - blobPaths: Map, - ): Pair> { - val attachments = mutableListOf() - val manifestBlobs = buildList { - for (manifestEntry in index.manifests) { - val entryDigest = manifestEntry.digest - // skip manifests with unknown platform (attestations, cache, etc.), they will be filtered out - if (manifestEntry.platformIsUnknown()) { - attachments.add(entryDigest) - continue - } - // resolve manifests from index - val manifestBlobPath = blobPaths[entryDigest] ?: continue - val manifestBlob = - UploadBlobPath(entryDigest, manifestEntry.mediaType, manifestBlobPath, manifestEntry.size) - addAll(resolveManifestsRecursively(entryDigest, blobPaths, manifestBlob)) - } - } - val manifests = index.manifests.filterNot { it.digest in attachments } - val index = when (index) { - is DockerManifestListV1 -> index.copy(manifests = manifests) - is OciManifestListV1 -> index.copy(manifests = manifests) - } - return index to manifestBlobs - } - - private suspend fun resolveManifestsRecursively( - entryDigest: Digest, - blobPaths: Map, - manifestBlob: UploadBlobPath, - ) = buildList { - when (val manifest = resolveManifest(blobPaths, entryDigest)) { - null -> Unit - is ManifestList -> { - // recursively resolve manifests - val (processedManifestList, processedManifests) = resolveManifestsAndBlobs(manifest, blobPaths) - addAll(processedManifests) - // add the manifest list itself as an image too, so that it can be referenced elsewhere - add(UploadSingleImage(processedManifestList, entryDigest, listOf(manifestBlob))) - } - - is ManifestSingle -> { - // associate manifests to their layer blobs and config blob - val blobs = (manifest.layers + manifest.config).map { layer -> - val blobPath = blobPaths[layer.digest] ?: throw KircException.CorruptArchiveError( - "Could not resolve blob for layer or " + - "config with digest '${layer.digest}' during upload", - ) - UploadBlobPath(layer.digest, layer.mediaType, blobPath, layer.size) - } - add(UploadSingleImage(manifest, entryDigest, blobs + manifestBlob)) - } - } - } - - private suspend fun resolveManifest(blobs: Map, manifestDigest: Digest): Manifest? { - val blobPath = blobs[manifestDigest] ?: return null - - val manifestStream = withContext(Dispatchers.IO) { - SystemFileSystem.source(blobPath.toKotlinPath()).buffered().asInputStream() - } - return runCatching { - JsonMapper.deserialize(manifestStream) - }.getOrElse { - throw KircException.CorruptArchiveError("Could not deserialize manifest (digest=$manifestDigest)", it) - } - } } diff --git a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/ImageExtractor.kt b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/ImageExtractor.kt new file mode 100644 index 00000000..37aef2b3 --- /dev/null +++ b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/ImageExtractor.kt @@ -0,0 +1,345 @@ +package de.cmdjulian.kirc.tar + +import de.cmdjulian.kirc.exception.KircException +import de.cmdjulian.kirc.image.Digest +import de.cmdjulian.kirc.impl.serialization.JsonMapper +import de.cmdjulian.kirc.impl.serialization.deserialize +import de.cmdjulian.kirc.spec.ManifestJson +import de.cmdjulian.kirc.spec.OciLayout +import de.cmdjulian.kirc.spec.Repositories +import de.cmdjulian.kirc.spec.image.DockerImageConfigV1 +import de.cmdjulian.kirc.spec.image.ImageConfig +import de.cmdjulian.kirc.spec.image.OciImageConfigV1 +import de.cmdjulian.kirc.spec.manifest.DockerManifestListV1 +import de.cmdjulian.kirc.spec.manifest.Manifest +import de.cmdjulian.kirc.spec.manifest.ManifestList +import de.cmdjulian.kirc.spec.manifest.ManifestSingle +import de.cmdjulian.kirc.spec.manifest.OciManifestListV1 +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import java.io.InputStream +import java.nio.file.Path +import java.util.zip.GZIPInputStream +import kotlin.io.path.inputStream + +/** + * Utility object to parse docker image tar archives. + */ +object ImageExtractor { + + /** + * Performs a single sequential pass through [input], spilling blobs to [tempDirectory], then resolves and + * returns a fully-parsed upload container image. + * + * [input] - the tar input stream to read from (consumed exactly once) + * [tempDirectory] - directory where blob files are spilled during parsing + */ + internal suspend fun parse(input: InputStream, tempDirectory: Path): UploadContainerImage = input.use { stream -> + val blobs = mutableMapOf() + val scan = TarArchiveInputStream(stream).use { tar -> + tar.scanEntries { digest, entry -> blobs[digest] = tar.processBlobEntry(entry, tempDirectory) } + } + + val (processedIndex, resolvedManifests) = resolveManifestsAndBlobs(scan.index, blobs) + UploadContainerImage( + index = processedIndex, + images = resolvedManifests, + manifest = scan.manifestJson, + repositories = scan.repositories, + layout = scan.ociLayout, + ) + } + + /** + * Parses metadata from an image tar at [path] without spilling blobs to disk. + * + * Performs a single sequential pass to collect the index and record which blob entry names exist, + * then reopens [path] on demand to deserialize each manifest and config blob. + * + * [path] - path to the docker image tar on disk (optionally gzip-compressed) + * [isGzipped] - whether the tar archive is gzip-compressed + */ + suspend fun parse(path: Path, isGzipped: Boolean = false): ContainerImageMetadata { + val blobEntryNames = mutableMapOf() + val scan = openTar(path, isGzipped).use { tar -> + tar.scanEntries { digest, entry -> blobEntryNames[digest] = entry.name } + } + + // reopens tar from path each time a blob needs to be read + suspend fun blobReader(digest: Digest): InputStream? { + val entryName = blobEntryNames[digest] ?: return null + return withContext(Dispatchers.IO) { readBlobEntry(path, isGzipped, entryName) } + } + + val (processedIndex, resolvedImages) = resolveManifestsMetadata(scan.index, scan.manifestJson, ::blobReader) + return ContainerImageMetadata( + index = processedIndex, + images = resolvedImages, + ) + } + + // --- UPLOAD RESOLUTION --- + + private suspend fun resolveManifestsAndBlobs( + index: ManifestList, + blobPaths: Map, + ): Pair> { + val attachments = mutableListOf() + val manifestBlobs = buildList { + for (manifestEntry in index.manifests) { + val entryDigest = manifestEntry.digest + if (manifestEntry.platformIsUnknown()) { + attachments.add(entryDigest) + continue + } + val manifestBlobPath = blobPaths[entryDigest] ?: continue + val manifestBlob = BlobPath(entryDigest, manifestEntry.mediaType, manifestBlobPath, manifestEntry.size) + addAll(resolveManifestsRecursively(entryDigest, blobPaths, manifestBlob)) + } + } + return index.withoutAttachments(attachments) to manifestBlobs + } + + private suspend fun resolveManifestsRecursively( + entryDigest: Digest, + blobPaths: Map, + manifestBlob: BlobPath, + ) = buildList { + when (val manifest = readManifest(entryDigest, diskBlobReader(blobPaths))) { + null -> Unit + + is ManifestList -> { + val (processedManifestList, processedManifests) = resolveManifestsAndBlobs(manifest, blobPaths) + addAll(processedManifests) + add(UploadSingleImage(processedManifestList, entryDigest, listOf(manifestBlob))) + } + + is ManifestSingle -> { + val blobs = (manifest.layers + manifest.config).map { layer -> + val blobPath = blobPaths[layer.digest] ?: throw KircException.CorruptArchiveError( + "Could not resolve blob for layer or config with digest '${layer.digest}' during upload", + ) + BlobPath(layer.digest, layer.mediaType, blobPath, layer.size) + } + add(UploadSingleImage(manifest, entryDigest, blobs + manifestBlob)) + } + } + } + + // --- METADATA RESOLUTION --- + + private suspend fun resolveManifestsMetadata( + index: ManifestList, + manifestJson: ManifestJson?, + blobReader: suspend (Digest) -> InputStream?, + ): Pair> { + val attachments = mutableListOf() + val images = buildList { + for (manifestEntry in index.manifests) { + val entryDigest = manifestEntry.digest + if (manifestEntry.platformIsUnknown()) { + attachments.add(entryDigest) + continue + } + addAll(resolveManifestsMetadataRecursively(entryDigest, index, manifestJson, blobReader)) + } + } + return index.withoutAttachments(attachments) to images + } + + private suspend fun resolveManifestsMetadataRecursively( + entryDigest: Digest, + index: ManifestList, + manifestJson: ManifestJson?, + blobReader: suspend (Digest) -> InputStream?, + ): List = buildList { + when (val manifest = readManifest(entryDigest, blobReader)) { + null -> Unit + + is ManifestList -> { + val (_, images) = resolveManifestsMetadata(manifest, manifestJson, blobReader) + addAll(images) + } + + is ManifestSingle -> { + val config = readConfig(manifest, blobReader) + val tags = resolveTags(entryDigest, manifest.config.digest, index, manifestJson) + add(ContainerImageSingleMetadata(manifest, entryDigest, manifest.layers, config, tags)) + } + } + } + + // --- SHARED BLOB READING --- + + /** + * Reads and deserialises a [Manifest] blob identified by [digest] using [blobReader]. + * Returns null if [blobReader] returns null for the given digest. + */ + private suspend fun readManifest(digest: Digest, blobReader: suspend (Digest) -> InputStream?): Manifest? { + val stream = blobReader(digest) ?: return null + return runCatching { + JsonMapper.deserialize(stream) + }.getOrElse { + throw KircException.CorruptArchiveError("Could not deserialize manifest (digest=$digest)", it) + } + } + + private suspend fun readConfig( + manifest: ManifestSingle, + blobReader: suspend (Digest) -> InputStream?, + ): ImageConfig { + val configDigest = manifest.config.digest + val stream = blobReader(configDigest) ?: throw KircException.CorruptArchiveError( + "Could not find config blob with digest '$configDigest'", + ) + return runCatching { + when (manifest.config.mediaType) { + OciImageConfigV1.MediaType -> JsonMapper.deserialize(stream) + + DockerImageConfigV1.MediaType -> JsonMapper.deserialize(stream) + + else -> throw KircException.CorruptArchiveError( + "Unknown config media type '${manifest.config.mediaType}'", + ) + } + }.getOrElse { + if (it is KircException) throw it + throw KircException.CorruptArchiveError("Could not deserialize config (digest=$configDigest)", it) + } + } + + /** + * Resolves human-readable tags for the image identified by [entryDigest] and [configDigest]. + * + * Two sources are checked and merged: + * - **OCI**: `org.opencontainers.image.ref.name` annotation on the [index] entry whose digest matches [entryDigest] + * - **Docker**: `repoTags` from the [manifestJson] entry whose config path ends with [configDigest]'s hash + */ + private fun resolveTags( + entryDigest: Digest, + configDigest: Digest, + index: ManifestList, + manifestJson: ManifestJson?, + ): List { + val ociTags = index.manifests + .firstOrNull { it.digest == entryDigest } + ?.annotations + ?.get("org.opencontainers.image.ref.name") + ?.let { listOf(it) } + ?: emptyList() + + val dockerTags = manifestJson + ?.firstOrNull { entry -> entry.config.endsWith(configDigest.hash) } + ?.repoTags + ?: emptyList() + + return (ociTags + dockerTags).distinct() + } + + /** + * Returns a [blobReader][readManifest] lambda that opens blobs from already-spilled files in [blobPaths]. + */ + private fun diskBlobReader(blobPaths: Map): suspend (Digest) -> InputStream? = { digest -> + blobPaths[digest]?.let { blobPath -> + withContext(Dispatchers.IO) { blobPath.inputStream() } + } + } + + // --- TAR UTILITIES --- + + /** + * Holds the structural (non-blob) entries collected during a [scanEntries] pass. + */ + private class TarScanResult( + val index: ManifestList, + val ociLayout: OciLayout, + val manifestJson: ManifestJson?, + val repositories: Repositories?, + ) + + /** + * Scans all entries in this [TarArchiveInputStream], dispatching structural entries into a [TarScanResult] + * and delegating every blob entry (`blobs/sha256/…`) to [onBlob]. + * + * Directories and unrecognised entries are skipped automatically. + * Throws [KircException.CorruptArchiveError] if `index.json` or `oci-layout` are missing. + */ + private suspend fun TarArchiveInputStream.scanEntries( + onBlob: suspend (digest: Digest, entry: TarArchiveEntry) -> Unit, + ): TarScanResult { + var indexFile: ManifestList? = null + var repositoriesFile: Repositories? = null + var manifestJsonFile: ManifestJson? = null + var ociLayoutFile: OciLayout? = null + + generateSequence(::getNextEntry).forEach { entry -> + when { + entry.isDirectory -> skip(entry.size) + + entry.name.startsWith("blobs/sha256/") -> { + val digest = Digest.of("sha256:" + entry.name.removePrefix("blobs/sha256/")) + onBlob(digest, entry) + } + + "index.json" == entry.name -> indexFile = deserializeEntry(entry) + + "repositories" == entry.name -> repositoriesFile = deserializeEntry(entry) + + "manifest.json" == entry.name -> manifestJsonFile = deserializeEntry(entry) + + "oci-layout" == entry.name -> ociLayoutFile = deserializeEntry(entry) + + else -> skip(entry.size) + } + } + + val index = indexFile ?: throw KircException.CorruptArchiveError( + "index should be present inside provided docker image", + ) + val ociLayout = ociLayoutFile ?: throw KircException.CorruptArchiveError( + "'oci-layout' file should be present inside the provided docker image", + ) + return TarScanResult(index, ociLayout, manifestJsonFile, repositoriesFile) + } + + /** + * Returns a [ManifestList] with entries whose digest is in [attachments] removed, + * preserving the concrete subtype. + */ + private fun ManifestList.withoutAttachments(attachments: Collection): ManifestList { + val filtered = manifests.filterNot { it.digest in attachments } + return when (this) { + is DockerManifestListV1 -> copy(manifests = filtered) + is OciManifestListV1 -> copy(manifests = filtered) + } + } + + private fun openTar(path: Path, isGzipped: Boolean): TarArchiveInputStream { + val raw = path.inputStream() + val unwrapped = if (isGzipped) GZIPInputStream(raw) else raw + return TarArchiveInputStream(unwrapped) + } + + /** + * Reopens the tar at [path] and scans forward to [entryName], returning its content as an [InputStream]. + * The returned stream must be closed by the caller. + */ + private fun readBlobEntry(path: Path, isGzipped: Boolean, entryName: String): InputStream { + val tar = openTar(path, isGzipped) + generateSequence(tar::getNextEntry).forEach { entry -> + if (entry.name == entryName) { + // wrap tar so closing the returned stream also closes the tar + return object : InputStream() { + override fun read(): Int = tar.read() + override fun read(b: ByteArray, off: Int, len: Int): Int = tar.read(b, off, len) + override fun close() = tar.close() + } + } + tar.skip(entry.size) + } + tar.close() + throw KircException.CorruptArchiveError("Blob entry '$entryName' not found in tar") + } +} diff --git a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/TarArchiveStreamExtensions.kt b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/TarArchiveStreamExtensions.kt similarity index 59% rename from kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/TarArchiveStreamExtensions.kt rename to kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/TarArchiveStreamExtensions.kt index fa47cc77..fd9cd4ba 100644 --- a/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/impl/delegate/TarArchiveStreamExtensions.kt +++ b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/TarArchiveStreamExtensions.kt @@ -1,8 +1,7 @@ -package de.cmdjulian.kirc.impl.delegate +package de.cmdjulian.kirc.tar import de.cmdjulian.kirc.impl.serialization.JsonMapper import de.cmdjulian.kirc.impl.serialization.deserialize -import de.cmdjulian.kirc.utils.toKotlinPath import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.io.asSource @@ -12,22 +11,24 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import java.nio.file.Path import kotlin.io.path.pathString - -internal suspend fun TarArchiveInputStream.readEntry(entry: TarArchiveEntry): ByteArray = - withContext(Dispatchers.IO) { readNBytes(entry.size.toInt()) } +import kotlinx.io.files.Path as KotlinPath // We deserialize entries which aren't blobs. They are small enough to be loaded to memory internal suspend inline fun TarArchiveInputStream.deserializeEntry(entry: TarArchiveEntry): T = - runCatching { JsonMapper.deserialize(readEntry(entry)) } - .getOrElse { throw IllegalStateException("Failed to deserialize tar entry '${entry.name}'", it) } + runCatching { + val readEntry = withContext(Dispatchers.IO) { readNBytes(entry.size.toInt()) } + JsonMapper.deserialize(readEntry) + }.getOrElse { + throw IllegalStateException("Failed to deserialize tar entry '${entry.name}'", it) + } internal suspend fun TarArchiveInputStream.processBlobEntry(entry: TarArchiveEntry, tempDirectory: Path): Path { val blobDigest = entry.name.removePrefix("blobs/sha256/") - val tempPath = Path.of(tempDirectory.pathString, blobDigest) + val tempPath = tempDirectory.resolve(blobDigest) withContext(Dispatchers.IO) { - SystemFileSystem.sink(tempPath.toKotlinPath()).buffered().also { path -> - path.write(this@processBlobEntry.asSource(), entry.size) - path.flush() + SystemFileSystem.sink(KotlinPath(tempPath.pathString)).buffered().also { sink -> + sink.write(this@processBlobEntry.asSource(), entry.size) + sink.flush() } } return tempPath diff --git a/kirc-core/src/main/kotlin/de/cmdjulian/kirc/spec/UploadContainerImage.kt b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/UploadContainerImage.kt similarity index 72% rename from kirc-core/src/main/kotlin/de/cmdjulian/kirc/spec/UploadContainerImage.kt rename to kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/UploadContainerImage.kt index bcf0dba9..65697254 100644 --- a/kirc-core/src/main/kotlin/de/cmdjulian/kirc/spec/UploadContainerImage.kt +++ b/kirc-suspending/src/main/kotlin/de/cmdjulian/kirc/tar/UploadContainerImage.kt @@ -1,12 +1,15 @@ -package de.cmdjulian.kirc.spec +package de.cmdjulian.kirc.tar import de.cmdjulian.kirc.image.Digest +import de.cmdjulian.kirc.spec.ManifestJson +import de.cmdjulian.kirc.spec.OciLayout +import de.cmdjulian.kirc.spec.Repositories import de.cmdjulian.kirc.spec.manifest.Manifest import de.cmdjulian.kirc.spec.manifest.ManifestList import java.nio.file.Path /** - * Represents the whole content of uploaded image + * Represents the whole content of an uploaded image tar archive * * @param index content of index.json * @param images collected manifests specified in index with their blobs @@ -14,7 +17,7 @@ import java.nio.file.Path * @param repositories content of OPTIONAL repositories file * @param layout content of oci-layout file */ -data class UploadContainerImage( +internal data class UploadContainerImage( val index: ManifestList, val images: List, val manifest: ManifestJson?, @@ -25,18 +28,18 @@ data class UploadContainerImage( /** * Represents the content of a single platform image * - * > config is handled as blob upon upload + * > config is handled as blob * * @param manifest manifest of platform image * @param digest digest of platform image manifest * @param blobs layer blobs + config blob ready for upload */ -data class UploadSingleImage(val manifest: Manifest, val digest: Digest, val blobs: List) +internal data class UploadSingleImage(val manifest: Manifest, val digest: Digest, val blobs: List) -class UploadBlobPath(val digest: Digest, val mediaType: String, val path: Path, val size: Long) { +internal class BlobPath(val digest: Digest, val mediaType: String, val path: Path, val size: Long) { override fun equals(other: Any?): Boolean = when { this === other -> true - other !is UploadBlobPath -> false + other !is BlobPath -> false mediaType != other.mediaType -> false digest != other.digest -> false path != other.path -> false diff --git a/kirc-suspending/src/test/kotlin/de/cmdjulian/kirc/impl/ImageExtractorTest.kt b/kirc-suspending/src/test/kotlin/de/cmdjulian/kirc/impl/ImageExtractorTest.kt index a89c0bf7..e0683dbb 100644 --- a/kirc-suspending/src/test/kotlin/de/cmdjulian/kirc/impl/ImageExtractorTest.kt +++ b/kirc-suspending/src/test/kotlin/de/cmdjulian/kirc/impl/ImageExtractorTest.kt @@ -1,62 +1,57 @@ package de.cmdjulian.kirc.impl import de.cmdjulian.kirc.image.Digest -import de.cmdjulian.kirc.impl.delegate.ImageExtractor +import de.cmdjulian.kirc.spec.manifest.ManifestSingle +import de.cmdjulian.kirc.tar.ImageExtractor import io.kotest.matchers.collections.shouldBeSingleton import io.kotest.matchers.collections.shouldHaveSize -import io.kotest.matchers.maps.shouldHaveSize import io.kotest.matchers.nulls.shouldNotBeNull +import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import kotlin.io.path.Path internal class ImageExtractorTest { private val resource = javaClass.getResource("/hello-world.tar") ?: error("Resource not found") - private val extractor = ImageExtractor(Path(resource.path)) + private val path = Path(resource.path) @Test - fun `extract index`() { - extractor.index().shouldNotBeNull().manifests.shouldHaveSize(1) + fun `extract index`() = runTest { + ImageExtractor.parse(path).index.manifests.shouldHaveSize(1) } @Test - fun `extract repositories`() { - val result = extractor.repositoriesJson().shouldNotBeNull() - result.shouldHaveSize(1) + fun `extract tags`() = runTest { + // hello-world.tar is a Docker-format archive that also carries an OCI index annotation, + // so we expect both "latest" (OCI ref.name) and "hello-world:latest" (Docker repoTags) + ImageExtractor.parse(path).images.shouldBeSingleton { it.tags.shouldHaveSize(2) } } @Test - fun `extract manifest json`() { - val result = extractor.manifestJson().shouldNotBeNull() - result.shouldBeSingleton { - it.layers.shouldHaveSize(1) - it.layerSources.shouldHaveSize(1) - it.repoTags.shouldHaveSize(1) - } - } - - @Test - fun `extract manifest - by digest`() { + fun `extract manifest - by digest`() = runTest { val digest = Digest("sha256:26c9f8a26a5f87d187957cf2d77efc7cf4d797e7fc55eee65316a0b62ae43034") - extractor.manifest(digest).shouldNotBeNull() + val metadata = ImageExtractor.parse(path) + metadata.images.firstOrNull { it.digest == digest }.shouldNotBeNull() } @Test - fun `extract manifest - by index`() { - val index = extractor.index().shouldNotBeNull() - extractor.manifest(index).shouldNotBeNull() + fun `extract manifest - by index`() = runTest { + val metadata = ImageExtractor.parse(path) + metadata.images.shouldBeSingleton() } @Test - fun `extract config - by digest`() { + fun `extract config - by digest`() = runTest { val digest = Digest("sha256:74cc54e27dc41bb10dc4b2226072d469509f2f22f1a3ce74f4a59661a1d44602") - extractor.config(digest).shouldNotBeNull() + val metadata = ImageExtractor.parse(path) + metadata.images.firstOrNull { + (it.manifest as? ManifestSingle)?.config?.digest == digest + }?.config.shouldNotBeNull() } @Test - fun `extract config - by manifest`() { - val index = extractor.index().shouldNotBeNull() - val manifest = extractor.manifest(index).shouldNotBeNull() - extractor.config(manifest).shouldNotBeNull() + fun `extract config - by manifest`() = runTest { + val metadata = ImageExtractor.parse(path) + metadata.images.shouldBeSingleton { it.config.shouldNotBeNull() } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 485b83a5..20fbf489 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -19,7 +19,11 @@ dependencyResolutionManagement { version("coroutines", "1.10.2") library("kotlinx-io", "org.jetbrains.kotlinx", "kotlinx-io-core").version("0.8.2") library("coroutines", "org.jetbrains.kotlinx", "kotlinx-coroutines-core").versionRef("coroutines") - library("coroutines-reactor", "org.jetbrains.kotlinx", "kotlinx-coroutines-reactor").versionRef("coroutines") + library( + "coroutines-reactor", + "org.jetbrains.kotlinx", + "kotlinx-coroutines-reactor", + ).versionRef("coroutines") } create("jackson") { library("bom", "com.fasterxml.jackson:jackson-bom:2.20.1")