Add package paragraph metadata checksum to uniqueness

pulp_deb/app/migrations/0041_package_metadata_sha256.py

@@ -0,0 +1,118 @@
+# Generated by Django 5.2.12 on 2026-07-20 07:37
+
+import hashlib
+import json
+
+from django.db import migrations, models
+
+
+PACKAGE_METADATA_FIELDS = (
+    "package",
+    "source",
+    "version",
+    "architecture",
+    "architecture_variant",
+    "section",
+    "priority",
+    "origin",
+    "tag",
+    "bugs",
+    "essential",
+    "build_essential",
+    "installed_size",
+    "maintainer",
+    "original_maintainer",
+    "description",
+    "description_md5",
+    "homepage",
+    "built_using",
+    "auto_built_package",
+    "multi_arch",
+    "breaks",
+    "conflicts",
+    "depends",
+    "recommends",
+    "suggests",
+    "enhances",
+    "pre_depends",
+    "provides",
+    "replaces",
+    "custom_fields",
+)
+
+
+def _metadata_sha256(package):
+    metadata = {}
+    for field_name in PACKAGE_METADATA_FIELDS:
+        value = getattr(package, field_name)
+        if field_name == "custom_fields":
+            value = value or {}
+        elif field_name == "installed_size" and value is not None:
+            value = int(value)
+        metadata[field_name] = value
+
+    payload = json.dumps(
+        metadata,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        allow_nan=False,
+    ).encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()
+
+
+def populate_metadata_sha256(apps, schema_editor):
+    for model_name in ("Package", "InstallerPackage"):
+        model = apps.get_model("deb", model_name)
+        batch = []
+        for package in model.objects.all().iterator(chunk_size=2000):
+            package.metadata_sha256 = _metadata_sha256(package)
+            batch.append(package)
+            if len(batch) == 2000:
+                model.objects.bulk_update(batch, ["metadata_sha256"], batch_size=2000)
+                batch.clear()
+        if batch:
+            model.objects.bulk_update(batch, ["metadata_sha256"], batch_size=2000)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("deb", "0040_architecture_variants"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="installerpackage",
+            name="metadata_sha256",
+            field=models.CharField(editable=False, max_length=64, null=True),
+        ),
+        migrations.AddField(
+            model_name="package",
+            name="metadata_sha256",
+            field=models.CharField(editable=False, max_length=64, null=True),
+        ),
+        migrations.RunPython(populate_metadata_sha256, migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="installerpackage",
+            name="metadata_sha256",
+            field=models.CharField(editable=False, max_length=64),
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="metadata_sha256",
+            field=models.CharField(editable=False, max_length=64),
+        ),
+        migrations.AlterUniqueTogether(
+            name="installerpackage",
+            unique_together={
+                ("relative_path", "sha256", "metadata_sha256", "_pulp_domain")
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name="package",
+            unique_together={
+                ("relative_path", "sha256", "metadata_sha256", "_pulp_domain")
+            },
+        ),
+    ]

pulp_deb/app/modelresource.py

@@ -174,7 +174,7 @@ class PackageForeignKeyWidget(ForeignKeyWidget):
         Class that lets us specify a multi-key link to Package.
 
         Format to be used at import-row time is:
-        str(<package.relative_path>|<package.sha256>)
+        str(<package.relative_path>|<package.sha256>|<package.metadata_sha256>)
         """
 
         def render(self, value, obj=None, **kwargs):
@@ -183,7 +183,8 @@ def render(self, value, obj=None, **kwargs):
                 return ""
             pkg_relative_path = value.relative_path
             pkg_sha256 = value.sha256
-            return f"{pkg_relative_path}|{pkg_sha256}"
+            pkg_metadata_sha256 = value.metadata_sha256
+            return f"{pkg_relative_path}|{pkg_sha256}|{pkg_metadata_sha256}"
 
     release_component = fields.Field(
         column_name="release_component",
@@ -208,13 +209,20 @@ def before_import_row(self, row, **kwargs):
         super().before_import_row(row, **kwargs)
 
         rc_dist, rc_comp = row["release_component"].split("|")
-        pkg_relative_path, pkg_sha256 = row["package"].split("|")
+        package_key = row["package"].split("|")
+        pkg_relative_path, pkg_sha256 = package_key[:2]
+        pkg_metadata_sha256 = package_key[2] if len(package_key) == 3 else None
         rc = ReleaseComponent.objects.filter(
             distribution=rc_dist, component=rc_comp, pulp_domain=get_domain()
         ).first()
-        pkg = Package.objects.filter(
-            relative_path=pkg_relative_path, sha256=pkg_sha256, pulp_domain=get_domain()
-        ).first()
+        package_filter = {
+            "relative_path": pkg_relative_path,
+            "sha256": pkg_sha256,
+            "pulp_domain": get_domain(),
+        }
+        if pkg_metadata_sha256 is not None:
+            package_filter["metadata_sha256"] = pkg_metadata_sha256
+        pkg = Package.objects.filter(**package_filter).first()
         row["release_component"] = str(rc.pulp_id)
         row["package"] = str(pkg.pulp_id)
 

pulp_deb/app/models/content/content.py

@@ -70,6 +70,8 @@ class BasePackage(Content):
     relative_path = models.TextField(null=False)
     # this digest is transferred to the content as a natural_key
     sha256 = models.TextField(null=False)
+    # digest of normalized metadata emitted in the Packages paragraph
+    metadata_sha256 = models.CharField(max_length=64, null=False, editable=False)
 
     signing_keys = ArrayField(models.TextField(), default=None, null=True)
 
@@ -105,7 +107,7 @@ def filename(
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
-        unique_together = (("relative_path", "sha256", "_pulp_domain"),)
+        unique_together = (("relative_path", "sha256", "metadata_sha256", "_pulp_domain"),)
         abstract = True
 
 

pulp_deb/app/models/repository.py

@@ -283,12 +283,19 @@ def handle_duplicate_packages(new_version):
 
         # Now remove existing packages that are duplicates of any packages added to new_version
         if package_qs_added.count() and content_qs_existing.count():
-            for batch in batch_qs(package_qs_added.values(*repo_key_fields, "sha256")):
+            for batch in batch_qs(
+                package_qs_added.values(*repo_key_fields, "sha256", "metadata_sha256")
+            ):
                 find_dup_qs = models.Q()
 
                 for content_dict in batch:
                     sha256 = content_dict.pop("sha256")
-                    item_query = models.Q(**content_dict) & ~models.Q(sha256=sha256)
+                    metadata_sha256 = content_dict.pop("metadata_sha256")
+                    content_identity = models.Q(
+                        sha256=sha256,
+                        metadata_sha256=metadata_sha256,
+                    )
+                    item_query = models.Q(**content_dict) & ~content_identity
                     find_dup_qs |= item_query
 
                 package_qs_duplicates = (

pulp_deb/app/package_metadata.py

@@ -0,0 +1,71 @@
+"""Helpers for deriving immutable package metadata identity."""
+
+import hashlib
+import json
+from collections.abc import Mapping
+from typing import Any
+
+PACKAGE_METADATA_FIELDS = (
+    "package",
+    "source",
+    "version",
+    "architecture",
+    "architecture_variant",
+    "section",
+    "priority",
+    "origin",
+    "tag",
+    "bugs",
+    "essential",
+    "build_essential",
+    "installed_size",
+    "maintainer",
+    "original_maintainer",
+    "description",
+    "description_md5",
+    "homepage",
+    "built_using",
+    "auto_built_package",
+    "multi_arch",
+    "breaks",
+    "conflicts",
+    "depends",
+    "recommends",
+    "suggests",
+    "enhances",
+    "pre_depends",
+    "provides",
+    "replaces",
+    "custom_fields",
+)
+
+
+def _normalize_value(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        return {str(key): _normalize_value(value[key]) for key in sorted(value, key=str)}
+    if isinstance(value, (list, tuple)):
+        return [_normalize_value(item) for item in value]
+    return value
+
+
+def normalize_package_metadata(data: Mapping[str, Any]) -> dict[str, Any]:
+    normalized = {}
+    for field_name in PACKAGE_METADATA_FIELDS:
+        value = data.get(field_name)
+        if field_name == "custom_fields":
+            value = value or {}
+        elif field_name == "installed_size" and value is not None:
+            value = int(value)
+        normalized[field_name] = _normalize_value(value)
+    return normalized
+
+
+def calculate_package_metadata_sha256(data: Mapping[str, Any]) -> str:
+    payload = json.dumps(
+        normalize_package_metadata(data),
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        allow_nan=False,
+    ).encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()

pulp_deb/app/serializers/content_serializers.py

@@ -49,6 +49,7 @@
     SourcePackage,
     SourcePackageReleaseComponent,
 )
+from pulp_deb.app.package_metadata import calculate_package_metadata_sha256
 
 log = logging.getLogger(__name__)
 
@@ -612,6 +613,7 @@ class BasePackageMixin(Serializer):
     provides = CharField(read_only=True)
     replaces = CharField(read_only=True)
     custom_fields = DictField(child=CharField(), allow_empty=True, required=False)
+    metadata_sha256 = CharField(read_only=True)
     signing_keys = ListField(
         child=PgpKeyFingerprintField(),
         help_text=_("List of signing key fingerprints used to sign this package."),
@@ -650,6 +652,7 @@ def deferred_validate(self, data):
         package_data = from822_serializer.validated_data
         data.update(package_data)
         data["sha256"] = data["artifact"].sha256
+        data["metadata_sha256"] = calculate_package_metadata_sha256(package_data)
 
         if "relative_path" not in data:
             data["relative_path"] = self.Meta.model(**package_data).filename()
@@ -665,6 +668,7 @@ def retrieve(self, validated_data):
         content = self.Meta.model.objects.filter(
             sha256=validated_data["sha256"],
             relative_path=validated_data["relative_path"],
+            metadata_sha256=validated_data["metadata_sha256"],
             pulp_domain=get_domain_pk(),
         )
 
@@ -702,6 +706,7 @@ class Meta:
             "provides",
             "replaces",
             "signing_keys",
+            "metadata_sha256",
         )
 
 

pulp_deb/app/tasks/synchronizing.py

@@ -72,6 +72,7 @@
     SourcePackage,
     SourcePackageReleaseComponent,
 )
+from pulp_deb.app.package_metadata import calculate_package_metadata_sha256
 from pulp_deb.app.serializers import (
     DscFile822Serializer,
     InstallerPackage822Serializer,
@@ -977,10 +978,12 @@ async def _handle_package_index(
                 log.debug(_("Downloading package {}").format(package_paragraph["Package"]))
                 serializer = serializer_class.from822(data=package_paragraph)
                 serializer.is_valid(raise_exception=True)
+                package_metadata = serializer.validated_data
                 package_content_unit = package_class(
                     relative_path=package_relpath,
                     sha256=package_sha256,
-                    **serializer.validated_data,
+                    metadata_sha256=calculate_package_metadata_sha256(package_metadata),
+                    **package_metadata,
                 )
                 package_path = quote(os.path.join(self.parsed_url.path, package_relpath), safe=":/")
                 package_da = DeclarativeArtifact(

pulp_deb/tests/functional/api/test_crud_packages.py

@@ -20,8 +20,9 @@ def test_create_package(apt_package_api, deb_package_factory):
     assert package.relative_path == DEB_PACKAGE_RELPATH
 
     # Verify that only one package with this relative path exists
-    package_list = apt_package_api.list(relative_path=package.relative_path)
-    assert package_list.count == 1
+    created_package = apt_package_api.read(package.pulp_href)
+    assert created_package.relative_path == DEB_PACKAGE_RELPATH
+    assert created_package.sha256 == package.sha256
 
     # Verify that reading the package works and has the same attributes
     package = apt_package_api.read(package.pulp_href)

pulp_deb/tests/functional/api/test_download_content.py

@@ -244,6 +244,5 @@ def test_apt_by_hash(
     # Verify that all by-hash files are available
     for path, sha256 in sha256_dict.items():
         content_url = f"dists/{DEB_FIXTURE_SINGLE_DIST}/{path}/by-hash/SHA256/{sha256}"
-        assert "404" not in download_content_unit(distribution.base_path, content_url).decode(
-            "utf-8"
-        )
+        content = download_content_unit(distribution.base_path, content_url)
+        assert len(content) > 0