Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions pulp_deb/app/metadata.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from django.conf import settings


def normalize_metadata_field_names(field_names):
return {field_name.lower() for field_name in (field_names or []) if field_name}


def excluded_package_metadata_fields(*field_name_lists):
"""
Return globally and locally configured excluded package metadata fields.

Field matching is case insensitive. Local lists are additive with the global
``EXCLUDED_PACKAGE_METADATA_FIELDS`` setting.
"""
excluded = normalize_metadata_field_names(
getattr(settings, "EXCLUDED_PACKAGE_METADATA_FIELDS", [])
)
for field_names in field_name_lists:
excluded.update(normalize_metadata_field_names(field_names))
return excluded
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Generated by Django 5.2.12 on 2026-07-07 07:48

import django.contrib.postgres.fields
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("deb", "0040_architecture_variants"),
]

operations = [
migrations.AddField(
model_name="aptpublication",
name="excluded_package_metadata_fields",
field=django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), default=list, size=None
),
),
migrations.AddField(
model_name="aptremote",
name="excluded_package_metadata_fields",
field=django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), default=list, size=None
),
),
migrations.AddField(
model_name="aptrepository",
name="excluded_package_metadata_fields",
field=django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), default=list, size=None
),
),
]
2 changes: 2 additions & 0 deletions pulp_deb/app/models/publication.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from contextlib import suppress
from datetime import timedelta

from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.utils import timezone
from django_lifecycle import AFTER_CREATE, AFTER_UPDATE, hook
Expand Down Expand Up @@ -72,6 +73,7 @@ class AptPublication(Publication, AutoAddObjPermsMixin):
simple = models.BooleanField(default=False)
structured = models.BooleanField(default=True)
layout = models.TextField(choices=LAYOUT_CHOICES, default=LAYOUT_TYPES.NESTED_ALPHABETICALLY)
excluded_package_metadata_fields = ArrayField(models.TextField(), default=list)
signing_service = models.ForeignKey(
AptReleaseSigningService, on_delete=models.PROTECT, null=True
)
Expand Down
2 changes: 2 additions & 0 deletions pulp_deb/app/models/remote.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from django.contrib.postgres.fields import ArrayField
from django.db import models

from pulpcore.plugin.models import AutoAddObjPermsMixin, Remote
Expand All @@ -18,6 +19,7 @@ class AptRemote(Remote, AutoAddObjPermsMixin):
sync_installer = models.BooleanField(default=False)
gpgkey = models.TextField(null=True)
ignore_missing_package_indices = models.BooleanField(default=False)
excluded_package_metadata_fields = ArrayField(models.TextField(), default=list)

class Meta:
default_related_name = "%(app_label)s_%(model_name)s"
Expand Down
3 changes: 3 additions & 0 deletions pulp_deb/app/models/repository.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from gettext import gettext as _

from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.utils.functional import cached_property

Expand Down Expand Up @@ -65,6 +66,7 @@ class AptRepository(Repository, AutoAddObjPermsMixin):
]

publish_upstream_release_fields = models.BooleanField(default=True)
excluded_package_metadata_fields = ArrayField(models.TextField(), default=list)

signing_service = models.ForeignKey(
AptReleaseSigningService, on_delete=models.PROTECT, null=True
Expand Down Expand Up @@ -101,6 +103,7 @@ def on_new_version(self, version):
simple=False,
structured=True,
signing_service_pk=getattr(self.signing_service, "pk", None),
excluded_package_metadata_fields=self.excluded_package_metadata_fields,
)

class Meta:
Expand Down
18 changes: 16 additions & 2 deletions pulp_deb/app/serializers/content_serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
PACKAGE_UPLOAD_DEFAULT_COMPONENT,
PACKAGE_UPLOAD_DEFAULT_DISTRIBUTION,
)
from pulp_deb.app.metadata import (
excluded_package_metadata_fields as get_excluded_package_metadata_fields,
)
from pulp_deb.app.models import (
BOOL_CHOICES,
BasePackage,
Expand Down Expand Up @@ -392,13 +395,16 @@ def from822(cls, data, **kwargs):
Translate deb822.Package to a dictionary for class instatiation.
"""
skip = ["Filename", "MD5sum", "Size", "SHA1", "SHA256", "SHA512"]
excluded_fields = get_excluded_package_metadata_fields(
kwargs.pop("excluded_package_metadata_fields", None)
)
package_fields = {}
custom_fields = {}
for k, v in data.items():
if k in cls.TRANSLATION_DICT_INV:
key = cls.TRANSLATION_DICT_INV[k]
package_fields[key] = v
elif k not in skip:
elif k not in skip and k.lower() not in excluded_fields:
# also save the fields not in TRANSLATION_DICT
custom_fields[k] = v

Expand Down Expand Up @@ -486,6 +492,7 @@ def to822(
remote_artifact_dict=None,
layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
basename_override=None,
excluded_package_metadata_fields=None,
):
"""Create deb822.Package object from model."""
ret = deb822.Packages()
Expand All @@ -497,7 +504,14 @@ def to822(

custom_fields = self.data.get("custom_fields")
if custom_fields:
ret.update(custom_fields)
excluded_fields = get_excluded_package_metadata_fields(excluded_package_metadata_fields)
ret.update(
{
field_name: value
for field_name, value in custom_fields.items()
if field_name.lower() not in excluded_fields
}
)

artifact = None
if artifact_dict and self.instance.sha256 in artifact_dict:
Expand Down
13 changes: 12 additions & 1 deletion pulp_deb/app/serializers/publication_serializers.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from rest_framework import serializers
from rest_framework.serializers import BooleanField, ValidationError
from rest_framework.serializers import BooleanField, CharField, ListField, ValidationError

from pulpcore.plugin.models import Publication
from pulpcore.plugin.serializers import (
Expand Down Expand Up @@ -54,6 +54,16 @@ class AptPublicationSerializer(PublicationSerializer):
view_name="signing-services-detail",
required=False,
)
excluded_package_metadata_fields = ListField(
child=CharField(),
help_text=(
"Package metadata field names to omit from generated Packages indices. "
"This only affects custom package metadata fields during structured publication. "
"Core package metadata fields are never filtered. Matching is case-insensitive. "
"Example: ['Phased-Update-Percentage']."
),
required=False,
)
publish_legacy_release_files = BooleanField(
help_text="Whether or not to publish Legacy per-component-and-architecture Release files.",
default=False,
Expand All @@ -76,6 +86,7 @@ class Meta:
"signing_service",
"publish_upstream_release_fields",
"publish_legacy_release_files",
"excluded_package_metadata_fields",
"layout",
)
model = AptPublication
Expand Down
20 changes: 19 additions & 1 deletion pulp_deb/app/serializers/remote_serializers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import re

from rest_framework.serializers import BooleanField, CharField, ChoiceField, ValidationError
from rest_framework.serializers import (
BooleanField,
CharField,
ChoiceField,
ListField,
ValidationError,
)

from pulpcore.plugin.models import Remote
from pulpcore.plugin.serializers import RemoteSerializer
Expand Down Expand Up @@ -68,6 +74,17 @@ class AptRemoteSerializer(RemoteSerializer):
required=False,
)

excluded_package_metadata_fields = ListField(
child=CharField(),
help_text=(
"Package metadata field names to drop while syncing package indices. "
"Only custom package metadata fields are affected. Core fields such as Package, "
"Version, Architecture, Filename, checksums, and Size are never filtered. "
"Matching is case-insensitive. Example: ['Phased-Update-Percentage']."
),
required=False,
)

policy = ChoiceField(
help_text="The policy to use when downloading content. The possible values include: "
"'immediate', 'on_demand', and 'streamed'. 'immediate' is the default.",
Expand All @@ -85,6 +102,7 @@ class Meta:
"sync_installer",
"gpgkey",
"ignore_missing_package_indices",
"excluded_package_metadata_fields",
)
model = AptRemote

Expand Down
14 changes: 14 additions & 0 deletions pulp_deb/app/serializers/repository_serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,19 @@ class AptRepositorySerializer(RepositorySerializer):
required=False,
allow_null=True,
)

excluded_package_metadata_fields = serializers.ListField(
child=serializers.CharField(),
help_text=_(
"Default package metadata field names to omit from generated Packages indices. "
"This repository level setting is used when a publication does not specify its own "
"excluded_package_metadata_fields. Only custom package metadata fields are affected. "
"Core fields are never filtered. Matching is case-insensitive. "
"Example: ['Phased-Update-Percentage']."
),
required=False,
)

package_signing_fingerprint = PgpKeyFingerprintField(
help_text=_(
"The pubkey fingerprint to be passed to the package signing service. "
Expand All @@ -138,6 +151,7 @@ class Meta:
"package_signing_fingerprint_release_overrides",
"package_signing_service",
"package_signing_fingerprint",
"excluded_package_metadata_fields",
)
model = AptRepository

Expand Down
1 change: 1 addition & 0 deletions pulp_deb/app/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@
STRUCTURED_EMPTY_REPO_COMPONENT = "empty"
STRUCTURED_EMPTY_REPO_ARCHITECTURES = ["all"]
MAX_PACKAGE_SIGNING_WORKERS = 5
EXCLUDED_PACKAGE_METADATA_FIELDS = []
15 changes: 15 additions & 0 deletions pulp_deb/app/tasks/publishing.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
NO_MD5_WARNING_MESSAGE,
NULL_VALUE,
)
from pulp_deb.app.metadata import (
excluded_package_metadata_fields as normalize_package_metadata_fields,
)
from pulp_deb.app.models import (
AptPublication,
AptReleaseSigningService,
Expand Down Expand Up @@ -85,6 +88,7 @@ def publish(
publish_upstream_release_fields=None,
layout=LAYOUT_TYPES.NESTED_ALPHABETICALLY,
publish_legacy_release_files=False,
excluded_package_metadata_fields=None,
):
"""
Use provided publisher to create a Publication based on a RepositoryVersion.
Expand All @@ -97,6 +101,8 @@ def publish(
signing_service_pk (str): Use this SigningService to sign the Release files.
layout (str): The layout determines the form the package urls take.
publish_legacy_release_files (bool): publish legacy per architecture release files
excluded_package_metadata_fields (list): Custom package metadata fields to omit from
generated package indices.

"""

Expand Down Expand Up @@ -129,6 +135,11 @@ def publish(
publication.publish_legacy_release_files = publish_legacy_release_files
publication.layout = layout
repository = AptRepository.objects.get(pk=repo_version.repository.pk)
publication.excluded_package_metadata_fields = (
excluded_package_metadata_fields
if excluded_package_metadata_fields is not None
else repository.excluded_package_metadata_fields
)

if simple:
release = Release(
Expand Down Expand Up @@ -413,6 +424,9 @@ def add_packages(self, package_pairs, artifact_dict, remote_artifact_dict):
package.pk: list(package.contentartifact_set.all()) for package in packages
}
layout = self.parent.publication.layout
excluded_fields = normalize_package_metadata_fields(
self.parent.publication.excluded_package_metadata_fields
)
seen_published_artifacts = set()
seen_package_index_entries = set()

Expand Down Expand Up @@ -468,6 +482,7 @@ def add_packages(self, package_pairs, artifact_dict, remote_artifact_dict):
remote_artifact_dict,
layout=layout,
basename_override=upstream_basename,
excluded_package_metadata_fields=excluded_fields,
).dump(self.package_index_files[metadata_arch][0])
except KeyError:
log.warning(
Expand Down
5 changes: 4 additions & 1 deletion pulp_deb/app/tasks/synchronizing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1030,7 +1030,10 @@ async def _handle_package_index(
package_class = InstallerPackage
serializer_class = InstallerPackage822Serializer
log.debug(_("Downloading package {}").format(package_paragraph["Package"]))
serializer = serializer_class.from822(data=package_paragraph)
serializer = serializer_class.from822(
data=package_paragraph,
excluded_package_metadata_fields=(self.remote.excluded_package_metadata_fields),
)
serializer.is_valid(raise_exception=True)
package_content_unit = package_class(
relative_path=package_relpath,
Expand Down
4 changes: 4 additions & 0 deletions pulp_deb/app/viewsets/publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,9 @@ def create(self, request):
)
publish_legacy_release_files = serializer.validated_data.get("publish_legacy_release_files")
layout = serializer.validated_data.get("layout")
excluded_package_metadata_fields = serializer.validated_data.get(
"excluded_package_metadata_fields"
)

kwargs = {
"repository_version_pk": repository_version.pk,
Expand All @@ -227,6 +230,7 @@ def create(self, request):
"publish_upstream_release_fields": publish_upstream_release_fields,
"publish_legacy_release_files": publish_legacy_release_files,
"layout": layout,
"excluded_package_metadata_fields": excluded_package_metadata_fields,
}
if checkpoint:
kwargs["checkpoint"] = True
Expand Down
Loading