diff --git a/CHANGELOG b/CHANGELOG index 9c963716..491846c0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +Unreleased + Remove db-diff dependency + 2023-10-30 Add support for Python 3.12 Add support for Django 5.0 diff --git a/src/cities_light/apps.py b/src/cities_light/apps.py index 69112766..9a66d101 100644 --- a/src/cities_light/apps.py +++ b/src/cities_light/apps.py @@ -1,6 +1,10 @@ from django.apps import AppConfig +from django.core.serializers import register_serializer class CitiesLightConfig(AppConfig): default_auto_field = 'django.db.models.AutoField' name = 'cities_light' + + def ready(self): + register_serializer('sorted_json', 'cities_light.serializers.json') diff --git a/src/cities_light/serializers/__init__.py b/src/cities_light/serializers/__init__.py new file mode 100644 index 00000000..52ab5a97 --- /dev/null +++ b/src/cities_light/serializers/__init__.py @@ -0,0 +1 @@ +"""Serializers with predictible (ordered) output.""" diff --git a/src/cities_light/serializers/base.py b/src/cities_light/serializers/base.py new file mode 100644 index 00000000..bd97b297 --- /dev/null +++ b/src/cities_light/serializers/base.py @@ -0,0 +1,83 @@ +"""Shared code for serializers.""" + +import collections +import datetime +import decimal + + +class BaseSerializerMixin(object): + """Serializer mixin for predictible and cross-db dumps.""" + + @classmethod + def recursive_dict_sort(cls, data): + """ + Return a recursive OrderedDict for a dict. + + Django's default model-to-dict logic - implemented in + django.core.serializers.python.Serializer.get_dump_object() - returns a + dict, this app registers a slightly modified version of the default + json serializer which returns OrderedDicts instead. + """ + ordered_data = collections.OrderedDict(sorted(data.items())) + + for key, value in ordered_data.items(): + if isinstance(value, dict): + ordered_data[key] = cls.recursive_dict_sort(value) + + return ordered_data + + @classmethod + def remove_microseconds(cls, data): + """ + Strip microseconds from datetimes for mysql. + + MySQL doesn't have microseconds in datetimes, so dbdiff's serializer + removes microseconds from datetimes so that fixtures are cross-database + compatible which make them usable for cross-database testing. + """ + for key, value in data['fields'].items(): + if not isinstance(value, datetime.datetime): + continue + + data['fields'][key] = datetime.datetime( + year=value.year, + month=value.month, + day=value.day, + hour=value.hour, + minute=value.minute, + second=value.second, + tzinfo=value.tzinfo + ) + + @classmethod + def normalize_decimals(cls, data): + """ + Strip trailing zeros for constitency. + + In addition, dbdiff serialization forces Decimal normalization, because + trailing zeros could happen in inconsistent ways. + """ + for key, value in data['fields'].items(): + if not isinstance(value, decimal.Decimal): + continue + + if value % 1 == 0: + data['fields'][key] = int(value) + else: + print(type(value), " => ", type(value.normalize())) + print(value, " => ", value.normalize()) + # data['fields'][key] = value + data['fields'][key] = value.normalize() + + def get_dump_object(self, obj): + """ + Actual method used by Django serializers to dump dicts. + + By overridding this method, we're able to run our various + data dump predictability methods. + """ + data = super(BaseSerializerMixin, self).get_dump_object(obj) + self.remove_microseconds(data) + self.normalize_decimals(data) + data = self.recursive_dict_sort(data) + return data diff --git a/src/cities_light/serializers/json.py b/src/cities_light/serializers/json.py new file mode 100644 index 00000000..69de31c1 --- /dev/null +++ b/src/cities_light/serializers/json.py @@ -0,0 +1,15 @@ +"""Django JSON Serializer override.""" + +from django.core.serializers import json as upstream + +from .base import BaseSerializerMixin + + +__all__ = ('Serializer', 'Deserializer') + + +class Serializer(BaseSerializerMixin, upstream.Serializer): + """Sorted dict JSON serializer.""" + + +Deserializer = upstream.Deserializer diff --git a/src/cities_light/tests/base.py b/src/cities_light/tests/base.py index eec14136..e5401332 100644 --- a/src/cities_light/tests/base.py +++ b/src/cities_light/tests/base.py @@ -1,4 +1,5 @@ """.""" +import json import os from unittest import mock @@ -6,6 +7,7 @@ from django.core import management from django.conf import settings +from io import StringIO class FixtureDir: """Helper class to construct fixture paths.""" @@ -80,3 +82,23 @@ def _patch(setting, *values): management.call_command('cities_light', progress=True, force_import_all=True, **options) + + def export_data(self, app_label=None) -> bytes: + out = StringIO() + management.call_command( + "dumpdata", + app_label or "cities_light", + format="sorted_json", + natural_foreign=True, + indent=4, + stdout=out + ) + return out.getvalue() + + def assertNoDiff(self, fixture_path, app_label=None): + """Assert that dumped data matches fixture.""" + + with open(fixture_path) as f: + self.assertListEqual( + json.loads(f.read()), json.loads(self.export_data(app_label)) + ) diff --git a/src/cities_light/tests/fixtures/update/noinsert.json b/src/cities_light/tests/fixtures/update/noinsert.json index 487d9c4e..b0cfe55a 100644 --- a/src/cities_light/tests/fixtures/update/noinsert.json +++ b/src/cities_light/tests/fixtures/update/noinsert.json @@ -29,6 +29,21 @@ "model": "cities_light.region", "pk": 1 }, +{ + "fields": { + "alternate_names": "Юргинский район", + "country": [2017370], + "display_name": "Yurginskiy Rayon, Russia", + "geoname_code": "1485714", + "geoname_id": 1485714, + "name": "Yurginskiy Rayon", + "name_ascii": "Yurginskiy Rayon", + "region": [1503900], + "slug": "yurginskiy-rayon" + }, + "model": "cities_light.subregion", + "pk": 1 +}, { "fields": { "alternate_names": "\u041a\u0435\u043c\u0435\u0440\u043e\u0432\u043e", diff --git a/src/cities_light/tests/test_fixtures.py b/src/cities_light/tests/test_fixtures.py index eef15afc..0fcecb56 100644 --- a/src/cities_light/tests/test_fixtures.py +++ b/src/cities_light/tests/test_fixtures.py @@ -1,5 +1,6 @@ """Test for cities_light_fixtures management command.""" import bz2 +import json import os from unittest import mock @@ -7,15 +8,14 @@ from django.core.management import call_command from django.core.management.base import CommandError -from dbdiff.fixture import Fixture from cities_light.settings import DATA_DIR, FIXTURES_BASE_URL from cities_light.management.commands.cities_light_fixtures import Command from cities_light.downloader import Downloader from cities_light.models import City -from .base import FixtureDir +from .base import TestImportBase, FixtureDir -class TestCitiesLigthFixtures(test.TransactionTestCase): +class TestCitiesLigthFixtures(TestImportBase): """Tests for cities_light_fixtures management command.""" def test_dump_fixtures(self): @@ -36,6 +36,26 @@ def test_dump_fixtures(self): mock_func.assert_any_call('cities_light.SubRegion', cmd.subregion_path) mock_func.assert_any_call('cities_light.City', cmd.city_path) + # def export_data(self, app_label=None) -> bytes: + # out = StringIO() + # management.call_command( + # "dumpdata", + # app_label or "cities_light", + # format="sorted_json", + # natural_foreign=True, + # indent=4, + # stdout=out + # ) + # return out.getvalue() + + def assertNoDiff(self, fixture_path, app_label=None): + """Assert that dumped data matches fixture.""" + + with open(fixture_path) as f: + self.assertListEqual( + json.loads(f.read()), json.loads(self.export_data(app_label)) + ) + def test_dump_fixture(self): """ Test dump_fixture calls dumpdata management command @@ -43,6 +63,7 @@ def test_dump_fixture(self): # Load test data destination = FixtureDir('import').get_file_path('angouleme.json') call_command('loaddata', destination) + # Dump try: fixture_path = os.path.join(os.path.dirname(__file__), "fixtures", "test_dump_fixture.json") @@ -52,7 +73,13 @@ def test_dump_fixture(self): data = bzfile.read() with open(fixture_path, mode='wb') as file: file.write(data) - Fixture(fixture_path, models=[City]).assertNoDiff() + + # with open(destination) as f2, open(fixture_path) as f: + # assert f.read() == f2.read() + # self.assertListEqual(json.loads(f.read()), json.loads(f2.read())) + # assert destination == fixture_path.read() + self.assertNoDiff(fixture_path, 'cities_light.City') + finally: if os.path.exists(fixture_path): os.remove(fixture_path) @@ -86,7 +113,7 @@ def test_load_fixtures(self): mock_func.assert_any_call( cmd.city_url, cmd.city_path, force=True) - def test_load_fixture(self): + def test_load_fixture_result(self): """Test loaded fixture matches database content.""" destination = FixtureDir('import').get_file_path('angouleme.json') with mock.patch.object(Downloader, 'download') as mock_func: @@ -94,7 +121,7 @@ def test_load_fixture(self): cmd.load_fixture(source='/abcdefg.json', destination=destination, force=True) - Fixture(destination).assertNoDiff() + self.assertNoDiff(destination) mock_func.assert_called_with(source='/abcdefg.json', destination=destination, force=True) diff --git a/src/cities_light/tests/test_import.py b/src/cities_light/tests/test_import.py index 6671f3bd..bf8af268 100644 --- a/src/cities_light/tests/test_import.py +++ b/src/cities_light/tests/test_import.py @@ -1,7 +1,9 @@ import glob import os -from dbdiff.fixture import Fixture +from django.core import management +from django.core.management.commands import dumpdata + from .base import TestImportBase, FixtureDir from ..settings import DATA_DIR @@ -20,7 +22,8 @@ def test_single_city(self): 'angouleme_city', 'angouleme_translations' ) - Fixture(fixture_dir.get_file_path('angouleme.json')).assertNoDiff() + + self.assertNoDiff(fixture_dir.get_file_path("angouleme.json")) def test_single_city_zip(self): """Load single city.""" @@ -38,7 +41,7 @@ def test_single_city_zip(self): 'angouleme_translations', file_type="zip" ) - Fixture(FixtureDir('import').get_file_path('angouleme.json')).assertNoDiff() + self.assertNoDiff(FixtureDir('import').get_file_path("angouleme.json")) def test_city_wrong_timezone(self): """Load single city with wrong timezone.""" @@ -51,7 +54,8 @@ def test_city_wrong_timezone(self): 'angouleme_city_wtz', 'angouleme_translations' ) - Fixture(fixture_dir.get_file_path('angouleme_wtz.json')).assertNoDiff() + + self.assertNoDiff(FixtureDir('import').get_file_path("angouleme_wtz.json")) from ..loading import get_cities_model city_model = get_cities_model('City') diff --git a/src/cities_light/tests/test_migrations.py b/src/cities_light/tests/test_migrations.py index 848c503c..79632bce 100644 --- a/src/cities_light/tests/test_migrations.py +++ b/src/cities_light/tests/test_migrations.py @@ -1,31 +1,27 @@ import unittest +from io import StringIO from django import test -from django.apps import apps -from django.db.migrations.autodetector import MigrationAutodetector -from django.db.migrations.loader import MigrationLoader -from django.db.migrations.questioner import ( - InteractiveMigrationQuestioner, ) -from django.db.migrations.state import ProjectState +from django.core.management import call_command +from django.test.utils import override_settings +@override_settings( + MIGRATION_MODULES={ + "cities_light": "cities_light.migrations", + }, +) class TestNoMigrationLeft(test.TestCase): - @unittest.skip("TODO: make the test pass") def test_no_migration_left(self): - loader = MigrationLoader(None, ignore_no_migrations=True) - conflicts = loader.detect_conflicts() - app_labels = ['cities_light'] - - autodetector = MigrationAutodetector( - loader.project_state(), - ProjectState.from_apps(apps), - InteractiveMigrationQuestioner(specified_apps=app_labels, dry_run=True), - ) - - changes = autodetector.changes( - graph=loader.graph, - trim_to_apps=app_labels or None, - convert_apps=app_labels or None, - ) - - assert 'cities_light' not in changes + out = StringIO() + try: + call_command( + "makemigrations", + "cities_light", + "--dry-run", + "--check", + stdout=out, + stderr=StringIO(), + ) + except SystemExit: # pragma: no cover + raise AssertionError("Pending migrations:\n" + out.getvalue()) from None diff --git a/src/cities_light/tests/test_update.py b/src/cities_light/tests/test_update.py index f0735231..e5e3fefd 100644 --- a/src/cities_light/tests/test_update.py +++ b/src/cities_light/tests/test_update.py @@ -1,7 +1,6 @@ """Tests for update records.""" import unittest -from dbdiff.fixture import Fixture from .base import TestImportBase, FixtureDir @@ -30,9 +29,9 @@ def test_update_fields(self): 'update_translations', ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('update_fields.json') - ).assertNoDiff() + ) def test_update_fields_wrong_timezone(self): """Test all fields are updated, but timezone field is wrong.""" @@ -56,9 +55,9 @@ def test_update_fields_wrong_timezone(self): 'update_translations', ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('update_fields_wtz.json') - ).assertNoDiff() + ) def test_change_country(self): """Test change country for region/city.""" @@ -82,9 +81,9 @@ def test_change_country(self): 'update_translations', ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('change_country.json') - ).assertNoDiff() + ) def test_change_region_and_country(self): """Test change region and country.""" @@ -108,9 +107,9 @@ def test_change_region_and_country(self): 'update_translations', ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('change_region_and_country.json') - ).assertNoDiff() + ) def test_keep_slugs(self): """Test --keep-slugs option.""" @@ -135,9 +134,9 @@ def test_keep_slugs(self): keep_slugs=True ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('keep_slugs.json'), - ).assertNoDiff() + ) def test_add_records(self): """Test that new records are added.""" @@ -161,9 +160,9 @@ def test_add_records(self): 'add_translations' ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('add_records.json') - ).assertNoDiff() + ) def test_noinsert(self): """Test --noinsert option.""" @@ -188,9 +187,9 @@ def test_noinsert(self): noinsert=True ) - Fixture( + self.assertNoDiff( fixture_dir.get_file_path('noinsert.json'), - ).assertNoDiff() + ) # TODO: make the test pass @unittest.skip("Obsolete records are not removed yet.") diff --git a/test_project/settings.py b/test_project/settings.py index 8dcf8506..baf199cc 100644 --- a/test_project/settings.py +++ b/test_project/settings.py @@ -181,4 +181,4 @@ LOGGING['loggers']['cities_light']['level'] = 'DEBUG' - INSTALLED_APPS += ('dbdiff',) + # INSTALLED_APPS += ('dbdiff',) diff --git a/tox.ini b/tox.ini index bdc43c61..30e752c6 100644 --- a/tox.ini +++ b/tox.ini @@ -30,8 +30,6 @@ deps = deps = # sphinx Sphinx==4.2.0 - ; django-dbdiff - git+https://github.com/yourlabs/django-dbdiff.git@master#egg=django-dbdiff [test] deps = @@ -42,8 +40,6 @@ deps = pylint pylint-django djangorestframework - ; django-dbdiff - git+https://github.com/yourlabs/django-dbdiff.git@master#egg=django-dbdiff django-ajax-selects==2.2.0 django-autoslug==1.9.9 graphene==3.3