summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'dev-python/django/files/1.7-test_serialize.patch')
-rw-r--r--dev-python/django/files/1.7-test_serialize.patch507
1 files changed, 507 insertions, 0 deletions
diff --git a/dev-python/django/files/1.7-test_serialize.patch b/dev-python/django/files/1.7-test_serialize.patch
new file mode 100644
index 000000000000..03509db74a07
--- /dev/null
+++ b/dev-python/django/files/1.7-test_serialize.patch
@@ -0,0 +1,507 @@
+https://github.com/django/django/commit/8c12d51ea27479555e226894c50c83043211d71d
+diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
+index a116da4..6d15063 100644
+--- a/django/conf/global_settings.py
++++ b/django/conf/global_settings.py
+@@ -578,6 +578,10 @@
+ # The name of the class to use to run the test suite
+ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
+
++# Apps that don't need to be serialized at test database creation time
++# (only apps with migrations are to start with)
++TEST_NON_SERIALIZED_APPS = []
++
+ ############
+ # FIXTURES #
+ ############
+diff --git a/django/core/management/commands/flush.py b/django/core/management/commands/flush.py
+index 38196ac..29e3a8e 100644
+--- a/django/core/management/commands/flush.py
++++ b/django/core/management/commands/flush.py
+@@ -22,10 +22,9 @@ class Command(NoArgsCommand):
+ make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True,
+ help='Tells Django not to load any initial data after database synchronization.'),
+ )
+- help = ('Returns the database to the state it was in immediately after '
+- 'migrate was first executed. This means that all data will be removed '
+- 'from the database, any post-migration handlers will be '
+- 're-executed, and the initial_data fixture will be re-installed.')
++ help = ('Removes ALL DATA from the database, including data added during '
++ 'migrations. Unmigrated apps will also have their initial_data '
++ 'fixture reloaded. Does not achieve a "fresh install" state.')
+
+ def handle_noargs(self, **options):
+ database = options.get('database')
+@@ -54,7 +53,7 @@ def handle_noargs(self, **options):
+ if interactive:
+ confirm = input("""You have requested a flush of the database.
+ This will IRREVERSIBLY DESTROY all data currently in the %r database,
+-and return each table to a fresh state.
++and return each table to an empty state.
+ Are you sure you want to do this?
+
+ Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
+diff --git a/django/core/management/commands/testserver.py b/django/core/management/commands/testserver.py
+index 0409660..78885bb 100644
+--- a/django/core/management/commands/testserver.py
++++ b/django/core/management/commands/testserver.py
+@@ -27,7 +27,7 @@ def handle(self, *fixture_labels, **options):
+ addrport = options.get('addrport')
+
+ # Create a test database.
+- db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive)
++ db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)
+
+ # Import the fixture data into the test database.
+ call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})
+diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py
+index 13d4789..2934bee 100644
+--- a/django/db/backends/creation.py
++++ b/django/db/backends/creation.py
+@@ -7,6 +7,11 @@
+ from django.utils.encoding import force_bytes
+ from django.utils.functional import cached_property
+ from django.utils.six.moves import input
++from django.utils.six import StringIO
++from django.core.management.commands.dumpdata import sort_dependencies
++from django.db import router
++from django.apps import apps
++from django.core import serializers
+
+ from .utils import truncate_name
+
+@@ -332,7 +337,7 @@ def sql_destroy_indexes_for_fields(self, model, fields, style):
+ ";",
+ ]
+
+- def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
++ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False, serialize=True):
+ """
+ Creates a test database, prompting the user for confirmation if the
+ database already exists. Returns the name of the test database created.
+@@ -364,25 +369,31 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
+ settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
+ self.connection.settings_dict["NAME"] = test_database_name
+
+- # Report migrate messages at one level lower than that requested.
++ # We report migrate messages at one level lower than that requested.
+ # This ensures we don't get flooded with messages during testing
+- # (unless you really ask to be flooded)
+- call_command('migrate',
++ # (unless you really ask to be flooded).
++ call_command(
++ 'migrate',
+ verbosity=max(verbosity - 1, 0),
+ interactive=False,
+ database=self.connection.alias,
+- load_initial_data=False,
+- test_database=True)
+-
+- # We need to then do a flush to ensure that any data installed by
+- # custom SQL has been removed. The only test data should come from
+- # test fixtures, or autogenerated from post_migrate triggers.
+- # This has the side effect of loading initial data (which was
+- # intentionally skipped in the syncdb).
+- call_command('flush',
++ test_database=True,
++ )
++
++ # We then serialize the current state of the database into a string
++ # and store it on the connection. This slightly horrific process is so people
++ # who are testing on databases without transactions or who are using
++ # a TransactionTestCase still get a clean database on every test run.
++ if serialize:
++ self.connection._test_serialized_contents = self.serialize_db_to_string()
++
++ # Finally, we flush the database to clean
++ call_command(
++ 'flush',
+ verbosity=max(verbosity - 1, 0),
+ interactive=False,
+- database=self.connection.alias)
++ database=self.connection.alias
++ )
+
+ call_command('createcachetable', database=self.connection.alias)
+
+@@ -391,6 +402,44 @@ def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
+
+ return test_database_name
+
++ def serialize_db_to_string(self):
++ """
++ Serializes all data in the database into a JSON string.
++ Designed only for test runner usage; will not handle large
++ amounts of data.
++ """
++ # Build list of all apps to serialize
++ from django.db.migrations.loader import MigrationLoader
++ loader = MigrationLoader(self.connection)
++ app_list = []
++ for app_config in apps.get_app_configs():
++ if (
++ app_config.models_module is not None and
++ app_config.label in loader.migrated_apps and
++ app_config.name not in settings.TEST_NON_SERIALIZED_APPS
++ ):
++ app_list.append((app_config, None))
++ # Make a function to iteratively return every object
++ def get_objects():
++ for model in sort_dependencies(app_list):
++ if not model._meta.proxy and router.allow_migrate(self.connection.alias, model):
++ queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
++ for obj in queryset.iterator():
++ yield obj
++ # Serialise to a string
++ out = StringIO()
++ serializers.serialize("json", get_objects(), indent=None, stream=out)
++ return out.getvalue()
++
++ def deserialize_db_from_string(self, data):
++ """
++ Reloads the database with data from a string generated by
++ the serialize_db_to_string method.
++ """
++ data = StringIO(data)
++ for obj in serializers.deserialize("json", data, using=self.connection.alias):
++ obj.save()
++
+ def _get_test_db_name(self):
+ """
+ Internal implementation - returns the name of the test DB that will be
+diff --git a/django/test/runner.py b/django/test/runner.py
+index 226238a..f4d0995 100644
+--- a/django/test/runner.py
++++ b/django/test/runner.py
+@@ -298,7 +298,11 @@ def setup_databases(verbosity, interactive, keepdb=False, **kwargs):
+ connection = connections[alias]
+ if test_db_name is None:
+ test_db_name = connection.creation.create_test_db(
+- verbosity, autoclobber=not interactive, keepdb=keepdb)
++ verbosity,
++ autoclobber=not interactive,
++ keepdb=keepdb,
++ serialize=connection.settings_dict.get("TEST_SERIALIZE", True),
++ )
+ destroy = True
+ else:
+ connection.settings_dict['NAME'] = test_db_name
+diff --git a/django/test/testcases.py b/django/test/testcases.py
+index 53ea93f..86d2b15 100644
+--- a/django/test/testcases.py
++++ b/django/test/testcases.py
+@@ -753,6 +753,12 @@ class TransactionTestCase(SimpleTestCase):
+ # Subclasses can define fixtures which will be automatically installed.
+ fixtures = None
+
++ # If transactions aren't available, Django will serialize the database
++ # contents into a fixture during setup and flush and reload them
++ # during teardown (as flush does not restore data from migrations).
++ # This can be slow; this flag allows enabling on a per-case basis.
++ serialized_rollback = False
++
+ def _pre_setup(self):
+ """Performs any pre-test setup. This includes:
+
+@@ -808,6 +814,17 @@ def _fixture_setup(self):
+ if self.reset_sequences:
+ self._reset_sequences(db_name)
+
++ # If we need to provide replica initial data from migrated apps,
++ # then do so.
++ if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
++ if self.available_apps is not None:
++ apps.unset_available_apps()
++ connections[db_name].creation.deserialize_db_from_string(
++ connections[db_name]._test_serialized_contents
++ )
++ if self.available_apps is not None:
++ apps.set_available_apps(self.available_apps)
++
+ if self.fixtures:
+ # We have to use this slightly awkward syntax due to the fact
+ # that we're using *args and **kwargs together.
+@@ -844,12 +861,14 @@ def _fixture_teardown(self):
+ # Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
+ # when flushing only a subset of the apps
+ for db_name in self._databases_names(include_mirrors=False):
++ # Flush the database
+ call_command('flush', verbosity=0, interactive=False,
+ database=db_name, skip_checks=True,
+ reset_sequences=False,
+ allow_cascade=self.available_apps is not None,
+ inhibit_post_migrate=self.available_apps is not None)
+
++
+ def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
+ items = six.moves.map(transform, qs)
+ if not ordered:
+diff --git a/docs/ref/migration-operations.txt b/docs/ref/migration-operations.txt
+index 40ef384..95c9a65 100644
+--- a/docs/ref/migration-operations.txt
++++ b/docs/ref/migration-operations.txt
+@@ -199,8 +199,9 @@ model::
+ # We get the model from the versioned app registry;
+ # if we directly import it, it'll be the wrong version
+ Country = apps.get_model("myapp", "Country")
+- Country.objects.create(name="USA", code="us")
+- Country.objects.create(name="France", code="fr")
++ db_alias = schema_editor.connection.alias
++ Country.objects.create(name="USA", code="us", using=db_alias)
++ Country.objects.create(name="France", code="fr", using=db_alias)
+
+ class Migration(migrations.Migration):
+
+@@ -236,6 +237,14 @@ Oracle). This should be safe, but may cause a crash if you attempt to use
+ the ``schema_editor`` provided on these backends; in this case, please
+ set ``atomic=False``.
+
++.. warning::
++
++ RunPython does not magically alter the connection of the models for you;
++ any model methods you call will go to the default database unless you
++ give them the current database alias (available from
++ ``schema_editor.connection.alias``, where ``schema_editor`` is the second
++ argument to your function).
++
+ SeparateDatabaseAndState
+ ------------------------
+
+diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt
+index 683d0f4..64f2d6e 100644
+--- a/docs/ref/settings.txt
++++ b/docs/ref/settings.txt
+@@ -2078,6 +2078,24 @@ Default: ``'django.test.runner.DiscoverRunner'``
+ The name of the class to use for starting the test suite. See
+ :ref:`other-testing-frameworks`.
+
++.. setting:: TEST_NON_SERIALIZED_APPS
++
++TEST_NON_SERIALIZED_APPS
++------------------------
++
++Default: ``[]``
++
++In order to restore the database state between tests for TransactionTestCases
++and database backends without transactions, Django will :ref:`serialize the
++contents of all apps with migrations <test-case-serialized-rollback>` when it
++starts the test run so it can then reload from that copy before tests that
++need it.
++
++This slows down the startup time of the test runner; if you have apps that
++you know don't need this feature, you can add their full names in here (e.g.
++``django.contrib.contenttypes``) to exclude them from this serialization
++process.
++
+ .. setting:: THOUSAND_SEPARATOR
+
+ THOUSAND_SEPARATOR
+diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt
+index e1bc3e5..cbef6b4 100644
+--- a/docs/releases/1.7.txt
++++ b/docs/releases/1.7.txt
+@@ -63,6 +63,10 @@ but a few of the key features are:
+ * ``initial_data`` fixtures are no longer loaded for apps with migrations; if
+ you want to load initial data for an app, we suggest you do it in a migration.
+
++* Test rollback behaviour is different for apps with migrations; in particular,
++ Django will no longer emulate rollbacks on non-transactional databases or
++ inside ``TransactionTestCase`` :ref:`unless specifically asked <test-case-serialized-rollback>`.
++
+ App-loading refactor
+ ~~~~~~~~~~~~~~~~~~~~
+
+diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt
+index 13041c1..7c81240 100644
+--- a/docs/topics/testing/advanced.txt
++++ b/docs/topics/testing/advanced.txt
+@@ -485,7 +485,7 @@ django.db.connection.creation
+ The creation module of the database backend also provides some utilities that
+ can be useful during testing.
+
+-.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False])
++.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False, serialize=True])
+
+ Creates a new test database and runs ``migrate`` against it.
+
+@@ -507,6 +507,12 @@ can be useful during testing.
+ a new database will be created, prompting the user to remove
+ the existing one, if present.
+
++ ``serialize`` determines if Django serializes the database into an
++ in-memory JSON string before running tests (used to restore the database
++ state between tests if you don't have transactions). You can set this to
++ False to significantly speed up creation time if you know you don't need
++ data persistance outside of test fixtures.
++
+ Returns the name of the test database that it created.
+
+ ``create_test_db()`` has the side effect of modifying the value of
+diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt
+index 2d9808f..026bbd8 100644
+--- a/docs/topics/testing/overview.txt
++++ b/docs/topics/testing/overview.txt
+@@ -234,6 +234,33 @@ the Django test runner reorders tests in the following way:
+ database by a given :class:`~django.test.TransactionTestCase` test, they
+ must be updated to be able to run independently.
+
++.. _test-case-serialized-rollback:
++
++Rollback emulation
++------------------
++
++Any initial data loaded in migrations will only be available in ``TestCase``
++tests and not in ``TransactionTestCase`` tests, and additionally only on
++backends where transactions are supported (the most important exception being
++MyISAM).
++
++Django can re-load that data for you on a per-testcase basis by
++setting the ``serialized_rollback`` option to ``True`` in the body of the
++``TestCase`` or ``TransactionTestCase``, but note that this will slow down
++that test suite by approximately 3x.
++
++Third-party apps or those developing against MyISAM will need to set this;
++in general, however, you should be developing your own projects against a
++transactional database and be using ``TestCase`` for most tests, and thus
++not need this setting.
++
++The initial serialization is usually very quick, but if you wish to exclude
++some apps from this process (and speed up test runs slightly), you may add
++those apps to :setting:`TEST_NON_SERIALIZED_APPS`.
++
++Apps without migrations are not affected; ``initial_data`` fixtures are
++reloaded as usual.
++
+ Other test conditions
+ ---------------------
+
+@@ -249,6 +276,7 @@ used. This behavior `may change`_ in the future.
+
+ .. _may change: https://code.djangoproject.com/ticket/11505
+
++
+ Understanding the test output
+ -----------------------------
+
+diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt
+index c41e35d..1d55b2e 100644
+--- a/docs/topics/testing/tools.txt
++++ b/docs/topics/testing/tools.txt
+@@ -600,9 +600,17 @@ to test the effects of commit and rollback:
+ guarantees that the rollback at the end of the test restores the database to
+ its initial state.
+
+- When running on a database that does not support rollback (e.g. MySQL with the
+- MyISAM storage engine), ``TestCase`` falls back to initializing the database
+- by truncating tables and reloading initial data.
++.. warning::
++
++ ``TestCase`` running on a database that does not support rollback (e.g. MySQL with the
++ MyISAM storage engine), and all instances of ``TransactionTestCase``, will
++ roll back at the end of the test by deleting all data from the test database
++ and reloading initial data for apps without migrations.
++
++ Apps with migrations :ref:`will not see their data reloaded <test-case-serialized-rollback>`;
++ if you need this functionality (for example, third-party apps should enable
++ this) you can set ``serialized_rollback = True`` inside the
++ ``TestCase`` body.
+
+ .. warning::
+
+diff --git a/tests/migration_test_data_persistence/__init__.py b/tests/migration_test_data_persistence/__init__.py
+new file mode 100644
+index 0000000..e69de29
+diff --git a/tests/migration_test_data_persistence/migrations/0001_initial.py b/tests/migration_test_data_persistence/migrations/0001_initial.py
+new file mode 100644
+index 0000000..0b13e8b
+--- /dev/null
++++ b/tests/migration_test_data_persistence/migrations/0001_initial.py
+@@ -0,0 +1,34 @@
++# -*- coding: utf-8 -*-
++from __future__ import unicode_literals
++
++from django.db import models, migrations
++
++
++def add_book(apps, schema_editor):
++ apps.get_model("migration_test_data_persistence", "Book").objects.using(
++ schema_editor.connection.alias,
++ ).create(
++ title="I Love Django",
++ )
++
++
++class Migration(migrations.Migration):
++
++ dependencies = [
++ ]
++
++ operations = [
++ migrations.CreateModel(
++ name='Book',
++ fields=[
++ ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
++ ('title', models.CharField(max_length=100)),
++ ],
++ options={
++ },
++ bases=(models.Model,),
++ ),
++ migrations.RunPython(
++ add_book,
++ ),
++ ]
+diff --git a/tests/migration_test_data_persistence/migrations/__init__.py b/tests/migration_test_data_persistence/migrations/__init__.py
+new file mode 100644
+index 0000000..e69de29
+diff --git a/tests/migration_test_data_persistence/models.py b/tests/migration_test_data_persistence/models.py
+new file mode 100644
+index 0000000..1b0b795
+--- /dev/null
++++ b/tests/migration_test_data_persistence/models.py
+@@ -0,0 +1,5 @@
++from django.db import models
++
++
++class Book(models.Model):
++ title = models.CharField(max_length=100)
+diff --git a/tests/migration_test_data_persistence/tests.py b/tests/migration_test_data_persistence/tests.py
+new file mode 100644
+index 0000000..1b89c17
+--- /dev/null
++++ b/tests/migration_test_data_persistence/tests.py
+@@ -0,0 +1,33 @@
++from django.test import TransactionTestCase
++from .models import Book
++
++
++class MigrationDataPersistenceTestCase(TransactionTestCase):
++ """
++ Tests that data loaded in migrations is available if we set
++ serialized_rollback = True.
++ """
++
++ available_apps = ["migration_test_data_persistence"]
++ serialized_rollback = True
++
++ def test_persistence(self):
++ self.assertEqual(
++ Book.objects.count(),
++ 1,
++ )
++
++
++class MigrationDataNoPersistenceTestCase(TransactionTestCase):
++ """
++ Tests the failure case
++ """
++
++ available_apps = ["migration_test_data_persistence"]
++ serialized_rollback = False
++
++ def test_no_persistence(self):
++ self.assertEqual(
++ Book.objects.count(),
++ 0,
++ )
+