-
Notifications
You must be signed in to change notification settings - Fork 3.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
allow pytest --migrations to succeed #14663
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,8 @@ | |
|
||
from django.db import migrations, models, connection | ||
|
||
from ._sqlite_helper import dbawaremigrations | ||
|
||
|
||
def migrate_event_data(apps, schema_editor): | ||
# see: https://github.com/ansible/awx/issues/6010 | ||
|
@@ -24,6 +26,11 @@ def migrate_event_data(apps, schema_editor): | |
cursor.execute(f'ALTER TABLE {tblname} ALTER COLUMN id TYPE bigint USING id::bigint;') | ||
|
||
|
||
def migrate_event_data_sqlite(apps, schema_editor): | ||
# TODO: cmeyers fill this in | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is something needed here? |
||
return | ||
|
||
|
||
class FakeAlterField(migrations.AlterField): | ||
def database_forwards(self, *args): | ||
# this is intentionally left blank, because we're | ||
|
@@ -37,7 +44,7 @@ class Migration(migrations.Migration): | |
] | ||
|
||
operations = [ | ||
migrations.RunPython(migrate_event_data), | ||
dbawaremigrations.RunPython(migrate_event_data, sqlite_code=migrate_event_data_sqlite), | ||
FakeAlterField( | ||
model_name='adhoccommandevent', | ||
name='id', | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,6 +3,8 @@ | |
import awx.main.models.notifications | ||
from django.db import migrations, models | ||
|
||
from ._sqlite_helper import dbawaremigrations | ||
|
||
|
||
class Migration(migrations.Migration): | ||
dependencies = [ | ||
|
@@ -104,11 +106,12 @@ class Migration(migrations.Migration): | |
name='deleted_actor', | ||
field=models.JSONField(null=True), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_activitystream RENAME setting TO setting_old; | ||
ALTER TABLE main_activitystream ALTER COLUMN setting_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_activitystream RENAME setting TO setting_old", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok I guess this is kind of nice. |
||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='activitystream', | ||
|
@@ -121,11 +124,12 @@ class Migration(migrations.Migration): | |
name='setting', | ||
field=models.JSONField(blank=True, default=dict), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_job RENAME survey_passwords TO survey_passwords_old; | ||
ALTER TABLE main_job ALTER COLUMN survey_passwords_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_job RENAME survey_passwords TO survey_passwords_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='job', | ||
|
@@ -138,11 +142,12 @@ class Migration(migrations.Migration): | |
name='survey_passwords', | ||
field=models.JSONField(blank=True, default=dict, editable=False), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_joblaunchconfig RENAME char_prompts TO char_prompts_old; | ||
ALTER TABLE main_joblaunchconfig ALTER COLUMN char_prompts_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_joblaunchconfig RENAME char_prompts TO char_prompts_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='joblaunchconfig', | ||
|
@@ -155,11 +160,12 @@ class Migration(migrations.Migration): | |
name='char_prompts', | ||
field=models.JSONField(blank=True, default=dict), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_joblaunchconfig RENAME survey_passwords TO survey_passwords_old; | ||
ALTER TABLE main_joblaunchconfig ALTER COLUMN survey_passwords_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_joblaunchconfig RENAME survey_passwords TO survey_passwords_old;", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='joblaunchconfig', | ||
|
@@ -172,11 +178,12 @@ class Migration(migrations.Migration): | |
name='survey_passwords', | ||
field=models.JSONField(blank=True, default=dict, editable=False), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_notification RENAME body TO body_old; | ||
ALTER TABLE main_notification ALTER COLUMN body_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_notification RENAME body TO body_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='notification', | ||
|
@@ -189,11 +196,12 @@ class Migration(migrations.Migration): | |
name='body', | ||
field=models.JSONField(blank=True, default=dict), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_unifiedjob RENAME job_env TO job_env_old; | ||
ALTER TABLE main_unifiedjob ALTER COLUMN job_env_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_unifiedjob RENAME job_env TO job_env_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='unifiedjob', | ||
|
@@ -206,11 +214,12 @@ class Migration(migrations.Migration): | |
name='job_env', | ||
field=models.JSONField(blank=True, default=dict, editable=False), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_workflowjob RENAME char_prompts TO char_prompts_old; | ||
ALTER TABLE main_workflowjob ALTER COLUMN char_prompts_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_workflowjob RENAME char_prompts TO char_prompts_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='workflowjob', | ||
|
@@ -223,11 +232,12 @@ class Migration(migrations.Migration): | |
name='char_prompts', | ||
field=models.JSONField(blank=True, default=dict), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_workflowjob RENAME survey_passwords TO survey_passwords_old; | ||
ALTER TABLE main_workflowjob ALTER COLUMN survey_passwords_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_workflowjob RENAME survey_passwords TO survey_passwords_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='workflowjob', | ||
|
@@ -240,11 +250,12 @@ class Migration(migrations.Migration): | |
name='survey_passwords', | ||
field=models.JSONField(blank=True, default=dict, editable=False), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_workflowjobnode RENAME char_prompts TO char_prompts_old; | ||
ALTER TABLE main_workflowjobnode ALTER COLUMN char_prompts_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_workflowjobnode RENAME char_prompts TO char_prompts_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='workflowjobnode', | ||
|
@@ -257,11 +268,12 @@ class Migration(migrations.Migration): | |
name='char_prompts', | ||
field=models.JSONField(blank=True, default=dict), | ||
), | ||
migrations.RunSQL( | ||
dbawaremigrations.RunSQL( | ||
""" | ||
ALTER TABLE main_workflowjobnode RENAME survey_passwords TO survey_passwords_old; | ||
ALTER TABLE main_workflowjobnode ALTER COLUMN survey_passwords_old DROP NOT NULL; | ||
""", | ||
sqlite_sql="ALTER TABLE main_workflowjobnode RENAME survey_passwords TO survey_passwords_old", | ||
state_operations=[ | ||
migrations.RemoveField( | ||
model_name='workflowjobnode', | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
from django.db import migrations | ||
|
||
|
||
class RunSQL(migrations.operations.special.RunSQL): | ||
""" | ||
Bit of a hack here. Django actually wants this decision made in the router | ||
and we can pass **hints. | ||
""" | ||
|
||
def __init__(self, *args, **kwargs): | ||
if 'sqlite_sql' not in kwargs: | ||
raise ValueError("sqlite_sql parameter required") | ||
sqlite_sql = kwargs.pop('sqlite_sql') | ||
|
||
self.sqlite_sql = sqlite_sql | ||
self.sqlite_reverse_sql = kwargs.pop('sqlite_reverse_sql', None) | ||
super().__init__(*args, **kwargs) | ||
|
||
def database_forwards(self, app_label, schema_editor, from_state, to_state): | ||
if not schema_editor.connection.vendor.startswith('postgres'): | ||
self.sql = self.sqlite_sql or migrations.RunSQL.noop | ||
super().database_forwards(app_label, schema_editor, from_state, to_state) | ||
|
||
def database_backwards(self, app_label, schema_editor, from_state, to_state): | ||
if not schema_editor.connection.vendor.startswith('postgres'): | ||
self.reverse_sql = self.sqlite_reverse_sql or migrations.RunSQL.noop | ||
super().database_backwards(app_label, schema_editor, from_state, to_state) | ||
|
||
|
||
class RunPython(migrations.operations.special.RunPython): | ||
""" | ||
Bit of a hack here. Django actually wants this decision made in the router | ||
and we can pass **hints. | ||
""" | ||
|
||
def __init__(self, *args, **kwargs): | ||
if 'sqlite_code' not in kwargs: | ||
raise ValueError("sqlite_code parameter required") | ||
sqlite_code = kwargs.pop('sqlite_code') | ||
|
||
self.sqlite_code = sqlite_code | ||
self.sqlite_reverse_code = kwargs.pop('sqlite_reverse_code', None) | ||
super().__init__(*args, **kwargs) | ||
|
||
def database_forwards(self, app_label, schema_editor, from_state, to_state): | ||
if not schema_editor.connection.vendor.startswith('postgres'): | ||
self.code = self.sqlite_code or migrations.RunPython.noop | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't love this behavior. One point, months ago, I did the same thing you did here - made the migrations compatible with sqlite3. But for the life of me I can not find the branch where I worked on that. I just added a condition in the migration logic depending on the vendor, and that didn't take as many changes as you could think. Sure there are lots of logic steps we just don't want to run, but doing this I expect we're going to miss some stuff we should have, like creating default system jobs, or creating default credential types. Also the naming of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't follow. Do you not like that the user can supply I'm fine with renaming There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Talked with @AlanCoding offline. TL;DR |
||
super().database_forwards(app_label, schema_editor, from_state, to_state) | ||
|
||
def database_backwards(self, app_label, schema_editor, from_state, to_state): | ||
if not schema_editor.connection.vendor.startswith('postgres'): | ||
self.reverse_code = self.sqlite_reverse_code or migrations.RunPython.noop | ||
super().database_backwards(app_label, schema_editor, from_state, to_state) | ||
|
||
|
||
class _sqlitemigrations: | ||
RunPython = RunPython | ||
RunSQL = RunSQL | ||
|
||
|
||
dbawaremigrations = _sqlitemigrations() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Still got a TODO here. Why? Are you really going to migrate event data? I see no way you will get the hourly event tables to work in sqlite3. Of course it doesn't support range partitioning from the get-go, but even if you created superficial "partitions" as separate tables, I'm skeptical that it could be made to work, or that it would be worth it if you could do it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Oh, this is for the BigInt migration. I'm sure we won't ever want to do this in sqlite3. The next file does migration to partitions, which again, may be feasible superficially, but questionable value.