From 2c9ab71864af87d6a70732ba078383208e980666 Mon Sep 17 00:00:00 2001 From: Liu Junxiao Date: Fri, 6 Jul 2018 11:18:49 +0800 Subject: [PATCH 1/3] Force close connection and then reconnect when db connection is closed accidentally --- djcelery/snapshot.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/djcelery/snapshot.py b/djcelery/snapshot.py index 39d8af87..af65f049 100644 --- a/djcelery/snapshot.py +++ b/djcelery/snapshot.py @@ -4,6 +4,8 @@ from datetime import timedelta from django.conf import settings +from django.db import connection +from django.db.utils import InterfaceError from celery import states from celery.events.state import Task @@ -127,9 +129,16 @@ def _handle_tasks(): for i, task in enumerate(state.tasks.items()): self.handle_task(task) - for worker in state.workers.items(): - self.handle_worker(worker) - _handle_tasks() + try: + for worker in state.workers.items(): + self.handle_worker(worker) + _handle_tasks() + except InterfaceError as e: + # When connection already closed exception is raised, + # force to close connection and Django will automatically reconnect + if str(e) == 'connection already closed': + connction.close() + logger.info('Django db connection is closed and will reconnect') def on_cleanup(self): expired = (self.TaskState.objects.expire_by_states(states, expires) From 2e52c37117ce4f4027af102566707ad533068da4 Mon Sep 17 00:00:00 2001 From: Liu Junxiao Date: Fri, 6 Jul 2018 11:26:07 +0800 Subject: [PATCH 2/3] Fixed typo --- djcelery/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/djcelery/snapshot.py b/djcelery/snapshot.py index af65f049..dace9ffc 100644 --- a/djcelery/snapshot.py +++ b/djcelery/snapshot.py @@ -137,7 +137,7 @@ def _handle_tasks(): # When connection already closed exception is raised, # force to close connection and Django will automatically reconnect if str(e) == 'connection already closed': - connction.close() + connection.close() logger.info('Django db connection is closed and will reconnect') def on_cleanup(self): From 941567686cf2dcfbc681a014cf4e33a6f1056d20 Mon Sep 17 00:00:00 2001 From: Liu Junxiao Date: Fri, 6 Jul 2018 11:39:05 +0800 Subject: [PATCH 3/3] Compatible with flake8 --- djcelery/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/djcelery/snapshot.py b/djcelery/snapshot.py index dace9ffc..ac3d80f4 100644 --- a/djcelery/snapshot.py +++ b/djcelery/snapshot.py @@ -138,7 +138,9 @@ def _handle_tasks(): # force to close connection and Django will automatically reconnect if str(e) == 'connection already closed': connection.close() - logger.info('Django db connection is closed and will reconnect') + logger.info( + 'Django db connection is closed and will reconnect' + ) def on_cleanup(self): expired = (self.TaskState.objects.expire_by_states(states, expires)