Creating a restart function for storage classes to fix broken or lost…

… database connections

cnxauthoring/storage/main.py

@@ -40,3 +40,7 @@ def search(self, **kwargs):
         """Retrieve any ``Document`` objects from storage that matches the
         search terms."""
         raise NotImplementedError()
+
+    def restart(self):
+        """Restart the storage interface """
+        raise NotImplementedError()

cnxauthoring/storage/memory.py

@@ -82,7 +82,10 @@ def persist(self):
     def abort(self):
         """Persist/commit the changes."""
         pass
-
+
+    def restart(self):
+        """Restart the interface"""
+        pass
 
     def search(self, limits, type_=Document, submitter_id=None):
         """Retrieve any ``Document`` objects from storage that matches the

cnxauthoring/storage/postgresql.py

@@ -61,6 +61,7 @@ class PostgresqlStorage(BaseStorage):
     def __init__(self, db_connection_string=None):
         #initialize db
         self.conn = psycopg2.connect(db_connection_string)
+        self.db_connection_string = db_connection_string
 
     def get(self, type_=Document, **kwargs):
         """Retrieve ``Document`` objects from storage."""
@@ -366,3 +367,7 @@ def search(self, limits, type_=Document, submitter_id=None):
             for item in res:
                 yield self._reassemble_model_from_document_entry(**item)
         raise StopIteration
+
+    def restart(self):
+        """Restart the interface"""
+        self.conn = psycopg2.connect(self.db_connection_string)

cnxauthoring/tests/test_functional.py

@@ -2799,6 +2799,33 @@ def test_user_contents_hide_documents_inside_binders(self):
                 },
             })
 
+    def test_db_restart(self):
+        '''
+        Test to see if the database resets itself after a broken
+        connection
+        '''
+        import psycopg2
+        from ..storage import storage
+
+        self.addCleanup(setattr, storage, 'conn',
+                        psycopg2.connect(storage.conn.dsn))
+
+        storage.conn.close()
+
+        response = self.testapp.post_json(
+            '/users/contents',
+            {'title': u'My document タイトル'},
+            status=503,
+            expect_errors=True)
+        self.assertEqual(response.status, '503 Service Unavailable')
+
+        response = self.testapp.post_json(
+            '/users/contents',
+            {'title': u'My document タイトル'},
+            status=201,
+            expect_errors=True)
+        self.assertEqual(response.status, '201 Created')
+
     def test_service_unavailable_response(self):
         '''
         Test service unavailable response when a request is made during a
@@ -2819,25 +2846,33 @@ def test_service_unavailable_response(self):
             expect_errors=True)
         self.assertEqual(response.status, '503 Service Unavailable')
 
+        storage.conn.close()
+
         response = self.testapp.get(
             '/resources/1234abcde',
             status=503,
             expect_errors=True)
         self.assertEqual(response.status, '503 Service Unavailable')
 
+        storage.conn.close()
+
         response = self.testapp.put_json(
             '/contents/[email protected]',
             {},
             status=503,
             expect_errors=True)
         self.assertEqual(response.status, '503 Service Unavailable')
 
+        storage.conn.close()
+
         response = self.testapp.get(
             '/search',
             status=503,
             expect_errors=True)
         self.assertEqual(response.status, '503 Service Unavailable')
 
+        storage.conn.close()
+
         response = self.testapp.delete(
             '/contents/{}@draft'.format(id),
             status=503,

cnxauthoring/tests/test_storage/test_postgresql.py

@@ -81,3 +81,23 @@ def test_add_get_and_remove_binder(self):
         self.assertEqual({k: tuple(sorted(v)) for k, v in result.acls.items()},
                          {'user2': ('view',)})
 
+    def test_restart(self):
+        from ...storage.database import CONNECTION_SETTINGS_KEY
+        import psycopg2
+        settings = integration_test_settings()
+        test_db = settings[CONNECTION_SETTINGS_KEY]
+        self.addCleanup(setattr, self.storage,
+                        'conn', psycopg2.connect(test_db))
+
+        # 0 if the connection is open, nonzero if it is closed or broken.
+        OPEN = 0
+
+        self.assertEqual(self.storage.conn.closed, OPEN)
+
+        self.storage.conn.close()
+
+        self.assertNotEqual(self.storage.conn.closed, OPEN)
+
+        self.storage.restart()
+
+        self.assertEqual(self.storage.conn.closed, OPEN)

cnxauthoring/views.py

@@ -62,7 +62,12 @@ def wrapper(*args, **kwargs):
                 storage.abort()
             except storage.Error:
                 logger.exception('Storage failed to abort')
-            raise httpexceptions.HTTPServiceUnavailable()
+                try:
+                    storage.restart()
+                except storage.Error:
+                    logger.exception('Storage failed to restart')
+            finally:
+                raise httpexceptions.HTTPServiceUnavailable()
     return wrapper