[Core] Turn on WAL mode for cluster job table (#3923)

* fix: add retry wrapper around db operation

* fix: enable wal mode for job table

* fix: remove retries from db utils

* fix: add WAL to jobs table

* fix: tidy up formatting

sky/skylet/job_lib.py

@@ -8,6 +8,7 @@
 import os
 import pathlib
 import shlex
+import sqlite3
 import subprocess
 import time
 import typing
@@ -55,6 +56,20 @@ class JobInfoLoc(enum.IntEnum):
 
 
 def create_table(cursor, conn):
+    # Enable WAL mode to avoid locking issues.
+    # See: issue #3863, #1441 and PR #1509
+    # https://github.com/microsoft/WSL/issues/2395
+    # TODO(romilb): We do not enable WAL for WSL because of known issue in WSL.
+    #  This may cause the database locked problem from WSL issue #1441.
+    if not common_utils.is_wsl():
+        try:
+            cursor.execute('PRAGMA journal_mode=WAL')
+        except sqlite3.OperationalError as e:
+            if 'database is locked' not in str(e):
+                raise
+            # If the database is locked, it is OK to continue, as the WAL mode
+            # is not critical and is likely to be enabled by other processes.
+
     cursor.execute("""\
         CREATE TABLE IF NOT EXISTS jobs (
         job_id INTEGER PRIMARY KEY AUTOINCREMENT,