Merge pull request #227 from opensafely-core/bump-jobrunner-and-pipeline

feat: Update to jobrunner 2.71.0 and pipeline 2023.11.6.145820

opensafely/_vendor/chardet-3.0.4.dist-info/RECORD

@@ -1,4 +1,4 @@
-../../bin/chardetect,sha256=k6iJnsNSUeMlNutkq2v__zPIQpuglvel238-vfYDBlA,283
+../../bin/chardetect,sha256=Iim50Xo_A-6jNKXhng0od1YWXEr5OEzrv2MBb1TfLes,256
 chardet-3.0.4.dist-info/DESCRIPTION.rst,sha256=PQ4sBsMyKFZkjC6QpmbpLn0UtCNyeb-ZqvCGEgyZMGk,2174
 chardet-3.0.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
 chardet-3.0.4.dist-info/METADATA,sha256=RV_2I4B1Z586DL8oVO5Kp7X5bUdQ5EuKAvNoAEF8wSw,3239

opensafely/_vendor/distro-1.8.0.dist-info/RECORD

@@ -1,4 +1,4 @@
-../../bin/distro,sha256=24TzW2cWStxsIbDGwqb31a4yu0LPhBq-sA9gV1wIgWs,274
+../../bin/distro,sha256=LL7TkGdbIp5yO7jgBe9K5tlwiCJXMlG9hKhj5qmlETo,247
 distro-1.8.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
 distro-1.8.0.dist-info/LICENSE,sha256=y16Ofl9KOYjhBjwULGDcLfdWBfTEZRXnduOspt-XbhQ,11325
 distro-1.8.0.dist-info/METADATA,sha256=NhYw94UPXb78_Z3_VtLxTJ1zQgUUKoTndg10uKJX800,6915

opensafely/_vendor/jobrunner/cli/local_run.py

@@ -450,9 +450,6 @@ def create_job_request_and_jobs(project_dir, actions, force_run_dependencies):
         workspace=project_dir.name,
         database_name="dummy",
         force_run_dependencies=force_run_dependencies,
-        # The default behaviour of refusing to run if a dependency has failed
-        # makes for an awkward workflow when iterating in development
-        force_run_failed=True,
         branch="",
         original={"created_by": getuser()},
     )

opensafely/_vendor/jobrunner/config.py

@@ -86,11 +86,21 @@ def _is_valid_backend_name(name):
 
 DOCKER_REGISTRY = os.environ.get("DOCKER_REGISTRY", "ghcr.io/opensafely-core")
 
-DATABASE_URLS = {
-    "full": os.environ.get("FULL_DATABASE_URL"),
-    "slice": os.environ.get("SLICE_DATABASE_URL"),
-    "dummy": os.environ.get("DUMMY_DATABASE_URL"),
-}
+
+def database_urls_from_env(env):
+    db_names = ["default", "include_t1oo"]
+    return {
+        db_name: db_url
+        for db_name, db_url in [
+            (db_name, env.get(f"{db_name.upper()}_DATABASE_URL"))
+            for db_name in db_names
+        ]
+        if db_url
+    }
+
+
+DATABASE_URLS = database_urls_from_env(os.environ)
+
 
 TEMP_DATABASE_NAME = os.environ.get("TEMP_DATABASE_NAME")
 
@@ -128,6 +138,31 @@ def _is_valid_backend_name(name):
 MAX_RETRIES = int(os.environ.get("MAX_RETRIES", 0))
 
 
+LEVEL4_MAX_FILESIZE = int(
+    os.environ.get("LEVEL4_MAX_FILESIZE", 16 * 1024 * 1024)
+)  # 16mb
+
+
+# TODO: we might want to take this list from pipeline if we implement it there.
+LEVEL4_FILE_TYPES = [
+    # tables
+    ".csv",
+    ".tsv",
+    # images
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".svg",
+    ".svgz",
+    # reports
+    ".html",
+    ".pdf",
+    ".txt",
+    ".log",
+    ".json",
+    ".md",
+]
+
 STATA_LICENSE = os.environ.get("STATA_LICENSE")
 STATA_LICENSE_REPO = os.environ.get(
     "STATA_LICENSE_REPO",

opensafely/_vendor/jobrunner/create_or_update_jobs.py

@@ -265,15 +265,9 @@ def job_should_be_rerun(job_request, job):
     # Otherwise if it succeeded last time there's no need to run again
     if job.state == State.SUCCEEDED:
         return False
-    # If it failed last time ...
+    # If it failed last time, re-run it by default
     elif job.state == State.FAILED:
-        # ... and we're forcing failed jobs to re-run then re-run it
-        if job_request.force_run_failed:
-            return True
-        # Otherwise it's an error condition
-        raise JobRequestError(
-            f"{job.action} failed on a previous run and must be re-run"
-        )
+        return True
     else:
         raise ValueError(f"Invalid state: {job}")
 
@@ -289,7 +283,7 @@ def assert_new_jobs_created(job_request, new_jobs, current_jobs):
     # is treated as a successful outcome because we've already done everything that was
     # requested.
     if RUN_ALL_COMMAND in job_request.requested_actions:
-        raise NothingToDoError()
+        raise NothingToDoError("All actions have already completed succesfully")
 
     # The other reason is that every requested action is already running or pending,
     # this is considered a user error.
@@ -298,7 +292,7 @@ def assert_new_jobs_created(job_request, new_jobs, current_jobs):
         current_job_states.get(action) for action in job_request.requested_actions
     }
     if requested_action_states <= {State.PENDING, State.RUNNING}:
-        raise JobRequestError("All requested actions were already scheduled to run")
+        raise NothingToDoError("All requested actions were already scheduled to run")
 
     # But if we get here then we've somehow failed to schedule new jobs despite the fact
     # that some of the actions we depend on have failed, which is a bug.