Add filesize sniffing and parallelize importing on jobs

.gitignore

@@ -14,6 +14,7 @@ venv*
 v3nv/
 tmp/
 /src/toil/test/cwl/spec*
+/src/toil/test/wdl/wdl-conformance-tests
 /cwltool_deps/
 /docs/generated_rst/
 /docker/Dockerfile

docs/conf.py

@@ -160,7 +160,8 @@ def setup(app):
 autoapi_dirs = ["../src/toil"]
 autodoc_typehints = "description"
 autoapi_keep_files = True
-autoapi_ignore = ["*.pyi", "*/test/cwl/spec*/*.py", "*/fake_mpi_run.py", "*/tutorial_*.py", "*/example_*.py", "*/mkFile.py", "*/debugWorkflow.py"]
+autoapi_ignore = ["*.pyi", "*/test/cwl/spec*/*.py", "*/fake_mpi_run.py", "*/tutorial_*.py", "*/example_*.py", "*/mkFile.py", "*/debugWorkflow.py",
+                  "*/test/wdl/wdl-conformance-tests/*"]
 autoapi_options = [
     "members",
     "undoc-members",

src/toil/common.py

@@ -750,7 +750,7 @@ def addOptions(
     add_cwl_options(parser, suppress=not cwl)
     add_wdl_options(parser, suppress=not wdl)
     # Add shared runner options
-    add_runner_options(parser)
+    add_runner_options(parser, cwl=cwl, wdl=wdl)
 
     def check_arguments(typ: str) -> None:
         """
@@ -764,7 +764,7 @@ def check_arguments(typ: str) -> None:
             add_cwl_options(check_parser)
         if typ == "cwl":
             add_wdl_options(check_parser)
-        add_runner_options(check_parser)
+
         for action in check_parser._actions:
             action.default = SUPPRESS
         other_options, _ = check_parser.parse_known_args(