DEV-1484: Add isort ymlfmt toml sort and reformat (#390)

Add isort, ymlfmt and toml-sort to pre-commit-config and reformat.

.pre-commit-config.yaml

@@ -1,21 +1,46 @@
+---
 repos:
-    - repo: [email protected]:Yelp/detect-secrets
-      rev: v0.13.0
-      hooks:
-          - id: detect-secrets
-            args: [ '--baseline', '.secrets.baseline' ]
-    - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.1.0 # last version to support py36
-      hooks:
-          - id: check-json
-          - id: check-toml
-          - id: check-yaml
-          - id: end-of-file-fixer
-          - id: fix-encoding-pragma
-            args: [ --remove ]
-          - id: no-commit-to-branch
-            args: [ --branch, develop, --branch, master, --pattern, release/.* ]
-          - id: pretty-format-json
-            args: [ --autofix ]
-          - id: trailing-whitespace
-            args: [ --markdown-linebreak-ext=md ]
+  - repo: [email protected]:Yelp/detect-secrets
+    rev: v0.13.0
+    hooks:
+      - id: detect-secrets
+        args: [--baseline, .secrets.baseline]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0 # last version to support py36
+    hooks:
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: fix-encoding-pragma
+        args: [--remove]
+      - id: no-commit-to-branch
+        args: [--branch, develop, --branch, master, --pattern, release/.*]
+      - id: pretty-format-json
+        args: [--autofix]
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.6.4    # last version to support pre-commit 1.21.0 in Jenkins
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: [--profile, black]
+      - id: isort
+        name: isort (cython)
+        types: [cython]
+        args: [--profile, black]
+      - id: isort
+        name: isort (pyi)
+        types: [pyi]
+        args: [--profile, black]
+  - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt
+    rev: 0.2.1
+    hooks:
+      - id: yamlfmt
+        args: [--mapping, '2', --sequence, '4', --offset, '2', --width, '80']
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.19.0 # last version to support py36
+    hooks:
+      - id: toml-sort
+        args: [--in-place]

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2022-09-14T17:17:24Z",
+  "generated_at": "2022-10-11T18:55:31Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -51,7 +51,7 @@
         "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 67,
+        "line_number": 63,
         "type": "Basic Auth Credentials"
       }
     ],
@@ -60,7 +60,7 @@
         "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 42,
+        "line_number": 41,
         "type": "Basic Auth Credentials"
       }
     ],
@@ -69,7 +69,7 @@
         "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 37,
+        "line_number": 33,
         "type": "Basic Auth Credentials"
       }
     ]

.travis.yml

@@ -1,3 +1,4 @@
+---
 language: python
 dist: focal
 
@@ -9,7 +10,7 @@ python:
   - 3.8
 
 addons:
-  postgresql: "13"
+  postgresql: '13'
   apt:
     sources:
       - sourceline: deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main 13

bin/update_related_case_caches.py

@@ -10,9 +10,9 @@
 import getpass
 import logging
 
-from gdcdatamodel import models as md
 from psqlgraph import PsqlGraphDriver
 
+from gdcdatamodel import models as md
 
 logging.basicConfig()
 logger = logging.getLogger("update_related_cases_caches")

docker-compose-ci.yaml

@@ -1,4 +1,5 @@
-version: "3.3"
+---
+version: '3.3'
 services:
   postgres:
     image: postgres:9.4
@@ -16,9 +17,9 @@ services:
       HTTPS_PROXY: http://cloud-proxy:3128
       HTTP_PROXY: http://cloud-proxy:3128
     volumes:
-    - .:/home/jenkins
-    - $SSH_AUTH_SOCK:$SSH_AUTH_SOCK
+      - .:/home/jenkins
+      - $SSH_AUTH_SOCK:$SSH_AUTH_SOCK
     command: bash -c "./wait-for-it.sh localhost:5432 -t 120 && tox"
-    network_mode: "service:postgres"
+    network_mode: service:postgres
     depends_on:
       - postgres

docs/bin/schemata_to_graphviz.py

@@ -1,7 +1,9 @@
 import os
-from gdcdatamodel import models as m
+
 from graphviz import Digraph
 
+from gdcdatamodel import models as m
+
 
 def build_visualization():
     print('Building schema documentation...')

gdcdatamodel/__main__.py

@@ -1,11 +1,11 @@
 import argparse
 import getpass
-import psqlgraph
 
-from models import *                              # noqa
-from psqlgraph import *                           # noqa
-from sqlalchemy import *                          # noqa
+import psqlgraph
+from models import *  # noqa
 from models.versioned_nodes import VersionedNode  # noqa
+from psqlgraph import *  # noqa
+from sqlalchemy import *  # noqa
 
 try:
     import IPython

gdcdatamodel/gdc_postgres_admin.py

@@ -11,18 +11,14 @@
 import time
 
 import sqlalchemy as sa
+from psqlgraph import create_all, ext
 from psqlgraph.base import ORMBase
 from sqlalchemy import create_engine
 from sqlalchemy.exc import OperationalError
 
 #: Required but 'unused' import to register GDC models
 from . import models  # noqa
 
-from psqlgraph import (
-    create_all,
-    ext,
-)
-
 logging.basicConfig()
 logger = logging.getLogger("gdc_postgres_admin")
 logger.setLevel(logging.INFO)

gdcdatamodel/models/__init__.py

@@ -23,54 +23,40 @@
 except ImportError:
     from functools32 import lru_cache
 
-from types import ModuleType
+import hashlib
 import logging
-
 from collections import defaultdict
+from types import ModuleType
 
+from psqlgraph import Edge, Node, ext, pg_property
+from sqlalchemy import and_, event
+from sqlalchemy.ext.hybrid import Comparator, hybrid_property
 from sqlalchemy.orm import configure_mappers
 
-import hashlib
 from gdcdatamodel.models import (
-    versioned_nodes,
+    batch,
     notifications,
-    submission,
-    redaction,
     qcreport,
+    redaction,
     released_data,
     studyrule,
-    batch,
+    submission,
+    versioned_nodes,
     versioning,
 )
-
-from sqlalchemy import event, and_
-
-from psqlgraph import Node, Edge, pg_property
-
-from psqlgraph import ext
-
-from sqlalchemy.ext.hybrid import (
-    Comparator,
-    hybrid_property,
-)
-
 from gdcdatamodel.models.caching import (
     NOT_RELATED_CASES_CATEGORIES,
     RELATED_CASES_LINK_NAME,
-    cache_related_cases_on_update,
-    cache_related_cases_on_insert,
     cache_related_cases_on_delete,
+    cache_related_cases_on_insert,
+    cache_related_cases_on_update,
     related_cases_from_cache,
     related_cases_from_parents,
 )
-
-from gdcdatamodel.models.indexes import (
-    cls_add_indexes,
-    get_secondary_key_indexes,
-)
+from gdcdatamodel.models.indexes import cls_add_indexes, get_secondary_key_indexes
 from gdcdatamodel.models.misc import FileReport  # noqa
-from gdcdatamodel.models.versioned_nodes import VersionedNode  # noqa
 from gdcdatamodel.models.utils import py3_to_bytes
+from gdcdatamodel.models.versioned_nodes import VersionedNode  # noqa
 
 logger = logging.getLogger("gdcdatamodel")
 

gdcdatamodel/models/indexes.py

@@ -10,14 +10,13 @@
 
 """
 
+import hashlib
 import logging
 
 from sqlalchemy import Index, func
-import hashlib
 
 from gdcdatamodel.models.utils import py3_to_bytes
 
-
 logger = logging.getLogger(__name__)
 
 

gdcdatamodel/models/versioned_nodes.py

@@ -1,8 +1,8 @@
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.postgresql import ARRAY, JSONB
-from sqlalchemy import Column, Text, DateTime, BigInteger, text, Index
 from copy import copy
 
+from sqlalchemy import BigInteger, Column, DateTime, Index, Text, text
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
+from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
 

gdcdatamodel/models/versioning.py

@@ -3,6 +3,7 @@
 
 import six
 from sqlalchemy import and_, event, select
+
 try:
     from functools import lru_cache
 except ImportError:

gdcdatamodel/query.py

@@ -1,4 +1,4 @@
-from psqlgraph import Node, Edge
+from psqlgraph import Edge, Node
 
 traversals = {}
 terminal_nodes = ['annotations', 'centers', 'archives', 'tissue_source_sites',

gdcdatamodel/validators/__init__.py

@@ -1,2 +1,2 @@
-from .json_validators import GDCJSONValidator
 from .graph_validators import GDCGraphValidator
+from .json_validators import GDCJSONValidator

gdcdatamodel/validators/json_validators.py

@@ -1,6 +1,7 @@
+import re
+
 from gdcdictionary import gdcdictionary
 from jsonschema import Draft4Validator
-import re
 
 missing_prop_re = re.compile("\'([a-zA-Z_-]+)\' is a required property")
 extra_prop_re = re.compile("Additional properties are not allowed \(u\'([a-zA-Z_-]+)\' was unexpected\)")

gdcdatamodel/xml_mappings/tcga_biospecimen.yaml

@@ -1,3 +1,4 @@
+---
 # example:
 #   -
 #     root: xpath
@@ -26,8 +27,7 @@
 #             year: xpath
 
 aliquot:
-  -
-    root: //bio:aliquot
+  - root: //bio:aliquot
     id: .//bio:bcr_aliquot_uuid
     edges:
       analytes: ancestor::bio:analyte/bio:bcr_analyte_uuid
@@ -72,8 +72,7 @@ aliquot:
         type: float
 
 analyte:
-  -
-    root: //bio:analyte
+  - root: //bio:analyte
     id: .//bio:bcr_analyte_uuid
     edges:
       portions: ancestor::bio:portion/bio:bcr_portion_uuid
@@ -109,8 +108,7 @@ analyte:
         type: str
 
 portion:
-  -
-    root: //bio:portion
+  - root: //bio:portion
     id: .//bio:bcr_portion_uuid
     edges:
       samples: ancestor::bio:sample/bio:bcr_sample_uuid
@@ -132,8 +130,7 @@ portion:
         day: ./bio:day_of_creation
         month: ./bio:month_of_creation
         year: ./bio:year_of_creation
-  -
-    root: //bio:shipment_portion
+  - root: //bio:shipment_portion
     id: .//bio:bcr_shipment_portion_uuid
     edges:
       samples: ancestor::bio:sample/bio:bcr_sample_uuid
@@ -183,8 +180,7 @@ portion:
         year: ./bio:shipment_portion_year_of_shipment
 
 sample:
-  -
-    root:
+  - root:
       - //bio_model:sample
       - //bio:sample
     id:
@@ -301,12 +297,11 @@ sample:
           - ./bio_model:time_between_excision_and_freezing
           - ./bio:time_between_excision_and_freezing
         type: float
-      tumor_code_id: null
-      tumor_code: null
+      tumor_code_id:
+      tumor_code:
 
 case:
-  -
-    root: //*[local-name()='patient']
+  - root: //*[local-name()='patient']
     id: .//shared:bcr_patient_uuid
     edges_by_property:
       tissue_source_sites:
@@ -319,8 +314,7 @@ case:
         type: str
 
 slide:
-  -
-    root: //bio:slide
+  - root: //bio:slide
     id: .//shared:bcr_slide_uuid
     edges:
       portions: ancestor::bio:portion/bio:bcr_portion_uuid