increase timeout for http requests; add join logic to query

CHANGES.md

@@ -2,7 +2,7 @@
 
 - Drop deprecated model methods
 
-### 0.0.3 [pending release]
+### 0.0.3 May 14, 2024
 
 - Add CiviCRM v4 API usage and make it the default
 - Drop allowing `CiviCRMBase` to be used directly (must subclass to access a CiviCRM entity)

README.md

@@ -8,7 +8,7 @@ older than 3.11 and want to use `pyproject.toml` configuration, install with
 ## Configuration
 
 Configure your credentials in environment variables, a `.civipy` file, or in
-`pyproject.toml` in a `tools.civipy` section. By default CiviPy will read a `.civipy`
+`pyproject.toml` in a `tools.civipy` section. By default, CiviPy will read a `.civipy`
 file anywhere in the current working directory up to your project root, or your
 project's `pyproject.toml` file, or a `.civipy` file in the user's home folder.
 Settings in environment variables will overwrite any file settings. Alternatively,
@@ -44,15 +44,16 @@ You can specify in an environment variable either a directory to find a `.civipy
 configuration file in, or a file to read as a `.civipy` configuration file.
 
 ## Usage
-There are class methods for retrieving and creating records and instance methods
-for working with them.
+There is a Django-style `.objects` attribute on each record model with `filter`, `values`,
+`order_by`, and `all` methods for querying; and `create`, `delete`, and `update` methods
+for making changes. Model instances also have `save` and `delete` methods.
 
 ```python
-from civipy import CiviContact, CiviEmail
+from civipy import CiviContact
 
-contact = CiviContact.find_by_email("[email protected]")
-email = CiviEmail.find_or_create(where={"contact_id": contact.id, "email": "[email protected]"})
-contact.update(nick_name="Ana")
+contact = CiviContact.objects.filter(email_primary__email="[email protected]")[0]
+contact.nick_name = "Ana"
+contact.save()
 ```
 
 Each CiviCRM Entity is represented by a subclass of CiviCRMBase; if you need an entity
@@ -66,9 +67,6 @@ class CiviNewEntity(civipy.CiviCRMBase):
     pass
 ```
 
-Many CiviCRM Actions have a corresponding method (e.g. `get`, `create`), and there are
-also a number of convenience methods which do more processing (e.g. `find_or_create`).
-
 ## Copyright & License
 civipy Copyright © 2020 Ana Nelson
 

civipy/base/base.py

@@ -102,7 +102,11 @@ def find_and_update(cls, where: CiviValue, **kwargs: CiviValue) -> CiviEntity |
 
         Returns an object of class cls populated with this object's data if found, otherwise
         returns None."""
-        warn("model.find_and_update will be removed in v0.1.0, use model.objects methods", DeprecationWarning, stacklevel=2)
+        warn(
+            "model.find_and_update will be removed in v0.1.0, use model.objects methods",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         query = cls.objects._interface().where(where)
         response = cls.get(**query)
         if response["count"] == 0:
@@ -128,16 +132,20 @@ def find_or_create(cls, where: CiviValue, do_update: bool = False, **kwargs: Civ
 
         Returns an object of class cls populated with the found, updated, or created
         object's data."""
-        warn("model.find_or_create will be removed in v0.1.0, use model.objects methods", DeprecationWarning, stacklevel=2)
+        warn(
+            "model.find_or_create will be removed in v0.1.0, use model.objects methods",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         if do_update:
             obj = cls.find_and_update(where, **kwargs)
         else:
-            obj = cls.find(**kwargs)
+            obj = cls.find(**where)
 
         if obj is None:
             query = where.copy()
             query.update(kwargs)
-            return cls.create(**query)
+            return cls.objects.create(**query)
         return obj
 
     @classmethod
@@ -171,9 +179,9 @@ def __repr__(self: CiviEntity):
     def __getattr__(self: CiviEntity, key: str):
         if key in self.civi:
             return self.civi[key]
-        elif key.startswith("civi_"):
+        elif key.startswith("civi_") and key[5:] in self.civi:
             return self.civi[key[5:]]
-        return object.__getattribute__(self, key)
+        return None
 
     def __setattr__(self: CiviEntity, key: str, value: str | int | None) -> None:
         if key == "civi":

civipy/contact.py

@@ -67,7 +67,7 @@ def query_filter_hook(cls, version: Literal["3", "4"], query: CiviValue) -> Civi
         return query
 
     @classmethod
-    def create_or_increment_relationship(
+    def create_or_increment_relationship(  # noqa PLR0913 (too many args)
         cls,
         contact_id_a: int,
         contact_id_b: int,

civipy/interface/base.py

@@ -1,5 +1,6 @@
 from typing import TypedDict, Literal, Callable
 from warnings import warn
+import urllib3
 
 CiviValue = dict[str, int | str]
 CiviV3Request = CiviValue
@@ -65,12 +66,17 @@ class BaseInterface:
 
     def __init__(self):
         self.func: Callable[[str, str, CiviValue], CiviResponse] | None = None
+        self.http: urllib3.PoolManager | None = None
+
+    def _configure_http_connection(self) -> None:
+        timeout = urllib3.util.Timeout(connect=10.0, read=30.0)
+        self.http = urllib3.PoolManager(timeout=timeout)
 
     def __call__(self, action: str, entity: str, params: CiviValue) -> CiviResponse:
         warn(
             "interface.__call__ will be removed in v0.1.0, use interface.execute instead",
             DeprecationWarning,
-            stacklevel=2
+            stacklevel=2,
         )
         return self.execute(action, entity, params)
 
@@ -96,6 +102,10 @@ def execute(self, action: str, entity: str, params: CiviValue) -> CiviResponse:
     def select(fields: list[str]) -> CiviValue | CiviV4Request:
         raise NotImplementedError
 
+    @staticmethod
+    def join(tables: list[tuple[str, str, str]]) -> CiviValue | CiviV4Request:
+        raise NotImplementedError
+
     @staticmethod
     def sort(kwargs: CiviValue) -> CiviValue | CiviV4Request:
         raise NotImplementedError

civipy/interface/query.py

@@ -15,6 +15,7 @@ def __init__(self, model: Type["CiviCRMBase"]) -> None:
             self._entity = model.__name__[4:] if model.__name__[:4] == "Civi" else model.__name__
         self._select = None
         self._filter = None
+        self._join = None
         self._limit = None
         self._order = None
         self._result_cache: list | None = None
@@ -59,7 +60,7 @@ def __getitem__(self, k: int | slice):
         start, stop, step = [None if v is None else int(v) for v in (k.start, k.stop, k.step)]
         query._limit = {} if start is None else {"offset": start}
         query._limit["limit"] = stop - query._limit.get("offset", 0)
-        return list(query)[:: step] if step else query
+        return list(query)[::step] if step else query
 
     _interface_reference: Interface | None = None
 
@@ -74,8 +75,9 @@ def _interface(cls) -> Interface:
     def _chain(self):
         query = self.__class__(model=self.model)
         query._select = self._select
-        query._limit = self._limit
         query._filter = self._filter
+        query._join = self._join
+        query._limit = self._limit
         query._order = self._order
         return query
 
@@ -85,6 +87,8 @@ def _compose(self, values: Optional["CiviValue"] = None) -> "CiviValue":
         params = {}
         if self._select is not None:
             self._deep_update(params, interface.select(self._select))
+        if self._join is not None:
+            self._deep_update(params, interface.join(self._join))
         if self._filter is not None:
             where = interface.where(self._filter)
             if hasattr(self.model, "query_filter_hook"):
@@ -131,7 +135,22 @@ def filter(self, **kwargs):
 
     def values(self, *args):
         query = self._chain()
-        query._select = args
+        query._select = []
+        for field in args:
+            foreign_key, _, join_field = field.partition(".")
+            if not join_field:
+                query._select.append(field)
+                continue
+            if query._join is None:
+                query._join = {}
+            if foreign_key not in query._join:
+                table = foreign_key[:-3]
+                if table == "entity":
+                    name, table = self.model._implicit_join
+                else:
+                    name = table.title()
+                query._join[foreign_key] = (name, table)
+            query._select.append(".".join((query._join[foreign_key][1], join_field)))
         return query
 
     def order_by(self, **kwargs):

civipy/interface/v3.py

@@ -21,6 +21,7 @@ def execute(self, action: str, entity: str, params: CiviValue) -> CiviV3Response
                 raise CiviProgrammingError(f"API version '{SETTINGS.api_version}' cannot use V3Interface")
             if SETTINGS.api_type == "http":
                 self.func = self.http_request
+                self._configure_http_connection()
             elif SETTINGS.api_type in ("drush", "wp"):
                 self.func = self.run_drush_or_wp_process
             elif SETTINGS.api_type == "cvcli":
@@ -44,7 +45,7 @@ def http_request(self, action: str, entity: str, kwargs: CiviValue) -> CiviV3Res
         method = "GET" if action.startswith("get") else "POST"
         # urllib3 uses the `fields` parameter to compose the query string for GET requests,
         # and uses the same parameter to compose form data for POST requests
-        response = urllib3.request(method, SETTINGS.rest_base, fields=params, headers=headers)
+        response = self.http.request(method, SETTINGS.rest_base, fields=params, headers=headers)
         return self.process_http_response(response)
 
     @staticmethod
@@ -77,7 +78,7 @@ def run_cv_cli_process(self, action: str, entity: str, params: CiviValue) -> Civ
         # cli.php -e entity -a action [-u user] [-s site] [--output|--json] [PARAMS]
         params = ["--%s=%s" % (k, v) for k, v in params.items()]
         process = subprocess.run(
-            [SETTINGS.rest_base, "-e", entity, "-a", action, "--json"] + params, capture_output=True
+            [SETTINGS.rest_base, "-e", entity, "-a", action, "--json"] + params, capture_output=True, check=True
         )
         return self.process_json_response(json.loads(process.stdout.decode("UTF-8")))
 
@@ -86,6 +87,7 @@ def run_drush_or_wp_process(self, action: str, entity: str, params: CiviValue) -
         process = subprocess.run(
             [SETTINGS.rest_base, "civicrm-api", "--out=json", "--in=json", "%s.%s" % (entity, action)],
             capture_output=True,
+            check=True,
             input=json.dumps(params).encode("UTF-8"),
         )
         return self.process_json_response(json.loads(process.stdout.decode("UTF-8")))
@@ -94,6 +96,7 @@ def run_drush_or_wp_process(self, action: str, entity: str, params: CiviValue) -
     def _pre_process(params: CiviValue) -> CiviValue:
         if "options" in params:
             params["options"] = json.dumps(params["options"], separators=(",", ":"))
+        return params
 
     @staticmethod
     def process_json_response(data: CiviV3Response) -> CiviV3Response:
@@ -105,6 +108,10 @@ def process_json_response(data: CiviV3Response) -> CiviV3Response:
     def select(fields: list[str]) -> CiviValue:
         return {"return": fields}
 
+    @staticmethod
+    def join(tables: list[tuple[str, str, str]]) -> CiviValue:
+        raise NotImplementedError
+
     @staticmethod
     def sort(kwargs: CiviValue) -> CiviValue:
         option = []
@@ -132,11 +139,11 @@ def where(cls, kwargs: CiviValue) -> CiviValue:
             if len(parts) > 1 and parts[-1] in cls.operators:
                 *parts, op = parts
                 if op == "isnull":
-                    val = {"IS NULL": 1} if val else {"IS NOT NULL": 1}
+                    val = {"IS NULL": 1} if val else {"IS NOT NULL": 1}  # noqa PLW2901 (loop var)
                 elif (op.endswith("between") or op.endswith("in")) and not isinstance(val, list):
-                    raise CiviProgrammingError(f"Must provide a list for `in` or `between` operators.")
+                    raise CiviProgrammingError(f"Must provide a list for `{op}` operator.")
                 else:
-                    val = {cls.operators[op]: val}
+                    val = {cls.operators[op]: val}  # noqa PLW2901 (loop var)
             option[".".join(parts)] = val
         return kwargs
 

civipy/interface/v4.py

@@ -22,6 +22,7 @@ def execute(self, action: str, entity: str, params: CiviValue) -> CiviV4Response
                 raise CiviProgrammingError(f"API version '{SETTINGS.api_version}' cannot use V4Interface")
             if SETTINGS.api_type == "http":
                 self.func = self.http_request
+                self._configure_http_connection()
             elif SETTINGS.api_type == "drush":
                 self.func = self.run_drush_process
             # API v4 not available to wp-cli - see https://docs.civicrm.org/dev/en/latest/api/v4/usage/#wp-cli
@@ -34,8 +35,9 @@ def execute(self, action: str, entity: str, params: CiviValue) -> CiviV4Response
     def http_request(self, action: str, entity: str, kwargs: CiviV4Request) -> CiviV4Response:
         # v4 see https://docs.civicrm.org/dev/en/latest/api/v4/rest/
         url = urljoin(SETTINGS.rest_base, "/".join((entity, action)))
-        body = urlencode({"params": json.dumps(kwargs, separators=(",", ":"))})
-        logger.debug("Request for %s: %s", url, body)
+        params = json.dumps(kwargs, separators=(",", ":"))
+        body = urlencode({"params": params})
+        logger.debug("Request for %s: %s", url, params)
 
         # header for v4 API per https://docs.civicrm.org/dev/en/latest/api/v4/rest/#x-requested-with
         headers = {
@@ -45,7 +47,7 @@ def http_request(self, action: str, entity: str, kwargs: CiviV4Request) -> CiviV
         }
 
         # v4 docs: "Requests are typically submitted with HTTP POST, but read-only operations may use HTTP GET."
-        response = urllib3.request("POST", url, body=body, headers=headers)
+        response = self.http.request("POST", url, body=body, headers=headers)
         return self.process_http_response(response)
 
     def process_http_response(self, response: urllib3.BaseHTTPResponse) -> CiviV4Response:
@@ -65,13 +67,15 @@ def run_cv_cli_process(self, action: str, entity: str, params: CiviValue) -> Civ
                 json.dumps(params, separators=(",", ":")).encode("UTF-8"),
             ],
             capture_output=True,
+            check=True,
         )
         return self.process_json_response(json.loads(process.stdout.decode("UTF-8")))
 
     def run_drush_process(self, action: str, entity: str, params: CiviValue) -> CiviV4Response:
         process = subprocess.run(
             [SETTINGS.rest_base, "civicrm-api", "version=4", "--out=json", "--in=json", "%s.%s" % (entity, action)],
             capture_output=True,
+            check=True,
             input=json.dumps(params, separators=(",", ":")).encode("UTF-8"),
         )
         return self.process_json_response(json.loads(process.stdout.decode("UTF-8")))
@@ -86,6 +90,13 @@ def process_json_response(data: CiviV4Response) -> CiviV4Response:
     def select(fields: list[str]) -> CiviV4Request:
         return {"select": fields}
 
+    @staticmethod
+    def join(tables: dict[str, tuple[str, str]]) -> CiviV4Request:
+        option = []
+        for foreign_key, (name, table) in tables.items():
+            option.append([f"{name} AS {table}", "LEFT", [foreign_key, "=", f"{table}.id"]])
+        return {"join": option}
+
     @staticmethod
     def sort(kwargs: CiviValue) -> CiviValue | CiviV4Request:
         option = {}
@@ -125,7 +136,7 @@ def where(cls, kwargs: CiviValue) -> CiviV4Request:
                 elif op == "isempty":
                     value = ["IS EMPTY"] if val else ["IS NOT EMPTY"]
                 elif (op.endswith("between") or op.endswith("in")) and not isinstance(val, list):
-                    raise CiviProgrammingError(f"Must provide a list for `in` or `between` operators.")
+                    raise CiviProgrammingError(f"Must provide a list for `{op}` operator.")
                 else:
                     value = [cls.operators[op], val]
             else:

civipy/user.py

@@ -2,16 +2,13 @@
 from civipy.exceptions import NoResultError, NonUniqueResultError
 
 
-class CiviUFField(CiviCRMBase):
-    ...
+class CiviUFField(CiviCRMBase): ...
 
 
-class CiviUFGroup(CiviCRMBase):
-    ...
+class CiviUFGroup(CiviCRMBase): ...
 
 
-class CiviUFJoin(CiviCRMBase):
-    ...
+class CiviUFJoin(CiviCRMBase): ...
 
 
 class CiviUFMatch(CiviCRMBase):
@@ -53,5 +50,4 @@ def connect(cls, host_user: int, contact_id: int, domain_id: int = 1):
         return cls.objects.create(domain_id=domain_id, uf_id=host_user, contact_id=contact_id)
 
 
-class CiviUser(CiviCRMBase):
-    ...
+class CiviUser(CiviCRMBase): ...

pyproject.toml

@@ -62,8 +62,10 @@ norecursedirs = ["__pycache__", "*.egg-info", ".*", "dist"]
 
 [tool.ruff]
 src = ["src"]
-select = ["E", "F", "PL"]
 line-length = 120
 target-version = "py310"
 extend-exclude = ["react"]
+
+[tool.ruff.lint]
+select = ["E", "F", "PL"]
 ignore = ["PLR2004"]