feat: add computed relationships

* work for select, mutations, rpc
* overrides detected relationships

CHANGELOG.md

@@ -40,7 +40,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
    + Can generate the plan for different media types using the `for` parameter: `Accept: application/vnd.pgrst.plan; for="application/vnd.pgrst.object"`
    + Different options for the plan can be used with the `options` parameter: `Accept: application/vnd.pgrst.plan; options=analyze|verbose|settings|buffers|wal`
    + The plan can be obtained in text or json by using different media type suffixes: `Accept: application/vnd.pgrst.plan+text` and `Accept: application/vnd.pgrst.plan+json`.
- - #2397, Fix race conditions managing database connection helper - @robx
+ - #2144, Allow extending/overriding relationships for resource embedding - @steve-chavez
 
 ### Fixed
 
@@ -65,6 +65,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #2376, OPTIONS requests no longer start an empty database transaction - @steve-chavez
  - #2395, Allow using columns with dollar sign($) without double quoting in filters and `select` - @steve-chavez
  - #2410, Fix loop crash error on startup in Postgres 15 beta 2. Log: "UNION types \"char\" and text cannot be matched". - @yevon
+ - #2397, Fix race conditions managing database connection helper - @robx
 
 ### Changed
 

postgrest.cabal

@@ -192,6 +192,7 @@ test-suite spec
                       Feature.OpenApi.SecurityOpenApiSpec
                       Feature.OptionsSpec
                       Feature.Query.AndOrParamsSpec
+                      Feature.Query.ComputedRelsSpec
                       Feature.Query.DeleteSpec
                       Feature.Query.EmbedDisambiguationSpec
                       Feature.Query.EmbedInnerJoinSpec

src/PostgREST/DbStructure.hs

@@ -90,18 +90,36 @@ queryDbStructure schemas extraSearchPath prepared = do
   keyDeps <- SQL.statement (schemas, extraSearchPath) $ allViewsKeyDependencies prepared
   m2oRels <- SQL.statement mempty $ allM2ORels pgVer prepared
   procs   <- SQL.statement schemas $ allProcs pgVer prepared
+  cRels   <- SQL.statement mempty $ allComputedRels prepared
 
   let tabsWViewsPks = addViewPrimaryKeys tabs keyDeps
-      rels          = relsToMap $ addO2MRels $ addM2MRels tabsWViewsPks $ addViewM2ORels keyDeps m2oRels
+      rels          = addO2MRels $ addM2MRels tabsWViewsPks $ addViewM2ORels keyDeps m2oRels
 
   return $ removeInternal schemas $ DbStructure {
       dbTables = tabsWViewsPks
-    , dbRelationships = rels
+    , dbRelationships = getOverrideRelationshipsMap rels cRels
     , dbProcs = procs
     }
+
+-- | overrides detected relationships with the computed relationships and gets the RelationshipsMap
+getOverrideRelationshipsMap :: [Relationship] -> [Relationship] -> RelationshipsMap
+getOverrideRelationshipsMap rels cRels =
+  sort <$> deformedRelMap patchedRels
   where
-    relsToMap = map sort . HM.fromListWith (++) . map ((\(x, fSch, y) -> ((x, fSch), [y])) . addKey)
-    addKey rel = (relTable rel, qiSchema $ relForeignTable rel, rel)
+    -- there can only be a single (table_type, func_name) pair in a function definition `test.function(table_type)`, so we use HM.fromList to disallow duplicates
+    computedRels  = HM.fromList $ relMapKey <$> cRels
+    -- here we override the detected relationships with the user computed relationships, HM.union makes sure computedRels prevail
+    patchedRels   = HM.union computedRels (relsMap rels)
+    relsMap = HM.fromListWith (++) . fmap relMapKey
+    relMapKey rel = case rel of
+      Relationship{relTable,relForeignTable} -> ((relTable, relForeignTable), [rel])
+      -- we use (relTable, relFunction) as key to override detected relationships with the function name
+      ComputedRelationship{relTable,relFunction} -> ((relTable, relFunction), [rel])
+    -- Since a relationship is between a table and foreign table, the logical way to index/search is by their table/ftable QualifiedIdentifier
+    -- However, because we allow searching a relationship by the columns of the foreign key(using the "column as target" disambiguation) we lose the
+    -- ability to index by the foreign table name, so we deform the key. TODO remove once support for "column as target" is gone.
+    deformedRelMap = HM.fromListWith (++) . fmap addDeformedRelKey . HM.toList
+    addDeformedRelKey ((relT, relFT), rls) = ((relT, qiSchema relFT), rls)
 
 -- | Remove db objects that belong to an internal schema(not exposed through the API) from the DbStructure.
 removeInternal :: [Schema] -> DbStructure -> DbStructure
@@ -113,7 +131,8 @@ removeInternal schemas dbStruct =
     , dbProcs         = dbProcs dbStruct -- procs are only obtained from the exposed schemas, no need to filter them.
     }
   where
-    hasInternalJunction rel = case relCardinality rel of
+    hasInternalJunction ComputedRelationship{} = False
+    hasInternalJunction Relationship{relCardinality=card} = case card of
       M2M Junction{junTable} -> qiSchema junTable `notElem` schemas
       _                      -> False
 
@@ -643,6 +662,50 @@ allM2ORels pgVer =
       else mempty) <>
     "ORDER BY conrelid, conname"
 
+allComputedRels :: Bool -> SQL.Statement () [Relationship]
+allComputedRels =
+  SQL.Statement sql HE.noParams (HD.rowList cRelRow)
+ where
+  sql = [q|
+    with
+    all_relations as (
+      select reltype
+      from pg_class
+      where relkind in ('v','r','m','f','p')
+    ),
+    computed_rels as (
+      select
+        p.pronamespace::regnamespace::text as schema,
+        p.proname::text                    as name,
+        arg_schema.nspname::text           as rel_table_schema,
+        arg_name.typname::text             as rel_table_name,
+        ret_schema.nspname::text           as rel_ftable_schema,
+        ret_name.typname::text             as rel_ftable_name,
+        p.prorows = 1                      as single_row
+      from pg_proc p
+        join pg_type      arg_name   on arg_name.oid = p.proargtypes[0]
+        join pg_namespace arg_schema on arg_schema.oid = arg_name.typnamespace
+        join pg_type      ret_name   on ret_name.oid = p.prorettype
+        join pg_namespace ret_schema on ret_schema.oid = ret_name.typnamespace
+      where
+        p.pronargs = 1
+        and p.proargtypes[0] in (select reltype from all_relations)
+        and p.prorettype in (select reltype from all_relations)
+    )
+    select
+      *,
+      row(rel_table_schema, rel_table_name) = row(rel_ftable_schema, rel_ftable_name) as is_self
+    from computed_rels;
+  |]
+
+  cRelRow =
+    ComputedRelationship <$>
+    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
+    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
+    (QualifiedIdentifier <$> column HD.text <*> column HD.text) <*>
+    column HD.bool <*>
+    column HD.bool
+
 -- | Returns all the views' primary keys and foreign keys dependencies
 allViewsKeyDependencies :: Bool -> SQL.Statement ([Schema], [Schema]) [ViewKeyDependency]
 allViewsKeyDependencies =

src/PostgREST/DbStructure/Relationship.hs

@@ -26,6 +26,13 @@ data Relationship = Relationship
   , relTableIsView  :: Bool
   , relFTableIsView :: Bool
   }
+  | ComputedRelationship
+  { relFunction     :: QualifiedIdentifier
+  , relTable        :: QualifiedIdentifier
+  , relForeignTable :: QualifiedIdentifier
+  , relToOne        :: Bool
+  , relIsSelf       :: Bool
+  }
   deriving (Eq, Ord, Generic, JSON.ToJSON)
 
 -- | The relationship cardinality

src/PostgREST/Error.hs

@@ -171,6 +171,8 @@ instance JSON.ToJSON ApiRequestError where
     "hint"    .= ("Try renaming the parameters or the function itself in the database so function overloading can be resolved" :: Text)]
 
 compressedRel :: Relationship -> JSON.Value
+-- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
+compressedRel ComputedRelationship{} = JSON.object mempty
 compressedRel Relationship{..} =
   let
     fmtEls els = "(" <> T.intercalate ", " els <> ")"
@@ -200,6 +202,8 @@ relHint rels = T.intercalate ", " (hintList <$> rels)
         M2M Junction{..} -> buildHint (qiName junTable)
         M2O cons _       -> buildHint cons
         O2M cons _       -> buildHint cons
+    -- An ambiguousness error cannot happen for computed relationships TODO refactor so this mempty is not needed
+    hintList ComputedRelationship{} = mempty
 
 data PgError = PgError Authenticated SQL.UsageError
 type Authenticated = Bool

src/PostgREST/Query/QueryBuilder.hs

@@ -1,4 +1,5 @@
 {-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE NamedFieldPuns        #-}
 {-|
 Module      : PostgREST.Query.QueryBuilder
 Description : PostgREST SQL queries generating functions.
@@ -40,31 +41,34 @@ readRequestToQuery :: ReadRequest -> SQL.Snippet
 readRequestToQuery (Node (Select colSelects mainQi tblAlias logicForest joinConditions_ ordts range, (_, rel, _, _, _, _)) forest) =
   "SELECT " <>
   intercalateSnippet ", " ((pgFmtSelectItem qi <$> colSelects) ++ selects) <> " " <>
-  "FROM " <> SQL.sql tabl <> implicitJoinF rel <> " " <>
+  fromFrag <> " " <>
   intercalateSnippet " " joins <> " " <>
   (if null logicForest && null joinConditions_
     then mempty
     else "WHERE " <> intercalateSnippet " AND " (map (pgFmtLogicTree qi) logicForest ++ map pgFmtJoinCondition joinConditions_)) <> " " <>
   orderF qi ordts <> " " <>
   limitOffsetF range
   where
-    tabl = fromQi mainQi <> maybe mempty (\a -> " AS " <> pgFmtIdent a) tblAlias
-    qi = maybe mainQi (QualifiedIdentifier mempty) tblAlias
+    fromFrag = fromF rel mainQi tblAlias
+    qi = getQualifiedIdentifier rel mainQi tblAlias
     (selects, joins) = foldr getSelectsJoins ([],[]) forest
 
 getSelectsJoins :: ReadRequest -> ([SQL.Snippet], [SQL.Snippet]) -> ([SQL.Snippet], [SQL.Snippet])
 getSelectsJoins (Node (_, (_, Nothing, _, _, _, _)) _) _ = ([], [])
-getSelectsJoins rr@(Node (_, (name, Just Relationship{relCardinality=card,relTable=QualifiedIdentifier{qiName=table}}, alias, _, joinType, _)) _) (selects,joins) =
+getSelectsJoins rr@(Node (_, (name, Just rel, alias, _, joinType, _)) _) (selects,joins) =
   let
     subquery = readRequestToQuery rr
     aliasOrName = fromMaybe name alias
-    locTblName = table <> "_" <> aliasOrName
+    locTblName = qiName (relTable rel) <> "_" <> aliasOrName
     localTableName = pgFmtIdent locTblName
     internalTableName = pgFmtIdent $ "_" <> locTblName
     correlatedSubquery sub al cond =
       (if joinType == Just JTInner then "INNER" else "LEFT") <> " JOIN LATERAL ( " <> sub <> " ) AS " <> SQL.sql al <> " ON " <> cond
-    (sel, joi) = case card of
-      M2O _ _ ->
+    (sel, joi) = case rel of
+      Relationship{relCardinality=M2O _ _} ->
+        ( SQL.sql ("row_to_json(" <> localTableName <> ".*) AS " <> pgFmtIdent aliasOrName)
+        , correlatedSubquery subquery localTableName "TRUE")
+      ComputedRelationship{relToOne=True} ->
         ( SQL.sql ("row_to_json(" <> localTableName <> ".*) AS " <> pgFmtIdent aliasOrName)
         , correlatedSubquery subquery localTableName "TRUE")
       _ ->
@@ -219,7 +223,7 @@ requestToCallProcQuery (FunctionCall qi params args returnsScalar multipleCall r
 -- Only for the nodes that have an INNER JOIN linked to the root level.
 readRequestToCountQuery :: ReadRequest -> SQL.Snippet
 readRequestToCountQuery (Node (Select{from=mainQi, fromAlias=tblAlias, where_=logicForest, joinConditions=joinConditions_}, (_, rel, _, _, _, _)) forest) =
-  "SELECT 1 FROM " <> SQL.sql tabl <> implicitJoinF rel <>
+  "SELECT 1 " <> fromFrag <>
   (if null logicForest && null joinConditions_ && null subQueries
     then mempty
     else " WHERE " ) <>
@@ -229,8 +233,8 @@ readRequestToCountQuery (Node (Select{from=mainQi, fromAlias=tblAlias, where_=lo
     subQueries
   )
   where
-    qi = maybe mainQi (QualifiedIdentifier mempty) tblAlias
-    tabl = fromQi mainQi <> maybe mempty (\a -> " AS " <> pgFmtIdent a) tblAlias
+    qi = getQualifiedIdentifier rel mainQi tblAlias
+    fromFrag = fromF rel mainQi tblAlias
     subQueries = foldr existsSubquery [] forest
     existsSubquery :: ReadRequest -> [SQL.Snippet] -> [SQL.Snippet]
     existsSubquery readReq@(Node (_, (_, _, _, _, joinType, _)) _) rest =
@@ -241,7 +245,19 @@ readRequestToCountQuery (Node (Select{from=mainQi, fromAlias=tblAlias, where_=lo
 limitedQuery :: SQL.Snippet -> Maybe Integer -> SQL.Snippet
 limitedQuery query maxRows = query <> SQL.sql (maybe mempty (\x -> " LIMIT " <> BS.pack (show x)) maxRows)
 
-implicitJoinF :: Maybe Relationship -> SQL.Snippet
-implicitJoinF rel = case relCardinality <$> rel of
-  Just (M2M Junction{junTable=jt}) -> ", " <> SQL.sql (fromQi jt)
-  _                                -> mempty
+-- TODO refactor so this function is uneeded and ComputedRelationship QualifiedIdentifier comes from the ReadQuery type
+getQualifiedIdentifier :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> QualifiedIdentifier
+getQualifiedIdentifier rel mainQi tblAlias = case rel of
+  Just ComputedRelationship{relFunction} -> QualifiedIdentifier mempty $ fromMaybe (qiName relFunction) tblAlias
+  _                                      -> maybe mainQi (QualifiedIdentifier mempty) tblAlias
+
+-- FROM clause plus implicit joins
+fromF :: Maybe Relationship -> QualifiedIdentifier -> Maybe Alias -> SQL.Snippet
+fromF rel mainQi tblAlias = SQL.sql $ "FROM " <>
+  (case rel of
+    Just ComputedRelationship{relFunction,relTable} -> fromQi relFunction <> "(" <> pgFmtIdent (qiName relTable) <> ")"
+    _                                               -> fromQi mainQi) <>
+  maybe mempty (\a -> " AS " <> pgFmtIdent a) tblAlias <>
+  (case rel of
+    Just Relationship{relCardinality=M2M Junction{junTable=jt}} -> ", " <> fromQi jt
+    _                                                           -> mempty)

src/PostgREST/Request/DbRequestBuilder.hs

@@ -137,6 +137,7 @@ addRels schema allRels parentNode (Node (query@Select{from=tbl}, (nodeName, _, a
 -- applies aliasing to join conditions TODO refactor, this should go into the querybuilder module
 addJoinConditions :: Maybe Alias -> ReadRequest -> ReadRequest
 addJoinConditions _ (Node node@(Select{fromAlias=tblAlias}, (_, Nothing, _, _, _, _)) forest) = Node node (addJoinConditions tblAlias <$> forest)
+addJoinConditions _ (Node node@(Select{fromAlias=tblAlias}, (_, Just ComputedRelationship{}, _, _, _, _)) forest) = Node node (addJoinConditions tblAlias <$> forest)
 addJoinConditions previousAlias (Node (query@Select{fromAlias=tblAlias}, nodeProps@(_, Just (Relationship QualifiedIdentifier{qiSchema=tSchema, qiName=tN} QualifiedIdentifier{qiName=ftN} _ card _ _), _, _, _, _)) forest) =
   Node (query{joinConditions=joinConds}, nodeProps) (addJoinConditions tblAlias <$> forest)
   where
@@ -188,8 +189,9 @@ findRel schema allRels origin target hint =
     isO2M card = case card of
       O2M _ _ -> True
       _       -> False
-    rels = filter (
-      \Relationship{..} ->
+    rels = filter (\case
+      ComputedRelationship{relFunction} -> target == qiName relFunction
+      Relationship{..} ->
         -- In a self-relationship we have a single foreign key but two relationships with different cardinalities: M2O/O2M. For disambiguation, we use the convention of getting:
         -- TODO: handle one-to-one and many-to-many self-relationships
         if relIsSelf
@@ -365,14 +367,21 @@ returningCols rr@(Node _ forest) pkCols
         Node (_, (_, Just Relationship{relCardinality=M2M Junction{junColumns1, junColumns2}}, _, _, _, _)) _ -> Just $ (fst <$> junColumns1) ++ (fst <$> junColumns2)
         _                                                        -> Nothing
       ) forest
+    hasComputedRel = isJust $ find (\case
+      Node (_, (_, Just ComputedRelationship{}, _, _, _, _)) _ -> True
+      _ -> False
+      ) forest
     -- However if the "client_id" is present, e.g. mutateRequest to
     -- /projects?select=client_id,name,clients(name) we would get `RETURNING
     -- client_id, name, client_id` and then we would produce the "column
     -- reference \"client_id\" is ambiguous" error from PostgreSQL. So we
     -- deduplicate with Set: We are adding the primary key columns as well to
     -- make sure, that a proper location header can always be built for
     -- INSERT/POST
-    returnings = S.toList . S.fromList $ fldNames ++ fkCols ++ pkCols
+    returnings =
+      if not hasComputedRel
+        then S.toList . S.fromList $ fldNames ++ fkCols ++ pkCols
+        else ["*"] -- on computed relationships we cannot know the required columns for an embedding to succeed, so we just return all
 
 -- Traditional filters(e.g. id=eq.1) are added as root nodes of the LogicTree
 -- they are later concatenated with AND in the QueryBuilder