feat: add db-extra-operators config

Allows adding custom operators

CHANGELOG.md

@@ -14,6 +14,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
    + Allows including the join table columns when resource embedding
    + Allows disambiguating a recursive m2m embed
    + Allows disambiguating an embed that has a many-to-many relationship using two foreign keys on a junction
+ - #2028, Allow adding custom operators with the `db-extra-operators` config - @steve-chavez
+   + The format is a comma separated list of `<alias>:<operator`, e.g. `db-extra-operators = "fuzzy:%, ltreefts:@"`
 
 ### Fixed
 

postgrest.cabal

@@ -197,6 +197,7 @@ test-suite spec
                       Feature.Query.DeleteSpec
                       Feature.Query.EmbedDisambiguationSpec
                       Feature.Query.EmbedInnerJoinSpec
+                      Feature.Query.ExtraOperatorsSpec
                       Feature.Query.PlanSpec
                       Feature.Query.HtmlRawOutputSpec
                       Feature.Query.InsertSpec
@@ -245,6 +246,7 @@ test-suite spec
                     , regex-tdfa        >= 1.2.2 && < 1.4
                     , text              >= 1.2.2 && < 1.3
                     , transformers-base >= 0.4.4 && < 0.5
+                    , unordered-containers >= 0.2.8 && < 0.3
                     , wai               >= 3.2.1 && < 3.3
                     , wai-extra         >= 3.0.19 && < 3.2
   ghc-options:        -O0 -Werror -Wall -fwarn-identities

src/PostgREST/ApiRequest.hs

@@ -182,7 +182,7 @@ data ApiRequest = ApiRequest {
 -- | Examines HTTP request and translates it into user intent.
 userApiRequest :: AppConfig -> SchemaCache -> Request -> RequestBody -> Either ApiRequestError ApiRequest
 userApiRequest conf sCache req reqBody = do
-  qPrms <- first QueryParamError $ QueryParams.parse $ rawQueryString req
+  qPrms <- first QueryParamError $ QueryParams.parse (configDbExtraOperators conf) $ rawQueryString req
   pInfo <- getPathInfo conf $ pathInfo req
   act <- getAction pInfo $ requestMethod req
   mediaTypes <- getMediaTypes conf (requestHeaders req) act pInfo

src/PostgREST/ApiRequest/QueryParams.hs

@@ -39,6 +39,7 @@ import Text.ParserCombinators.Parsec (GenParser, ParseError, Parser,
                                       optionMaybe, sepBy1, string,
                                       try, (<?>))
 
+import PostgREST.Config                  (ExtraOperators)
 import PostgREST.RangeQuery              (NonnegRange, allRange,
                                           rangeGeq, rangeLimit,
                                           rangeOffset, restrictRange)
@@ -112,36 +113,36 @@ data QueryParams =
 --
 -- The canonical representation of the query string has parameters sorted alphabetically:
 --
--- >>> qsCanonical <$> parse "a=1&c=3&b=2&d"
+-- >>> qsCanonical <$> parse HM.empty "a=1&c=3&b=2&d"
 -- Right "a=1&b=2&c=3&d="
 --
 -- 'select' is a reserved parameter that selects the fields to be returned:
 --
--- >>> qsSelect <$> parse "select=name,location"
+-- >>> qsSelect <$> parse HM.empty "select=name,location"
 -- Right [Node {rootLabel = SelectField {selField = ("name",[]), selCast = Nothing, selAlias = Nothing}, subForest = []},Node {rootLabel = SelectField {selField = ("location",[]), selCast = Nothing, selAlias = Nothing}, subForest = []}]
 --
 -- Filters are parameters whose value contains an operator, separated by a '.' from its value:
 --
--- >>> qsFilters <$> parse "a.b=eq.0"
+-- >>> qsFilters <$> parse HM.empty "a.b=eq.0"
 -- Right [(["a"],Filter {field = ("b",[]), opExpr = OpExpr False (Op OpEqual "0")})]
 --
 -- If the operator specified in a filter does not exist, parsing the query string fails:
 --
--- >>> qsFilters <$> parse "a.b=noop.0"
+-- >>> qsFilters <$> parse HM.empty "a.b=noop.0"
 -- Left (QPError "\"failed to parse filter (noop.0)\" (line 1, column 6)" "unknown single value operator noop")
-parse :: ByteString -> Either QPError QueryParams
-parse qs =
+parse :: ExtraOperators -> ByteString -> Either QPError QueryParams
+parse extraOperators qs =
   QueryParams
     canonical
     params
     ranges
     <$> pRequestOrder `traverse` order
-    <*> pRequestLogicTree `traverse` logic
+    <*> pRequestLogicTree extraOperators `traverse` logic
     <*> pRequestColumns columns
     <*> pRequestSelect select
-    <*> pRequestFilter `traverse` filters
-    <*> (fmap snd <$> (pRequestFilter `traverse` filtersRoot))
-    <*> pRequestFilter `traverse` filtersNotRoot
+    <*> pRequestFilter extraOperators `traverse` filters
+    <*> (fmap snd <$> (pRequestFilter extraOperators `traverse` filtersRoot))
+    <*> pRequestFilter extraOperators `traverse` filtersNotRoot
     <*> pure (S.fromList (fst <$> filters))
     <*> sequenceA (pRequestOnConflict <$> onConflict)
   where
@@ -188,7 +189,7 @@ parse qs =
         "not" : _ : _ -> True
         "is" : _      -> True
         "in" : _      -> True
-        x : _         -> isJust (operator x) || isJust (ftsOperator x)
+        x : _         -> isJust (operator extraOperators x) || isJust (ftsOperator x)
         _             -> False
 
     hasFtsOperator val =
@@ -213,8 +214,8 @@ parse qs =
         offsetParams =
           HM.fromList [(k, maybe allRange rangeGeq (readMaybe v)) | (k,v) <- offsets]
 
-operator :: Text -> Maybe SimpleOperator
-operator = \case
+operator :: ExtraOperators -> Text -> Maybe SimpleOperator
+operator extraOperators = \case
   "eq"     -> Just OpEqual
   "gte"    -> Just OpGreaterThanEqual
   "gt"     -> Just OpGreaterThan
@@ -233,7 +234,7 @@ operator = \case
   "adj"    -> Just OpAdjacent
   "match"  -> Just OpMatch
   "imatch" -> Just OpIMatch
-  _        -> Nothing
+  other    -> OpCustom <$> HM.lookup other extraOperators
 
 ftsOperator :: Text -> Maybe FtsOperator
 ftsOperator = \case
@@ -244,9 +245,6 @@ ftsOperator = \case
   _       -> Nothing
 
 
--- PARSERS
-
-
 pRequestSelect :: Text -> Either QPError [Tree SelectItem]
 pRequestSelect selStr =
   mapError $ P.parse pFieldForest ("failed to parse select parameter (" <> toS selStr <> ")") (toS selStr)
@@ -255,11 +253,11 @@ pRequestOnConflict :: Text -> Either QPError [FieldName]
 pRequestOnConflict oncStr =
   mapError $ P.parse pColumns ("failed to parse on_conflict parameter (" <> toS oncStr <> ")") (toS oncStr)
 
-pRequestFilter :: (Text, Text) -> Either QPError (EmbedPath, Filter)
-pRequestFilter (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
+pRequestFilter :: ExtraOperators -> (Text, Text) -> Either QPError (EmbedPath, Filter)
+pRequestFilter extraOperators (k, v) = mapError $ (,) <$> path <*> (Filter <$> fld <*> oper)
   where
     treePath = P.parse pTreePath ("failed to parse tree path (" ++ toS k ++ ")") $ toS k
-    oper = P.parse (pOpExpr pSingleVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
+    oper = P.parse (pOpExpr extraOperators pSingleVal) ("failed to parse filter (" ++ toS v ++ ")") $ toS v
     path = fst <$> treePath
     fld = snd <$> treePath
 
@@ -276,16 +274,16 @@ pRequestRange (k, v) = mapError $ (,) <$> path <*> pure v
     treePath = P.parse pTreePath ("failed to parse tree path (" ++ toS k ++ ")") $ toS k
     path = fst <$> treePath
 
-pRequestLogicTree :: (Text, Text) -> Either QPError (EmbedPath, LogicTree)
-pRequestLogicTree (k, v) = mapError $ (,) <$> embedPath <*> logicTree
+pRequestLogicTree :: ExtraOperators -> (Text, Text) -> Either QPError (EmbedPath, LogicTree)
+pRequestLogicTree extraOperators (k, v) = mapError $ (,) <$> embedPath <*> logicTree
   where
     path = P.parse pLogicPath ("failed to parse logic path (" ++ toS k ++ ")") $ toS k
     embedPath = fst <$> path
     logicTree = do
       op <- snd <$> path
       -- Concat op and v to make pLogicTree argument regular,
       -- in the form of "?and=and(.. , ..)" instead of "?and=(.. , ..)"
-      P.parse pLogicTree ("failed to parse logic tree (" ++ toS v ++ ")") $ toS (op <> v)
+      P.parse (pLogicTree extraOperators) ("failed to parse logic tree (" ++ toS v ++ ")") $ toS (op <> v)
 
 pRequestColumns :: Maybe Text -> Either QPError (Maybe (S.Set FieldName))
 pRequestColumns colStr =
@@ -570,11 +568,11 @@ pEmbedParams = do
 -- |
 -- Parse operator expression used in horizontal filtering
 --
--- >>> P.parse (pOpExpr pSingleVal) "" "fts().value"
+-- >>> P.parse (pOpExpr HM.empty pSingleVal) "" "fts().value"
 -- Left (line 1, column 7):
 -- unknown single value operator fts()
-pOpExpr :: Parser SingleVal -> Parser OpExpr
-pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
+pOpExpr :: ExtraOperators -> Parser SingleVal -> Parser OpExpr
+pOpExpr extraOperators pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)) <|> OpExpr False <$> pOperation
   where
     pOperation :: Parser Operation
     pOperation = pIn <|> pIs <|> try pFts <|> pOp <?> "operator (eq, gt, ...)"
@@ -584,7 +582,7 @@ pOpExpr pSVal = try ( string "not" *> pDelimiter *> (OpExpr True <$> pOperation)
 
     pOp = do
       opStr <- try (P.manyTill anyChar (try pDelimiter))
-      op <- parseMaybe ("unknown single value operator " <> opStr) . operator $ toS opStr
+      op <- parseMaybe ("unknown single value operator " <> opStr) . operator extraOperators $ toS opStr
       Op op <$> pSVal
 
     pTriVal = try (ciString "null"    $> TriNull)
@@ -672,12 +670,13 @@ pOrder = lexeme (try pOrderRelationTerm <|> pOrderTerm) `sepBy1` char ','
 
     pEnd = try (void $ lookAhead (char ',')) <|> try eof
 
-pLogicTree :: Parser LogicTree
-pLogicTree = Stmnt <$> try pLogicFilter
-             <|> Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree `sepBy1` lexeme (char ',') <* lexeme (char ')'))
+pLogicTree :: ExtraOperators -> Parser LogicTree
+pLogicTree extraOperators =
+  Stmnt <$> try pLogicFilter <|>
+  Expr <$> pNot <*> pLogicOp <*> (lexeme (char '(') *> pLogicTree extraOperators `sepBy1` lexeme (char ',') <* lexeme (char ')'))
   where
     pLogicFilter :: Parser Filter
-    pLogicFilter = Filter <$> pField <* pDelimiter <*> pOpExpr pLogicSingleVal
+    pLogicFilter = Filter <$> pField <* pDelimiter <*> pOpExpr extraOperators pLogicSingleVal
     pNot :: Parser Bool
     pNot = try (string "not" *> pDelimiter $> True)
            <|> pure False

src/PostgREST/ApiRequest/Types.hs

@@ -223,6 +223,7 @@ data SimpleOperator
   | OpAdjacent
   | OpMatch
   | OpIMatch
+  | OpCustom ByteString
   deriving Eq
 
 -- | Operators for full text search operators

src/PostgREST/Config.hs

@@ -24,6 +24,7 @@ module PostgREST.Config
   , readPGRSTEnvironment
   , toURI
   , parseSecret
+  , ExtraOperators
   ) where
 
 import qualified Crypto.JOSE.Types      as JOSE
@@ -33,6 +34,7 @@ import qualified Data.ByteString        as BS
 import qualified Data.ByteString.Base64 as B64
 import qualified Data.ByteString.Lazy   as LBS
 import qualified Data.Configurator      as C
+import qualified Data.HashMap.Strict    as HM
 import qualified Data.Map.Strict        as M
 import qualified Data.Text              as T
 import qualified Data.Text.Encoding     as T
@@ -96,8 +98,11 @@ data AppConfig = AppConfig
   , configServerUnixSocket         :: Maybe FilePath
   , configServerUnixSocketMode     :: FileMode
   , configAdminServerPort          :: Maybe Int
+  , configDbExtraOperators         :: ExtraOperators
   }
 
+type ExtraOperators = HM.HashMap Text ByteString
+
 data LogLevel = LogCrit | LogError | LogWarn | LogInfo
 
 dumpLogLevel :: LogLevel -> Text
@@ -126,6 +131,7 @@ toText conf =
       [("db-anon-role",              q . fromMaybe "" . configDbAnonRole)
       ,("db-channel",                q . configDbChannel)
       ,("db-channel-enabled",            T.toLower . show . configDbChannelEnabled)
+      ,("db-extra-operators",        q . showExtraOperators . configDbExtraOperators)
       ,("db-extra-search-path",      q . T.intercalate "," . configDbExtraSearchPath)
       ,("db-max-rows",                   maybe "\"\"" show . configDbMaxRows)
       ,("db-plan-enabled",               T.toLower . show . configDbPlanEnabled)
@@ -172,6 +178,7 @@ toText conf =
       where
         secret = fromMaybe mempty $ configJwtSecret c
     showSocketMode c = showOct (configServerUnixSocketMode c) mempty
+    showExtraOperators m = T.intercalate "," $ (\(k, v) -> k <> ":" <> decodeUtf8 v) <$> HM.toList m
 
 -- This class is needed for the polymorphism of overrideFromDbOrEnvironment
 -- because C.required and C.optional have different signatures
@@ -250,6 +257,7 @@ parser optPath env dbSettings =
     <*> (fmap T.unpack <$> optString "server-unix-socket")
     <*> parseSocketFileMode "server-unix-socket-mode"
     <*> optInt "admin-server-port"
+    <*> parseExtraOperators "db-extra-operators"
   where
     parseAppSettings :: C.Key -> C.Parser C.Config [(Text, Text)]
     parseAppSettings key = addFromEnv . fmap (fmap coerceText) <$> C.subassocs key C.value
@@ -322,6 +330,12 @@ parser optPath env dbSettings =
         Nothing  -> pure [JSPKey "role"]
         Just rck -> either (fail . show) pure $ pRoleClaimKey rck
 
+    parseExtraOperators :: C.Key -> C.Parser C.Config ExtraOperators
+    parseExtraOperators k =
+      optString k >>= \case
+        Nothing  -> pure HM.empty
+        Just val -> pure $ HM.fromList $ (\(x, y) -> (T.strip x, T.encodeUtf8 $ T.strip $ T.drop 1 y)) <$> (T.breakOn ":" . T.strip <$> T.splitOn "," val)
+
     optWithAlias :: C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a) -> C.Parser C.Config (Maybe a)
     optWithAlias orig alias =
       orig >>= \case

src/PostgREST/Query/SqlFragment.hs

@@ -111,6 +111,7 @@ singleValOperator = \case
   OpAdjacent         -> "-|-"
   OpMatch            -> "~"
   OpIMatch           -> "~*"
+  OpCustom op        -> op
 
 ftsOperator :: FtsOperator -> SqlFragment
 ftsOperator = \case

test/io/configs/expected/aliases.config

@@ -1,6 +1,7 @@
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
+db-extra-operators = ""
 db-extra-search-path = "public"
 db-max-rows = 1000
 db-plan-enabled = false

test/io/configs/expected/boolean-numeric.config

@@ -1,6 +1,7 @@
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
+db-extra-operators = ""
 db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false

test/io/configs/expected/boolean-string.config

@@ -1,6 +1,7 @@
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
+db-extra-operators = ""
 db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false

test/io/configs/expected/defaults.config

@@ -1,6 +1,7 @@
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
+db-extra-operators = ""
 db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false

test/io/configs/expected/no-defaults-with-db-other-authenticator.config

@@ -1,6 +1,7 @@
 db-anon-role = "other"
 db-channel = "postgrest"
 db-channel-enabled = false
+db-extra-operators = "fuzzy:%,ltreefts:@"
 db-extra-search-path = "public,extensions,other"
 db-max-rows = 100
 db-plan-enabled = true

test/io/configs/expected/no-defaults-with-db.config

@@ -1,6 +1,7 @@
 db-anon-role = "anonymous"
 db-channel = "postgrest"
 db-channel-enabled = false
+db-extra-operators = "fuzzy:%,ltreefts:@"
 db-extra-search-path = "public,extensions,private"
 db-max-rows = 1000
 db-plan-enabled = true

test/io/configs/expected/no-defaults.config

@@ -1,6 +1,7 @@
 db-anon-role = "root"
 db-channel = "postgrest"
 db-channel-enabled = false
+db-extra-operators = "fuzzy:%"
 db-extra-search-path = "public,test"
 db-max-rows = 1000
 db-plan-enabled = true

test/io/configs/expected/types.config

@@ -1,6 +1,7 @@
 db-anon-role = ""
 db-channel = "pgrst"
 db-channel-enabled = true
+db-extra-operators = ""
 db-extra-search-path = "public"
 db-max-rows = ""
 db-plan-enabled = false

test/io/configs/no-defaults-env.yaml

@@ -3,6 +3,7 @@ PGRST_APP_SETTINGS_test: test
 PGRST_DB_ANON_ROLE: root
 PGRST_DB_CHANNEL: postgrest
 PGRST_DB_CHANNEL_ENABLED: false
+PGRST_DB_EXTRA_OPERATORS: 'fuzzy:%'
 PGRST_DB_EXTRA_SEARCH_PATH: public, test
 PGRST_DB_MAX_ROWS: 1000
 PGRST_DB_PLAN_ENABLED: true

test/io/configs/no-defaults.config

@@ -1,6 +1,7 @@
 db-anon-role = "root"
 db-channel = "postgrest"
 db-channel-enabled = false
+db-extra-operators = "fuzzy:%"
 db-extra-search-path = "public, test"
 db-max-rows = 1000
 db-plan-enabled = true

test/io/db_config.sql

@@ -17,6 +17,7 @@ ALTER ROLE db_config_authenticator SET pgrst.db_pre_request = 'test.custom_heade
 ALTER ROLE db_config_authenticator SET pgrst.db_max_rows = '1000';
 ALTER ROLE db_config_authenticator SET pgrst.db_extra_search_path = 'public, extensions';
 ALTER ROLE db_config_authenticator SET pgrst.not_existing = 'should be ignored';
+ALTER ROLE db_config_authenticator SET pgrst.db_extra_operators = 'fuzzy:%, ltreefts:@';
 
 -- override with database specific setting
 ALTER ROLE db_config_authenticator IN DATABASE :DBNAME SET pgrst.jwt_secret = 'OVERRIDE=REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
@@ -60,6 +61,7 @@ ALTER ROLE other_authenticator SET pgrst.db_max_rows = '100';
 ALTER ROLE other_authenticator SET pgrst.db_extra_search_path = 'public, extensions, other';
 ALTER ROLE other_authenticator SET pgrst.openapi_mode = 'disabled';
 ALTER ROLE other_authenticator SET pgrst.openapi_security_active = 'false';
+ALTER ROLE other_authenticator SET pgrst.db_extra_operators = 'ltreefts:@, fuzzy:%';
 
 -- authenticator used for tests that manipulate statement timeout
 CREATE ROLE timeout_authenticator LOGIN NOINHERIT;