From 3d4e6194de6d06dd1f8a527f1bdebe2b5c99d2c8 Mon Sep 17 00:00:00 2001 From: Hector Ayala Date: Sun, 18 Feb 2024 02:56:51 -0400 Subject: [PATCH] Handle array type custom decoders (#470) * feat: add logic to handle array types for custom decoders * chore: update docs * chore: fix format * chore: bump version, fix type name * chore: update test readme * chore: format readme --- connection/connection_params.ts | 12 +++++- deno.json | 2 +- docs/README.md | 36 ++++++++++++++-- query/array_parser.ts | 10 +++++ query/decode.ts | 24 +++++++++-- tests/README.md | 10 +++-- tests/query_client_test.ts | 73 +++++++++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 13 deletions(-) diff --git a/connection/connection_params.ts b/connection/connection_params.ts index 7b68ea9c..ac4f650e 100644 --- a/connection/connection_params.ts +++ b/connection/connection_params.ts @@ -3,6 +3,7 @@ import { ConnectionParamsError } from "../client/error.ts"; import { fromFileUrl, isAbsolute } from "../deps.ts"; import { OidType } from "../query/oid.ts"; import { DebugControls } from "../debug.ts"; +import { ParseArrayFunction } from "../query/array_parser.ts"; /** * The connection string must match the following URI structure. All parameters but database and user are optional @@ -108,9 +109,16 @@ export type Decoders = { /** * A decoder function that takes a string value and returns a parsed value of some type. - * the Oid is also passed to the function for reference + * + * @param value The string value to parse + * @param oid The OID of the column type the value is from + * @param parseArray A helper function that parses SQL array-formatted strings and parses each array value using a transform function. */ -export type DecoderFunction = (value: string, oid: number) => unknown; +export type DecoderFunction = ( + value: string, + oid: number, + parseArray: ParseArrayFunction, +) => unknown; /** * Control the behavior for the client instance diff --git a/deno.json b/deno.json index 51a2bcf8..a95580a3 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "lock": false, "name": "@bartlomieju/postgres", - "version": "0.19.0", + "version": "0.19.1", "exports": "./mod.ts" } diff --git a/docs/README.md b/docs/README.md index 477b86f4..c4763079 100644 --- a/docs/README.md +++ b/docs/README.md @@ -758,10 +758,10 @@ available: You can also provide custom decoders to the client that will be used to decode the result data. This can be done by setting the `decoders` controls option in the client configuration. This option is a map object where the keys are the -type names or Oid numbers and the values are the custom decoder functions. +type names or OID numbers and the values are the custom decoder functions. You can use it with the decode strategy. Custom decoders take precedence over -the strategy and internal parsers. +the strategy and internal decoders. ```ts { @@ -785,7 +785,37 @@ the strategy and internal parsers. const result = await client.queryObject( "SELECT ID, NAME, IS_ACTIVE FROM PEOPLE", ); - console.log(result.rows[0]); // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} + console.log(result.rows[0]); + // {id: '1', name: 'Javier', is_active: { value: false, type: "boolean"}} +} +``` + +The driver takes care of parsing the related `array` OID types automatically. +For example, if a custom decoder is defined for the `int4` type, it will be +applied when parsing `int4[]` arrays. If needed, you can have separate custom +decoders for the array and non-array types by defining another custom decoders +for the array type itself. + +```ts +{ + const client = new Client({ + database: "some_db", + user: "some_user", + controls: { + decodeStrategy: "string", + decoders: { + // Custom decoder for int4 (OID 23 = int4) + // convert to int and multiply by 100 + 23: (value: string) => parseInt(value, 10) * 100, + }, + }, + }); + + const result = await client.queryObject( + "SELECT ARRAY[ 2, 2, 3, 1 ] AS scores, 8 final_score;", + ); + console.log(result.rows[0]); + // { scores: [ 200, 200, 300, 100 ], final_score: 800 } } ``` diff --git a/query/array_parser.ts b/query/array_parser.ts index 9fd043bd..b7983b41 100644 --- a/query/array_parser.ts +++ b/query/array_parser.ts @@ -6,6 +6,16 @@ type AllowedSeparators = "," | ";"; type ArrayResult = Array>; type Transformer = (value: string) => T; +export type ParseArrayFunction = typeof parseArray; + +/** + * Parse a string into an array of values using the provided transform function. + * + * @param source The string to parse + * @param transform A function to transform each value in the array + * @param separator The separator used to split the string into values + * @returns + */ export function parseArray( source: string, transform: Transformer, diff --git a/query/decode.ts b/query/decode.ts index c2b5ec42..fb13afa3 100644 --- a/query/decode.ts +++ b/query/decode.ts @@ -1,4 +1,4 @@ -import { Oid, OidTypes, OidValue } from "./oid.ts"; +import { Oid, OidType, OidTypes, OidValue } from "./oid.ts"; import { bold, yellow } from "../deps.ts"; import { decodeBigint, @@ -36,6 +36,7 @@ import { decodeTidArray, } from "./decoders.ts"; import { ClientControls } from "../connection/connection_params.ts"; +import { parseArray } from "./array_parser.ts"; export class Column { constructor( @@ -216,12 +217,29 @@ export function decode( // check if there is a custom decoder if (controls?.decoders) { + const oidType = OidTypes[column.typeOid as OidValue]; // check if there is a custom decoder by oid (number) or by type name (string) const decoderFunc = controls.decoders?.[column.typeOid] || - controls.decoders?.[OidTypes[column.typeOid as OidValue]]; + controls.decoders?.[oidType]; if (decoderFunc) { - return decoderFunc(strValue, column.typeOid); + return decoderFunc(strValue, column.typeOid, parseArray); + } // if no custom decoder is found and the oid is for an array type, check if there is + // a decoder for the base type and use that with the array parser + else if (oidType.includes("_array")) { + const baseOidType = oidType.replace("_array", "") as OidType; + // check if the base type is in the Oid object + if (baseOidType in Oid) { + // check if there is a custom decoder for the base type by oid (number) or by type name (string) + const decoderFunc = controls.decoders?.[Oid[baseOidType]] || + controls.decoders?.[baseOidType]; + if (decoderFunc) { + return parseArray( + strValue, + (value: string) => decoderFunc(value, column.typeOid, parseArray), + ); + } + } } } diff --git a/tests/README.md b/tests/README.md index c17f1a58..c8c3e4e9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,11 @@ # Testing -To run tests, first prepare your configuration file by copying +To run tests, we recommend using Docker. With Docker, there is no need to modify +any configuration, just run the build and test commands. + +If running tests on your host, prepare your configuration file by copying `config.example.json` into `config.json` and updating it appropriately based on -your environment. If you use the Docker based configuration below there's no -need to modify the configuration. +your environment. ## Running the Tests @@ -23,7 +25,7 @@ docker-compose run tests If you have Docker installed then you can run the following to set up a running container that is compatible with the tests: -``` +```sh docker run --rm --env POSTGRES_USER=test --env POSTGRES_PASSWORD=test \ --env POSTGRES_DB=deno_postgres -p 5432:5432 postgres:12-alpine ``` diff --git a/tests/query_client_test.ts b/tests/query_client_test.ts index 0e71da69..c096049a 100644 --- a/tests/query_client_test.ts +++ b/tests/query_client_test.ts @@ -241,6 +241,79 @@ Deno.test( ), ); +Deno.test( + "Custom decoders with arrays", + withClient( + async (client) => { + const result = await client.queryObject( + `SELECT + ARRAY[true, false, true] AS _bool_array, + ARRAY['2024-01-01'::date, '2024-01-02'::date, '2024-01-03'::date] AS _date_array, + ARRAY[1.5:: REAL, 2.5::REAL, 3.5::REAL] AS _float_array, + ARRAY[10, 20, 30] AS _int_array, + ARRAY[ + '{"key1": "value1", "key2": "value2"}'::jsonb, + '{"key3": "value3", "key4": "value4"}'::jsonb, + '{"key5": "value5", "key6": "value6"}'::jsonb + ] AS _jsonb_array, + ARRAY['string1', 'string2', 'string3'] AS _text_array + ;`, + ); + + assertEquals(result.rows, [ + { + _bool_array: [ + { boolean: true }, + { boolean: false }, + { boolean: true }, + ], + _date_array: [ + new Date("2024-01-11T00:00:00.000Z"), + new Date("2024-01-12T00:00:00.000Z"), + new Date("2024-01-13T00:00:00.000Z"), + ], + _float_array: [15, 25, 35], + _int_array: [110, 120, 130], + _jsonb_array: [ + { key1: "value1", key2: "value2" }, + { key3: "value3", key4: "value4" }, + { key5: "value5", key6: "value6" }, + ], + _text_array: ["string1_!", "string2_!", "string3_!"], + }, + ]); + }, + { + controls: { + decoders: { + // convert to object + [Oid.bool]: (value: string) => ({ boolean: value === "t" }), + // 1082 = date : convert to date and add 10 days + "1082": (value: string) => { + const d = new Date(value); + return new Date(d.setDate(d.getDate() + 10)); + }, + // multiply by 20, should not be used! + float4: (value: string) => parseFloat(value) * 20, + // multiply by 10 + float4_array: (value: string, _, parseArray) => + parseArray(value, (v) => parseFloat(v) * 10), + // return 0, should not be used! + [Oid.int4]: () => 0, + // add 100 + [Oid.int4_array]: (value: string, _, parseArray) => + parseArray(value, (v) => parseInt(v, 10) + 100), + // split string and reverse, should not be used! + [Oid.text]: (value: string) => value.split("").reverse(), + // 1009 = text_array : append "_!" to each string + 1009: (value: string, _, parseArray) => + parseArray(value, (v) => `${v}_!`), + }, + }, + }, + ), +); + Deno.test( "Custom decoder precedence", withClient(