Finalize Koios v1.1.0 (#250)

## Description
<!--- Describe your changes -->
- Make use of asset-txo-cache for top assets on mainnet, and use this
cache for `asset_addresses` and `policy_asset_addresses`
- Add v0 RPC redirectors to keep serve v0 endpoints from v1
- Convert few simple RPC functions from PLPGSQL to SQL language to help
with inline filtering
- Address linting results from SQLFluff
- Move db-scripts from guild-operators repository to koios-artifacts
repository
- Drop stale db-scripts/genesis_table.sql file
- Add 3 additional indexes for collateral and reference inputs based on
query times
- Add top 3 assets for preview/preprod to asset-txo-cache
- Bump schema version for koios-1.1.0

.deepsource.toml

@@ -0,0 +1,15 @@
+version = 1
+
+test_patterns = ["tests/**"]
+
+[[analyzers]]
+name = "shell"
+
+[[analyzers]]
+name = "python"
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"
+
+[[analyzers]]
+name = "sql"

files/grest/.sqlfluff

@@ -2,7 +2,6 @@
 dialect = postgres
 exclude_rules = structure.column_order, references.keywords
 max_line_length=260
-recurse = 0
 capitalisation_policy = upper
 extended_capitalisation_policy = upper
 idented_joins = True

files/grest/cron/jobs/asset-txo-cache-update.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+DB_NAME=cexplorer
+
+tip=$(psql ${DB_NAME} -qbt -c "SELECT EXTRACT(epoch FROM time)::integer FROM block ORDER BY id DESC LIMIT 1;" | xargs)
+
+if [[ $(( $(date +%s) - tip )) -gt 300 ]]; then
+  echo "$(date +%F_%H:%M:%S) Skipping as database has not received a new block in past 300 seconds!" && exit 1
+fi
+
+echo "$(date +%F_%H:%M:%S) Running asset txo cache update..."
+psql ${DB_NAME} -qbt -c "SELECT grest.asset_txo_cache_update();" 1>/dev/null 2>&1
+echo "$(date +%F_%H:%M:%S) Job done!"

files/grest/rpc/00_blockchain/genesis.sql

@@ -13,10 +13,8 @@ RETURNS TABLE (
   securityparam varchar,
   alonzogenesis varchar
 )
-LANGUAGE plpgsql
+LANGUAGE sql STABLE
 AS $$
-BEGIN
-  RETURN QUERY
   SELECT
     g.networkmagic,
     g.networkid,
@@ -30,7 +28,5 @@ BEGIN
     g.maxkesrevolutions,
     g.securityparam,
     g.alonzogenesis
-  FROM
-    grest.genesis AS g;
-END;
+  FROM grest.genesis AS g;
 $$;

files/grest/rpc/00_blockchain/param_updates.sql

@@ -6,52 +6,48 @@ RETURNS TABLE (
   epoch_no word31type,
   data jsonb
 )
-LANGUAGE plpgsql
+LANGUAGE sql STABLE
 AS $$
-BEGIN
-  RETURN QUERY
-    SELECT DISTINCT ON (pp.registered_tx_id)
-      ENCODE(t.hash,'hex') AS tx_hash,
-      b.block_no AS block_height,
-      EXTRACT(EPOCH FROM b.time)::integer AS block_time,
-      b.epoch_no,
-      JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT(
-          'min_fee_a', pp.min_fee_a,
-          'min_fee_b', pp.min_fee_b,
-          'max_block_size', pp.max_block_size,
-          'max_tx_size', pp.max_tx_size,
-          'max_bh_size', pp.max_bh_size,
-          'key_deposit', pp.key_deposit,
-          'pool_deposit', pp.pool_deposit,
-          'max_epoch', pp.max_epoch,
-          'optimal_pool_count', pp.optimal_pool_count,
-          'influence', pp.influence,
-          'monetary_expand_rate', pp.monetary_expand_rate,
-          'treasury_growth_rate', pp.treasury_growth_rate,
-          'decentralisation', pp.decentralisation,
-          'entropy', pp.entropy,
-          'protocol_major', pp.protocol_major,
-          'protocol_minor', pp.protocol_minor,
-          'min_utxo_value', pp.min_utxo_value,
-          'min_pool_cost', pp.min_pool_cost,
-          'cost_model', CM.costs,
-          'price_mem', pp.price_mem,
-          'price_step', pp.price_step,
-          'max_tx_ex_mem', pp.max_tx_ex_mem,
-          'max_tx_ex_steps', pp.max_tx_ex_steps,
-          'max_block_ex_mem', pp.max_block_ex_mem,
-          'max_block_ex_steps', pp.max_block_ex_steps,
-          'max_val_size', pp.max_val_size,
-          'collateral_percent', pp.collateral_percent,
-          'max_collateral_inputs', pp.max_collateral_inputs,
-          'coins_per_utxo_size', pp.coins_per_utxo_size
-        )) AS data
-    FROM
-      public.param_proposal pp
-      INNER JOIN tx t ON t.id = pp.registered_tx_id
-      INNER JOIN block b ON t.block_id = b.id
-      LEFT JOIN cost_model CM ON CM.id = pp.cost_model_id;
-END;
+  SELECT DISTINCT ON (pp.registered_tx_id)
+    ENCODE(t.hash,'hex') AS tx_hash,
+    b.block_no AS block_height,
+    EXTRACT(EPOCH FROM b.time)::integer AS block_time,
+    b.epoch_no,
+    JSONB_STRIP_NULLS(JSONB_BUILD_OBJECT(
+        'min_fee_a', pp.min_fee_a,
+        'min_fee_b', pp.min_fee_b,
+        'max_block_size', pp.max_block_size,
+        'max_tx_size', pp.max_tx_size,
+        'max_bh_size', pp.max_bh_size,
+        'key_deposit', pp.key_deposit,
+        'pool_deposit', pp.pool_deposit,
+        'max_epoch', pp.max_epoch,
+        'optimal_pool_count', pp.optimal_pool_count,
+        'influence', pp.influence,
+        'monetary_expand_rate', pp.monetary_expand_rate,
+        'treasury_growth_rate', pp.treasury_growth_rate,
+        'decentralisation', pp.decentralisation,
+        'entropy', pp.entropy,
+        'protocol_major', pp.protocol_major,
+        'protocol_minor', pp.protocol_minor,
+        'min_utxo_value', pp.min_utxo_value,
+        'min_pool_cost', pp.min_pool_cost,
+        'cost_model', CM.costs,
+        'price_mem', pp.price_mem,
+        'price_step', pp.price_step,
+        'max_tx_ex_mem', pp.max_tx_ex_mem,
+        'max_tx_ex_steps', pp.max_tx_ex_steps,
+        'max_block_ex_mem', pp.max_block_ex_mem,
+        'max_block_ex_steps', pp.max_block_ex_steps,
+        'max_val_size', pp.max_val_size,
+        'collateral_percent', pp.collateral_percent,
+        'max_collateral_inputs', pp.max_collateral_inputs,
+        'coins_per_utxo_size', pp.coins_per_utxo_size
+      )) AS data
+  FROM public.param_proposal pp
+    INNER JOIN tx t ON t.id = pp.registered_tx_id
+    INNER JOIN block b ON t.block_id = b.id
+    LEFT JOIN cost_model CM ON CM.id = pp.cost_model_id;
 $$;
 
 COMMENT ON FUNCTION grest.param_updates IS 'Parameter updates applied to the network'; -- noqa: LT01

files/grest/rpc/00_blockchain/reserve_withdrawals.sql

@@ -8,7 +8,7 @@ RETURNS TABLE (
   amount text,
   stake_address text
 )
-LANGUAGE SQL STABLE
+LANGUAGE sql STABLE
 AS $$
   SELECT
     b.epoch_no,
@@ -24,4 +24,4 @@ AS $$
     LEFT JOIN stake_address AS sa ON sa.id = r.addr_id;
 $$;
 
-COMMENT ON FUNCTION grest.reserve_withdrawals IS 'A list of withdrawals made from reserves (MIRs)'; --noqa: LT01
+COMMENT ON FUNCTION grest.reserve_withdrawals IS 'A list of withdrawals made from reserves (MIRs)'; --noqa: LT01

files/grest/rpc/00_blockchain/tip.sql

@@ -7,23 +7,18 @@ RETURNS TABLE (
   block_no word31type,
   block_time integer
 )
-LANGUAGE plpgsql
+LANGUAGE sql STABLE
 AS $$
-BEGIN
-  RETURN QUERY
   SELECT
     ENCODE(b.hash::bytea, 'hex') AS block_hash,
     b.epoch_no AS epoch_no,
     b.slot_no AS abs_slot,
     b.epoch_slot_no AS epoch_slot,
     b.block_no,
     EXTRACT(EPOCH FROM b.time)::integer
-  FROM
-    block b
-  ORDER BY
-    b.id DESC
+  FROM block b
+  ORDER BY b.id DESC
   LIMIT 1;
-END;
 $$;
 
 COMMENT ON FUNCTION grest.tip IS 'Get the tip info about the latest block seen by chain'; -- noqa: LT01

files/grest/rpc/00_blockchain/totals.sql

@@ -7,38 +7,19 @@ RETURNS TABLE (
   supply text,
   reserves text
 )
-LANGUAGE plpgsql
+LANGUAGE sql STABLE
 AS $$
-BEGIN
-  IF _epoch_no IS NULL THEN
-    RETURN QUERY (
-      SELECT
-        ap.epoch_no,
-        ap.utxo::text,
-        ap.treasury::text,
-        ap.rewards::text,
-        (ap.treasury + ap.rewards + ap.utxo + ap.deposits + ap.fees)::text AS supply,
-        ap.reserves::text
-      FROM
-        public.ada_pots AS ap
-      ORDER BY
-        ap.epoch_no DESC);
-  ELSE
-    RETURN QUERY (
-      SELECT
-        ap.epoch_no, ap.utxo::text,
-        ap.treasury::text,
-        ap.rewards::text,
-        (ap.treasury + ap.rewards + ap.utxo + ap.deposits + ap.fees)::text AS supply,
-        ap.reserves::text
-      FROM
-        public.ada_pots AS ap
-      WHERE
-        ap.epoch_no = _epoch_no
-      ORDER BY
-        ap.epoch_no DESC);
-  END IF;
-END;
+  SELECT
+    ap.epoch_no,
+    ap.utxo::text,
+    ap.treasury::text,
+    ap.rewards::text,
+    (ap.treasury + ap.rewards + ap.utxo + ap.deposits + ap.fees)::text AS supply,
+    ap.reserves::text
+  FROM public.ada_pots AS ap
+  WHERE (_epoch_no IS NOT NULL AND ap.epoch_no = _epoch_no)
+    OR (_epoch_no IS NULL)
+  ORDER BY ap.epoch_no DESC;
 $$;
 
 COMMENT ON FUNCTION grest.totals IS 'Get the circulating utxo, treasury, rewards, supply and reserves in lovelace for specified epoch, all epochs if empty'; -- noqa: LT01

files/grest/rpc/00_blockchain/treasury_withdrawals.sql

@@ -8,7 +8,7 @@ RETURNS TABLE (
   amount text,
   stake_address text
 )
-LANGUAGE SQL STABLE
+LANGUAGE sql STABLE
 AS $$
   SELECT
     b.epoch_no,
@@ -24,4 +24,4 @@ AS $$
     LEFT JOIN stake_address AS sa ON sa.id = t.addr_id;
 $$;
 
-COMMENT ON FUNCTION grest.treasury_withdrawals IS 'A list of withdrawals made from treasury'; --noqa: LT01
+COMMENT ON FUNCTION grest.treasury_withdrawals IS 'A list of withdrawals made from treasury'; --noqa: LT01

files/grest/rpc/01_cached_tables/asset_txo_cache.sql

@@ -0,0 +1,68 @@
+CREATE OR REPLACE FUNCTION grest.asset_txo_cache_update()
+RETURNS void
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  IF (
+    SELECT COUNT(pid) > 1
+    FROM pg_stat_activity
+    WHERE state = 'active'
+      AND query ILIKE '%grest.asset_txo_cache_update%'
+      AND datname = (SELECT current_database())
+  ) THEN
+    RAISE EXCEPTION 'Previous asset_txo_cache_update query still running but should have completed! Exiting...';
+  END IF;
+
+  CREATE TEMP TABLE tmp_ma AS (
+    SELECT ma1.id
+    FROM grest.asset_cache_control AS acc1
+      LEFT JOIN multi_asset AS ma1 ON ma1.policy = acc1.policy
+      LEFT JOIN grest.asset_tx_out_cache AS atoc1 ON ma1.id = atoc1.ma_id
+    WHERE atoc1.ma_id IS NULL
+  );
+
+  WITH
+    ma_filtered AS
+      (
+        (SELECT
+          mto.tx_out_id,
+          mto.quantity,
+          mto.ident
+        FROM grest.asset_cache_control AS acc
+          LEFT JOIN multi_asset AS ma ON ma.policy = acc.policy
+          LEFT JOIN ma_tx_out AS mto ON mto.ident = ma.id
+        WHERE ma.id IN
+          (SELECT id FROM tmp_ma)
+        )
+        UNION ALL
+        (
+          SELECT
+            mto.tx_out_id,
+            mto.quantity,
+            mto.ident
+          FROM grest.asset_cache_control AS acc
+            LEFT JOIN multi_asset AS ma ON ma.policy = acc.policy
+            LEFT JOIN ma_tx_out AS mto ON mto.ident = ma.id
+          WHERE mto.tx_out_id > (SELECT COALESCE(MAX(atoc.txo_id),0) FROM grest.asset_tx_out_cache AS atoc)
+        )
+      )
+  INSERT INTO grest.asset_tx_out_cache
+    SELECT
+      mf.ident,
+      mf.tx_out_id,
+      mf.quantity
+    FROM ma_filtered AS mf
+      LEFT JOIN tx_out AS txo ON mf.tx_out_id = txo.id
+    WHERE txo.consumed_by_tx_in_id IS NULL AND txo.id < (SELECT MAX(id) from tx_out)
+  ;
+
+  DELETE FROM grest.asset_tx_out_cache WHERE txo_id IN
+    (SELECT atoc.txo_id
+      FROM grest.asset_tx_out_cache AS atoc
+        LEFT JOIN tx_out AS txo ON atoc.txo_id = txo.id
+        WHERE txo.consumed_by_tx_in_id IS NOT NULL
+          OR txo.id IS NULL);
+  DROP TABLE tmp_ma;
+
+END;
+$$;

files/grest/rpc/02_indexes/13_1_00.sql

@@ -1,4 +1,4 @@
-/* Unique Indexes that got dropped at 13.1.0.0 */
+/* Indexes additional to vanila dbsync instance */
 CREATE UNIQUE INDEX IF NOT EXISTS unique_ada_pots ON public.ada_pots USING btree (block_id);
 CREATE UNIQUE INDEX IF NOT EXISTS unique_col_txin ON public.collateral_tx_in USING btree (tx_in_id, tx_out_id, tx_out_index);
 CREATE UNIQUE INDEX IF NOT EXISTS unique_col_txout ON public.collateral_tx_out USING btree (tx_id, index);
@@ -21,6 +21,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS unique_treasury ON public.treasury USING btree
 CREATE UNIQUE INDEX IF NOT EXISTS unique_tx_metadata ON public.tx_metadata USING btree (key, tx_id);
 CREATE UNIQUE INDEX IF NOT EXISTS unique_txin ON tx_in USING btree (tx_out_id, tx_out_index);
 CREATE UNIQUE INDEX IF NOT EXISTS unique_withdrawal ON public.withdrawal USING btree (addr_id, tx_id);
-
-/* Help multi asset queries */
-CREATE INDEX IF NOT EXISTS idx_ma_tx_out_ident ON ma_tx_out (ident);
+CREATE INDEX IF NOT EXISTS idx_ma_tx_out_ident ON ma_tx_out (ident) INCLUDE (tx_out_id, quantity);
+CREATE INDEX IF NOT EXISTS idx_collateral_tx_in_tx_in_id ON collateral_tx_in (tx_in_id);
+CREATE INDEX IF NOT EXISTS idx_reference_tx_in_tx_in_id ON reference_tx_in (tx_in_id);

files/grest/rpc/account/account_assets.sql

@@ -43,4 +43,4 @@ BEGIN
 END;
 $$;
 
-COMMENT ON FUNCTION grest.account_assets IS 'Get the native asset balance of given accounts'; -- noqa: LT01
+COMMENT ON FUNCTION grest.account_assets IS 'Get the native asset balance of given accounts'; -- noqa: LT01

files/grest/rpc/account/account_utxos.sql

@@ -91,4 +91,4 @@ BEGIN
 END;
 $$;
 
-COMMENT ON FUNCTION grest.account_utxos IS  'Get UTxO details for requested stake account'; -- noqa: LT01
+COMMENT ON FUNCTION grest.account_utxos IS  'Get UTxO details for requested stake account'; -- noqa: LT01

files/grest/rpc/address/address_info.sql

@@ -116,4 +116,4 @@ BEGIN
 END;
 $$;
 
-COMMENT ON FUNCTION grest.address_info IS 'Get bulk address info - balance, associated stake address (if any) and UTXO set'; -- noqa: LT01
+COMMENT ON FUNCTION grest.address_info IS 'Get bulk address info - balance, associated stake address (if any) and UTXO set'; -- noqa: LT01

files/grest/rpc/address/address_utxos.sql

@@ -91,4 +91,4 @@ BEGIN
 END;
 $$;
 
-COMMENT ON FUNCTION grest.address_utxos IS  'Get UTxO details for requested addresses'; -- noqa: LT01
+COMMENT ON FUNCTION grest.address_utxos IS  'Get UTxO details for requested addresses'; -- noqa: LT01

files/grest/rpc/address/credential_utxos.sql

@@ -97,4 +97,4 @@ BEGIN
 END;
 $$;
 
-COMMENT ON FUNCTION grest.credential_utxos IS 'Get UTxO details for requested payment credentials'; -- noqa: LT01
+COMMENT ON FUNCTION grest.credential_utxos IS 'Get UTxO details for requested payment credentials'; -- noqa: LT01