From f518504d6e3e91b05e3f42aa88adc998b9df8243 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 12:44:53 -0400
Subject: [PATCH 01/28] Change where the cache is happening

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/database/callable.ts                      |   5 +-
 src/database/schemas.ts                       |   2 +-
 src/extension.ts                              |   2 +-
 src/language/index.ts                         |  15 --
 src/language/providers/completionItemCache.ts |  22 ---
 src/language/providers/completionProvider.ts  | 145 ++++++++----------
 src/language/providers/index.ts               |  15 ++
 .../providers/{ => logic}/available.ts        |   4 +-
 src/language/providers/logic/cache.ts         | 118 ++++++++++++++
 .../providers/{ => logic}/callable.ts         |  33 +---
 .../providers/{ => logic}/completion.ts       |   3 -
 src/language/providers/parameterProvider.ts   |   6 +-
 src/uriHandler.ts                             |   2 +-
 src/views/results/index.ts                    |   7 +-
 14 files changed, 217 insertions(+), 162 deletions(-)
 delete mode 100644 src/language/index.ts
 delete mode 100644 src/language/providers/completionItemCache.ts
 create mode 100644 src/language/providers/index.ts
 rename src/language/providers/{ => logic}/available.ts (69%)
 create mode 100644 src/language/providers/logic/cache.ts
 rename src/language/providers/{ => logic}/callable.ts (81%)
 rename src/language/providers/{ => logic}/completion.ts (89%)

diff --git a/src/database/callable.ts b/src/database/callable.ts
index 464a8bbe..4b17025d 100644
--- a/src/database/callable.ts
+++ b/src/database/callable.ts
@@ -1,7 +1,7 @@
 
 import vscode from "vscode"
 import { JobManager } from "../config";
-import { QueryOptions } from "../connection/types";
+import { QueryOptions } from "@ibm/mapepire-js/dist/src/types";
 const {instance} = vscode.extensions.getExtension(`halcyontechltd.code-for-ibmi`).exports;
 
 export type CallableType = "PROCEDURE"|"FUNCTION";
@@ -91,14 +91,13 @@ export default class Callable {
       parameters.push(name);
     }
 
-    const options : QueryOptions = { parameters };
     return JobManager.runSQL<SQLParm>(
       [
         `SELECT * FROM QSYS2.SYSPARMS`,
         `WHERE SPECIFIC_SCHEMA = ? AND ROW_TYPE = ? AND ${specificNameClause}`,
         `ORDER BY ORDINAL_POSITION`
       ].join(` `),
-      options
+      { parameters }
     );
   }
 }
\ No newline at end of file
diff --git a/src/database/schemas.ts b/src/database/schemas.ts
index 6e97aa48..6da419d4 100644
--- a/src/database/schemas.ts
+++ b/src/database/schemas.ts
@@ -4,7 +4,7 @@ import { getInstance } from "../base";
 import { JobManager } from "../config";
 
 export type SQLType = "schemas" | "tables" | "views" | "aliases" | "constraints" | "functions" | "variables" | "indexes" | "procedures" | "sequences" | "packages" | "triggers" | "types" | "logicals";
-type PageData = { filter?: string, offset?: number, limit?: number };
+export type PageData = { filter?: string, offset?: number, limit?: number };
 
 const typeMap = {
   'tables': [`T`, `P`, `M`],
diff --git a/src/extension.ts b/src/extension.ts
index fcd14bed..78e1773d 100644
--- a/src/extension.ts
+++ b/src/extension.ts
@@ -10,7 +10,7 @@ import { getInstance, loadBase } from "./base";
 import { JobManager, onConnectOrServerInstall, initConfig } from "./config";
 import { queryHistory } from "./views/queryHistoryView";
 import { ExampleBrowser } from "./views/examples/exampleBrowser";
-import { languageInit } from "./language";
+import { languageInit } from "./language/providers";
 import { initialiseTestSuite } from "./testing";
 import { JobManagerView } from "./views/jobManager/jobManagerView";
 import { ServerComponent } from "./connection/serverComponent";
diff --git a/src/language/index.ts b/src/language/index.ts
deleted file mode 100644
index 89200b0a..00000000
--- a/src/language/index.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import { completionProvider } from "./providers/completionProvider";
-import { formatProvider } from "./providers/formatProvider";
-import { signatureProvider } from "./providers/parameterProvider";
-
-export function languageInit() {
-  let functionality = [];
-
-  functionality.push(
-    completionProvider,
-    formatProvider,
-    signatureProvider
-  );
-  
-  return functionality;
-}
diff --git a/src/language/providers/completionItemCache.ts b/src/language/providers/completionItemCache.ts
deleted file mode 100644
index 48b69d86..00000000
--- a/src/language/providers/completionItemCache.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { CompletionItem } from "vscode";
-import LRU from "lru-cache";
-import { QualifiedObject } from "../sql/types";
-
-export let changedCache: Set<string> = new Set<string>();
-
-export interface CompletionItemCacheObj {
-  cacheType: "columns" | "all";
-  cacheList: CompletionItem[];
-}
-
-export default class CompletionItemCache extends LRU {
-  constructor() {
-    super({
-      max: 50,
-    });
-  }
-}
-
-export function toKey(context: string, sqlObj: QualifiedObject|string) {
-  return `${context}-` + (typeof sqlObj === `string` ? sqlObj : `${sqlObj.schema}.${sqlObj.name}`).toUpperCase();
-}
\ No newline at end of file
diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index a3887f06..0da38d87 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -3,20 +3,16 @@ import { JobManager } from "../../config";
 import {
   default as Database,
   SQLType,
-  default as Schemas,
 } from "../../database/schemas";
 import Statement from "../../database/statement";
-import Table from "../../database/table";
 import Document from "../sql/document";
 import * as LanguageStatement from "../sql/statement";
-import { CTEReference, CallableReference, ClauseType, ObjectRef, StatementType } from "../sql/types";
-import CompletionItemCache, { changedCache, toKey } from "./completionItemCache";
-import Callable, { CallableType } from "../../database/callable";
-import { ServerComponent } from "../../connection/serverComponent";
-import { prepareParamType, createCompletionItem, getParmAttributes, completionItemCache } from "./completion";
-import { isCallableType, getCallableParameters } from "./callable";
-import { variable } from "sql-formatter/lib/src/lexer/regexFactory";
-import { localAssistIsEnabled, remoteAssistIsEnabled } from "./available";
+import { CTEReference, ClauseType, ObjectRef, StatementType } from "../sql/types";
+import { CallableType } from "../../database/callable";
+import { prepareParamType, createCompletionItem, getParmAttributes } from "./logic/completion";
+import { isCallableType, getCallableParameters } from "./logic/callable";
+import { localAssistIsEnabled, remoteAssistIsEnabled } from "./logic/available";
+import { DbCache } from "./logic/cache";
 
 export interface CompletionType {
   order: string;
@@ -91,59 +87,50 @@ async function getObjectColumns(
   isUDTF = false
 ): Promise<CompletionItem[]> {
 
-  const cacheKey = toKey(`columns`, schema + name)
-  const tableUpdate: boolean = changedCache.delete(cacheKey);
-  const isCached = completionItemCache.has(cacheKey);
+  let completionItems: CompletionItem[];
 
-  if (!isCached || tableUpdate) {
     schema = Statement.noQuotes(Statement.delimName(schema, true));
     name = Statement.noQuotes(Statement.delimName(name, true));
-
-    let completionItems: CompletionItem[] = [];
     
-    if (isUDTF) {
-      const resultSet = await Callable.getResultColumns(schema, name, true);
-      
-      if (!resultSet?.length ? true : false) {
-        completionItemCache.set(cacheKey, []);
-        return [];
-      }
-      
-      completionItems = resultSet.map((i) =>
-        createCompletionItem(
-          Statement.prettyName(i.PARAMETER_NAME),
-          CompletionItemKind.Field,
-          getParmAttributes(i),
-          `Schema: ${schema}\nObject: ${name}\n`,
-          `a@objectcolumn`
-        )
-      );
-
-    } else {
-      const columns = await Table.getItems(schema, name);
+  if (isUDTF) {
+    const resultSet = await DbCache.getResultColumns(schema, name, true);
+    
+    if (!resultSet?.length ? true : false) {
+      return [];
+    }
+    
+    completionItems = resultSet.map((i) =>
+      createCompletionItem(
+        Statement.prettyName(i.PARAMETER_NAME),
+        CompletionItemKind.Field,
+        getParmAttributes(i),
+        `Schema: ${schema}\nObject: ${name}\n`,
+        `a@objectcolumn`
+      )
+    );
 
-      if (!columns?.length ? true : false) {
-        completionItemCache.set(cacheKey, []);
-        return [];
-      }
+  } else {
+    const columns = await DbCache.getItems(schema, name);
 
-      completionItems = columns.map((i) =>
-        createCompletionItem(
-          Statement.prettyName(i.COLUMN_NAME),
-          CompletionItemKind.Field,
-          getColumnAttributes(i),
-          `Schema: ${schema}\nTable: ${name}\n`,
-          `a@objectcolumn`
-        )
-      );
+    if (!columns?.length ? true : false) {
+      return [];
     }
-    
-    const allCols = getAllColumns(name, schema, completionItems);
-    completionItems.push(allCols);
-    completionItemCache.set(cacheKey, completionItems);
+
+    completionItems = columns.map((i) =>
+      createCompletionItem(
+        Statement.prettyName(i.COLUMN_NAME),
+        CompletionItemKind.Field,
+        getColumnAttributes(i),
+        `Schema: ${schema}\nTable: ${name}\n`,
+        `a@objectcolumn`
+      )
+    );
   }
+  
+  const allCols = getAllColumns(name, schema, completionItems);
+  completionItems.push(allCols);
 
-  return completionItemCache.get(cacheKey);
+  return completionItems;
 }
 
 /**
@@ -153,37 +140,32 @@ async function getObjectCompletions(
   forSchema: string,
   sqlTypes: { [index: string]: CompletionType }
 ): Promise<CompletionItem[]> {
-  forSchema = Statement.noQuotes(Statement.delimName(forSchema, true));
-  const schemaUpdate: boolean = changedCache.delete(forSchema);
-  if (!completionItemCache.has(forSchema) || schemaUpdate) {
-    const promises = Object.entries(sqlTypes).map(async ([_, value]) => {
-      const data = await Database.getObjects(forSchema, [value.type]);
-      return data.map((table) =>
-        createCompletionItem(
-          Statement.prettyName(table.name),
-          value.icon,
-          value.label,
-          `Schema: ${table.schema}`,
-          value.order
-        )
-      );
-    });
+  const promises = Object.entries(sqlTypes).map(async ([_, value]) => {
+    const data = await DbCache.getObjects(forSchema, [value.type]);
+    return data.map((table) =>
+      createCompletionItem(
+        Statement.prettyName(table.name),
+        value.icon,
+        value.label,
+        `Schema: ${table.schema}`,
+        value.order
+      )
+    );
+  });
 
-    const results = await Promise.allSettled(promises);
-    const list = results
-      .filter((result) => result.status == "fulfilled")
-      .map((result) => (result as PromiseFulfilledResult<any>).value)
-      .flat();
+  const results = await Promise.allSettled(promises);
+  const list = results
+    .filter((result) => result.status == "fulfilled")
+    .map((result) => (result as PromiseFulfilledResult<any>).value)
+    .flat();
 
-    completionItemCache.set(forSchema, list);
-  }
-  return completionItemCache.get(forSchema);
+  return list;
 }
 
 async function getCompletionItemsForSchema(
   schema: string
 ): Promise<CompletionItem[]> {
-  const data = (await Database.getObjects(schema, ["procedures"]));
+  const data = await DbCache.getObjects(schema, ["procedures"]);
 
   return data
     .filter((v, i, a) => a.findIndex(t => (t.name === v.name)) === i) //Hide overloads here
@@ -313,11 +295,7 @@ async function getCompletionItemsForTriggerDot(
 }
 
 async function getCachedSchemas() {
-  if (completionItemCache.has(`SCHEMAS-FOR-SYSTEM`)) {
-    return completionItemCache.get(`SCHEMAS-FOR-SYSTEM`);
-  }
-
-  const allSchemas: BasicSQLObject[] = await Schemas.getObjects(
+  const allSchemas: BasicSQLObject[] = await DbCache.getObjects(
     undefined,
     [`schemas`]
   );
@@ -330,7 +308,6 @@ async function getCachedSchemas() {
     )
   );
 
-  completionItemCache.set(`SCHEMAS-FOR-SYSTEM`, completionItems);
   return completionItems;
 }
 
diff --git a/src/language/providers/index.ts b/src/language/providers/index.ts
new file mode 100644
index 00000000..8d20ba9d
--- /dev/null
+++ b/src/language/providers/index.ts
@@ -0,0 +1,15 @@
+import { completionProvider } from "./completionProvider";
+import { formatProvider } from "./formatProvider";
+import { signatureProvider } from "./parameterProvider";
+
+export function languageInit() {
+  let functionality = [];
+
+  functionality.push(
+    completionProvider,
+    formatProvider,
+    signatureProvider
+  );
+  
+  return functionality;
+}
diff --git a/src/language/providers/available.ts b/src/language/providers/logic/available.ts
similarity index 69%
rename from src/language/providers/available.ts
rename to src/language/providers/logic/available.ts
index ff056916..020d91a6 100644
--- a/src/language/providers/available.ts
+++ b/src/language/providers/logic/available.ts
@@ -1,7 +1,7 @@
 
 import { env } from "process";
-import { ServerComponent } from "../../connection/serverComponent";
-import { JobManager } from "../../config";
+import { ServerComponent } from "../../../connection/serverComponent";
+import { JobManager } from "../../../config";
 
 export function localAssistIsEnabled() {
   return (env.DB2I_DISABLE_CA !== `true`);
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
new file mode 100644
index 00000000..cc20266f
--- /dev/null
+++ b/src/language/providers/logic/cache.ts
@@ -0,0 +1,118 @@
+import Callable, { CallableRoutine, CallableSignature, CallableType } from "../../../database/callable";
+import Schemas, { PageData, SQLType } from "../../../database/schemas";
+import Table from "../../../database/table";
+
+export class DbCache {
+  private static parmCache: Map<string, SQLParm[]> = new Map();
+  private static tableColumns: Map<string, TableColumn[]> = new Map();
+  private static schemaObjects: Map<string, BasicSQLObject[]> = new Map();
+  private static callables: Map<string, CallableRoutine|false> = new Map();
+  private static callableSignatures: Map<string, CallableSignature[]> = new Map();
+
+  private static toReset: string[] = [];
+
+  // TODO: call on connect
+  static async resetCache() {
+    this.parmCache.clear();
+    this.tableColumns.clear();
+    this.schemaObjects.clear();
+    this.toReset = [];
+  }
+
+  static resetObject(name: string) {
+    this.toReset.push(name.toLowerCase());
+  }
+
+  private static shouldReset(name: string) {
+    const inx = this.toReset.indexOf(name);
+
+    if (inx > -1) {
+      this.toReset.splice(inx, 1);
+      return true;
+    }
+
+    return false;
+  }
+
+  static async getResultColumns(schema: string, name: string, resolveName?: boolean) {
+    const key = getKey(`routine`, schema, name);
+    
+    if (!this.parmCache.has(key) || this.shouldReset(name)) {
+      const result = await Callable.getResultColumns(schema, name, resolveName);
+      if (result) {
+        this.parmCache.set(key, result);
+      }
+    }
+
+    return this.parmCache.get(key) || [];
+  }
+
+  static async getItems(schema: string, name: string) {
+    const key = getKey(`columns`, schema, name);
+    
+    if (!this.parmCache.has(key) || this.shouldReset(name)) {
+      const result = await Table.getItems(schema, name);
+      if (result) {
+        this.tableColumns.set(key, result);
+      }
+    }
+
+    return this.tableColumns.get(key) || [];
+  }
+
+  static async getObjects(schema: string, types: SQLType[], details?: PageData) {
+    const key = getKey(`objects`, schema, types.join(`&`));
+    
+    if (!this.schemaObjects.has(key) || this.shouldReset(schema)) {
+      const result = await Schemas.getObjects(schema, types, details);
+      if (result) {
+        this.schemaObjects.set(key, result);
+      }
+    }
+
+    return this.schemaObjects.get(key) || [];
+  }
+
+  static async getType(schema: string, name: string, type: CallableType) {
+    const key = getKey(type, schema, name);
+    
+    if (!this.callables.has(key) || this.shouldReset(name)) {
+      const result = await Callable.getType(schema, name, type);
+      if (result) {
+        this.callables.set(key, result);
+      } else {
+        this.callables.set(key, false);
+        return false;
+      }
+    }
+
+    return this.callables.get(key) || undefined;
+  }
+
+  static getCachedType(schema: string, name: string, type: CallableType) {
+    const key = getKey(type, schema, name);
+    return this.callables.get(key) || undefined
+  }
+
+  static async getSignaturesFor(schema: string, name: string, specificNames: string[]) {
+    const key = getKey(`signatures`, schema, name);
+    
+    if (!this.callableSignatures.has(key) || this.shouldReset(name)) {
+      const result = await Callable.getSignaturesFor(schema, specificNames);
+      if (result) {
+        this.callableSignatures.set(key, result);
+      }
+    }
+
+    return this.callableSignatures.get(key) || [];
+  }
+
+  static getCachedSignatures(schema: string, name: string) {
+    const key = getKey(`signatures`, schema, name);
+    return this.callableSignatures.get(key) || [];
+  }
+}
+
+function getKey(type: string, schema: string, name: string = `all`) {
+  return `${type}.${schema}.${name}`.toLowerCase();
+}
\ No newline at end of file
diff --git a/src/language/providers/callable.ts b/src/language/providers/logic/callable.ts
similarity index 81%
rename from src/language/providers/callable.ts
rename to src/language/providers/logic/callable.ts
index 1a8b078a..c74d87ac 100644
--- a/src/language/providers/callable.ts
+++ b/src/language/providers/logic/callable.ts
@@ -1,11 +1,9 @@
 import { CompletionItem, CompletionItemKind, SnippetString } from "vscode";
-import Callable, { CallableSignature, CallableType } from "../../database/callable";
-import { ObjectRef, QualifiedObject, CallableReference } from "../sql/types";
-import Statement from "../../database/statement";
-import { completionItemCache, createCompletionItem, getParmAttributes } from "./completion";
-import { toKey } from "./completionItemCache";
-
-const SIGNATURE_CONTEXT_KEY = `sigs`;
+import { CallableSignature, CallableType } from "../../../database/callable";
+import { ObjectRef, CallableReference } from "../../sql/types";
+import Statement from "../../../database/statement";
+import { createCompletionItem, getParmAttributes } from "./completion";
+import { DbCache } from "./cache";
 
 /**
  * Checks if the ref exists as a procedure or function. Then,
@@ -16,21 +14,13 @@ export async function isCallableType(ref: ObjectRef, type: CallableType) {
     ref.object.schema = Statement.noQuotes(Statement.delimName(ref.object.schema, true));
     ref.object.name = Statement.noQuotes(Statement.delimName(ref.object.name, true));
 
-    const cacheKey = toKey(SIGNATURE_CONTEXT_KEY, ref.object);
-
-    if (completionItemCache.has(cacheKey)) {
-      return true;
-    }
-
-    const callableRoutine = await Callable.getType(ref.object.schema, ref.object.name, type);
+    const callableRoutine = await DbCache.getType(ref.object.schema, ref.object.name, type);
 
     if (callableRoutine) {
-      const parms = await Callable.getSignaturesFor(ref.object.schema, callableRoutine.specificNames);
-      completionItemCache.set(cacheKey, parms);
+      DbCache.getSignaturesFor(ref.object.schema, ref.object.name, callableRoutine.specificNames);
       return true;
     } else {
       // Not callable, let's just cache it as empty to stop spamming the db
-      completionItemCache.set(cacheKey, []);
     }
   }
 
@@ -42,7 +32,7 @@ export async function isCallableType(ref: ObjectRef, type: CallableType) {
  * that are stored in the cache for a specific procedure
  */
 export function getCallableParameters(ref: CallableReference, offset: number): CompletionItem[] {
-  const signatures = getCachedSignatures(ref);
+  const signatures = DbCache.getCachedSignatures(ref.parentRef.object.schema, ref.parentRef.object.name)
   if (signatures) {
     const { firstNamedParameter, currentCount } = getPositionData(ref, offset);
 
@@ -149,11 +139,4 @@ export function getPositionData(ref: CallableReference, offset: number) {
     currentCount: paramCommas.length + 1,
     firstNamedParameter
   };
-}
-
-export function getCachedSignatures(ref: CallableReference): CallableSignature[] | undefined {
-  const key = toKey(SIGNATURE_CONTEXT_KEY, ref.parentRef.object);
-  if (completionItemCache.has(key)) {
-    return completionItemCache.get(key);
-  }
 }
\ No newline at end of file
diff --git a/src/language/providers/completion.ts b/src/language/providers/logic/completion.ts
similarity index 89%
rename from src/language/providers/completion.ts
rename to src/language/providers/logic/completion.ts
index 6b377f41..1aac7e20 100644
--- a/src/language/providers/completion.ts
+++ b/src/language/providers/logic/completion.ts
@@ -1,7 +1,4 @@
 import { CompletionItemKind, CompletionItem } from "vscode";
-import CompletionItemCache from "./completionItemCache";
-
-export const completionItemCache = new CompletionItemCache();
 
 export function createCompletionItem(
   name: string,
diff --git a/src/language/providers/parameterProvider.ts b/src/language/providers/parameterProvider.ts
index 235fc0fe..69e05fec 100644
--- a/src/language/providers/parameterProvider.ts
+++ b/src/language/providers/parameterProvider.ts
@@ -1,11 +1,11 @@
 import { MarkdownString, ParameterInformation, Position, Range, SignatureHelp, SignatureInformation, TextEdit, languages } from "vscode";
 import Statement from "../../database/statement";
 import Document from "../sql/document";
-import { getCachedSignatures, getCallableParameters, getPositionData, isCallableType } from "./callable";
-import { getParmAttributes, prepareParamType } from "./completion";
+import { getCachedSignatures, getCallableParameters, getPositionData, isCallableType } from "./logic/callable";
+import { getParmAttributes, prepareParamType } from "./logic/completion";
 import { CallableType } from "../../database/callable";
 import { StatementType } from "../sql/types";
-import { remoteAssistIsEnabled } from "./available";
+import { remoteAssistIsEnabled } from "./logic/available";
 
 export const signatureProvider = languages.registerSignatureHelpProvider({ language: `sql` }, {
   async provideSignatureHelp(document, position, token, context) {
diff --git a/src/uriHandler.ts b/src/uriHandler.ts
index f01dfb04..39ec126b 100644
--- a/src/uriHandler.ts
+++ b/src/uriHandler.ts
@@ -2,7 +2,7 @@ import { commands, env, Selection, Uri, UriHandler, window, workspace } from "vs
 import querystring from "querystring";
 import Document from "./language/sql/document";
 import { ServerComponent } from "./connection/serverComponent";
-import { remoteAssistIsEnabled } from "./language/providers/available";
+import { remoteAssistIsEnabled } from "./language/providers/logic/available";
 
 export class Db2iUriHandler implements UriHandler {
   handleUri(uri: Uri) {
diff --git a/src/views/results/index.ts b/src/views/results/index.ts
index d5a97ea7..88be25e6 100644
--- a/src/views/results/index.ts
+++ b/src/views/results/index.ts
@@ -5,7 +5,6 @@ import * as csv from "csv/sync";
 
 import { JobManager } from "../../config";
 import Document from "../../language/sql/document";
-import { changedCache } from "../../language/providers/completionItemCache";
 import { ParsedEmbeddedStatement, StatementGroup, StatementType } from "../../language/sql/types";
 import Statement from "../../language/sql/statement";
 import { ExplainTree } from "./explain/nodes";
@@ -16,6 +15,7 @@ import { ResultSetPanelProvider } from "./resultSetPanelProvider";
 import { generateSqlForAdvisedIndexes } from "./explain/advice";
 import { updateStatusBar } from "../jobManager/statusBar";
 import { ExplainType } from "@ibm/mapepire-js/dist/src/types";
+import { DbCache } from "../../language/providers/logic/cache";
 
 export type StatementQualifier = "statement" | "explain" | "onlyexplain" | "json" | "csv" | "cl" | "sql";
 
@@ -202,7 +202,10 @@ async function runHandler(options?: StatementInfo) {
         statement.type === StatementType.Create && ref.createType.toUpperCase() === `schema`
           ? ref.object.schema || ``
           : ref.object.schema + ref.object.name;
-      changedCache.add((databaseObj || ``).toUpperCase());
+
+      if (databaseObj) {
+        DbCache.resetObject(databaseObj);
+      }
     }
 
     if (statementDetail.content.trim().length > 0) {

From 576a8933143a5a3a7fda3a29d9212f77f08c9993 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:03:10 -0400
Subject: [PATCH 02/28] Fix to object list

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/completionProvider.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index 0da38d87..0d53f0b3 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -140,6 +140,8 @@ async function getObjectCompletions(
   forSchema: string,
   sqlTypes: { [index: string]: CompletionType }
 ): Promise<CompletionItem[]> {
+  forSchema = Statement.noQuotes(Statement.delimName(forSchema, true));
+  
   const promises = Object.entries(sqlTypes).map(async ([_, value]) => {
     const data = await DbCache.getObjects(forSchema, [value.type]);
     return data.map((table) =>

From 35c28d82ca89ae13623aa052e340b78494a708c5 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:03:19 -0400
Subject: [PATCH 03/28] Parameter provider to use newer API

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/parameterProvider.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/language/providers/parameterProvider.ts b/src/language/providers/parameterProvider.ts
index 69e05fec..624a360e 100644
--- a/src/language/providers/parameterProvider.ts
+++ b/src/language/providers/parameterProvider.ts
@@ -1,11 +1,12 @@
 import { MarkdownString, ParameterInformation, Position, Range, SignatureHelp, SignatureInformation, TextEdit, languages } from "vscode";
 import Statement from "../../database/statement";
 import Document from "../sql/document";
-import { getCachedSignatures, getCallableParameters, getPositionData, isCallableType } from "./logic/callable";
+import { getCallableParameters, getPositionData, isCallableType } from "./logic/callable";
 import { getParmAttributes, prepareParamType } from "./logic/completion";
 import { CallableType } from "../../database/callable";
 import { StatementType } from "../sql/types";
 import { remoteAssistIsEnabled } from "./logic/available";
+import { DbCache } from "./logic/cache";
 
 export const signatureProvider = languages.registerSignatureHelpProvider({ language: `sql` }, {
   async provideSignatureHelp(document, position, token, context) {
@@ -24,7 +25,7 @@ export const signatureProvider = languages.registerSignatureHelpProvider({ langu
         if (callableRef) {
           const isValid = await isCallableType(callableRef.parentRef, routineType);
           if (isValid) {
-            let signatures = getCachedSignatures(callableRef);
+            let signatures = DbCache.getCachedSignatures(callableRef.parentRef.object.schema, callableRef.parentRef.object.name);
             if (signatures) {
               const help = new SignatureHelp();
 

From b4b1b2fa1b0c1aacd56a693533a284f37c61223d Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:04:00 -0400
Subject: [PATCH 04/28] Remove LRU

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 package-lock.json | 13 ++++++++-----
 package.json      |  1 -
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 2b65baab..5af274c1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,18 +1,17 @@
 {
   "name": "vscode-db2i",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vscode-db2i",
-      "version": "1.3.1",
+      "version": "1.4.1",
       "dependencies": {
         "@ibm/mapepire-js": "^0.3.0",
         "chart.js": "^4.4.2",
         "csv": "^6.1.3",
         "json-to-markdown-table": "^1.0.0",
-        "lru-cache": "^6.0.0",
         "node-fetch": "^3.3.1",
         "showdown": "^2.1.0",
         "sql-formatter": "^14.0.0"
@@ -3228,6 +3227,7 @@
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
       "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
       "dependencies": {
         "yallist": "^4.0.0"
       },
@@ -5700,7 +5700,8 @@
     "node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
     },
     "node_modules/yocto-queue": {
       "version": "1.0.0",
@@ -8090,6 +8091,7 @@
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
       "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+      "dev": true,
       "requires": {
         "yallist": "^4.0.0"
       }
@@ -9690,7 +9692,8 @@
     "yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "dev": true
     },
     "yocto-queue": {
       "version": "1.0.0",
diff --git a/package.json b/package.json
index 2bea56f4..1a75a059 100644
--- a/package.json
+++ b/package.json
@@ -65,7 +65,6 @@
     "chart.js": "^4.4.2",
     "csv": "^6.1.3",
     "json-to-markdown-table": "^1.0.0",
-    "lru-cache": "^6.0.0",
     "node-fetch": "^3.3.1",
     "showdown": "^2.1.0",
     "sql-formatter": "^14.0.0"

From 9104f2fb44b073d716411ff05346f2bda5620415 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:05:24 -0400
Subject: [PATCH 05/28] Reset cache on connect

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/extension.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/extension.ts b/src/extension.ts
index 78e1773d..1830ccda 100644
--- a/src/extension.ts
+++ b/src/extension.ts
@@ -21,6 +21,7 @@ import { SelfTreeDecorationProvider, selfCodesResultsView } from "./views/jobMan
 import Configuration from "./configuration";
 import { JDBCOptions } from "@ibm/mapepire-js/dist/src/types";
 import { Db2iUriHandler, getStatementUri } from "./uriHandler";
+import { DbCache } from "./language/providers/logic/cache";
 
 export interface Db2i {
   sqlJobManager: SQLJobManager,
@@ -88,6 +89,7 @@ export function activate(context: vscode.ExtensionContext): Db2i {
   const instance = getInstance();
 
   instance.subscribe(context, `connected`, `db2i-connected`, () => {
+    DbCache.resetCache();
     selfCodesView.setRefreshEnabled(false);
     selfCodesView.setJobOnly(false);
     // Refresh the examples when we have it, so we only display certain examples

From a6fb63800502427fe0ad0c6e115151892854a8e1 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:08:09 -0400
Subject: [PATCH 06/28] Name refactor

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/completionProvider.ts |  4 +-
 src/language/providers/logic/cache.ts        | 46 ++++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index 0d53f0b3..52a4a91b 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -93,7 +93,7 @@ async function getObjectColumns(
     name = Statement.noQuotes(Statement.delimName(name, true));
     
   if (isUDTF) {
-    const resultSet = await DbCache.getResultColumns(schema, name, true);
+    const resultSet = await DbCache.getRoutineColumns(schema, name, true);
     
     if (!resultSet?.length ? true : false) {
       return [];
@@ -110,7 +110,7 @@ async function getObjectColumns(
     );
 
   } else {
-    const columns = await DbCache.getItems(schema, name);
+    const columns = await DbCache.getColumns(schema, name);
 
     if (!columns?.length ? true : false) {
       return [];
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index cc20266f..f1cc78c3 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -3,18 +3,18 @@ import Schemas, { PageData, SQLType } from "../../../database/schemas";
 import Table from "../../../database/table";
 
 export class DbCache {
-  private static parmCache: Map<string, SQLParm[]> = new Map();
-  private static tableColumns: Map<string, TableColumn[]> = new Map();
+  private static routineColumns: Map<string, SQLParm[]> = new Map();
+  private static objectColumns: Map<string, TableColumn[]> = new Map();
   private static schemaObjects: Map<string, BasicSQLObject[]> = new Map();
-  private static callables: Map<string, CallableRoutine|false> = new Map();
-  private static callableSignatures: Map<string, CallableSignature[]> = new Map();
+  private static routines: Map<string, CallableRoutine|false> = new Map();
+  private static routineSignatures: Map<string, CallableSignature[]> = new Map();
 
   private static toReset: string[] = [];
 
   // TODO: call on connect
   static async resetCache() {
-    this.parmCache.clear();
-    this.tableColumns.clear();
+    this.routineColumns.clear();
+    this.objectColumns.clear();
     this.schemaObjects.clear();
     this.toReset = [];
   }
@@ -34,30 +34,30 @@ export class DbCache {
     return false;
   }
 
-  static async getResultColumns(schema: string, name: string, resolveName?: boolean) {
+  static async getRoutineColumns(schema: string, name: string, resolveName?: boolean) {
     const key = getKey(`routine`, schema, name);
     
-    if (!this.parmCache.has(key) || this.shouldReset(name)) {
+    if (!this.routineColumns.has(key) || this.shouldReset(name)) {
       const result = await Callable.getResultColumns(schema, name, resolveName);
       if (result) {
-        this.parmCache.set(key, result);
+        this.routineColumns.set(key, result);
       }
     }
 
-    return this.parmCache.get(key) || [];
+    return this.routineColumns.get(key) || [];
   }
 
-  static async getItems(schema: string, name: string) {
+  static async getColumns(schema: string, name: string) {
     const key = getKey(`columns`, schema, name);
     
-    if (!this.parmCache.has(key) || this.shouldReset(name)) {
+    if (!this.routineColumns.has(key) || this.shouldReset(name)) {
       const result = await Table.getItems(schema, name);
       if (result) {
-        this.tableColumns.set(key, result);
+        this.objectColumns.set(key, result);
       }
     }
 
-    return this.tableColumns.get(key) || [];
+    return this.objectColumns.get(key) || [];
   }
 
   static async getObjects(schema: string, types: SQLType[], details?: PageData) {
@@ -76,40 +76,40 @@ export class DbCache {
   static async getType(schema: string, name: string, type: CallableType) {
     const key = getKey(type, schema, name);
     
-    if (!this.callables.has(key) || this.shouldReset(name)) {
+    if (!this.routines.has(key) || this.shouldReset(name)) {
       const result = await Callable.getType(schema, name, type);
       if (result) {
-        this.callables.set(key, result);
+        this.routines.set(key, result);
       } else {
-        this.callables.set(key, false);
+        this.routines.set(key, false);
         return false;
       }
     }
 
-    return this.callables.get(key) || undefined;
+    return this.routines.get(key) || undefined;
   }
 
   static getCachedType(schema: string, name: string, type: CallableType) {
     const key = getKey(type, schema, name);
-    return this.callables.get(key) || undefined
+    return this.routines.get(key) || undefined
   }
 
   static async getSignaturesFor(schema: string, name: string, specificNames: string[]) {
     const key = getKey(`signatures`, schema, name);
     
-    if (!this.callableSignatures.has(key) || this.shouldReset(name)) {
+    if (!this.routineSignatures.has(key) || this.shouldReset(name)) {
       const result = await Callable.getSignaturesFor(schema, specificNames);
       if (result) {
-        this.callableSignatures.set(key, result);
+        this.routineSignatures.set(key, result);
       }
     }
 
-    return this.callableSignatures.get(key) || [];
+    return this.routineSignatures.get(key) || [];
   }
 
   static getCachedSignatures(schema: string, name: string) {
     const key = getKey(`signatures`, schema, name);
-    return this.callableSignatures.get(key) || [];
+    return this.routineSignatures.get(key) || [];
   }
 }
 

From d0227bdfe38dd354f81a5f12bd50ac3beb837ae6 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:09:32 -0400
Subject: [PATCH 07/28] Improved name for routine columns

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/completionProvider.ts | 2 +-
 src/language/providers/logic/cache.ts        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index 52a4a91b..0a307533 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -93,7 +93,7 @@ async function getObjectColumns(
     name = Statement.noQuotes(Statement.delimName(name, true));
     
   if (isUDTF) {
-    const resultSet = await DbCache.getRoutineColumns(schema, name, true);
+    const resultSet = await DbCache.getRoutineResultColumns(schema, name, true);
     
     if (!resultSet?.length ? true : false) {
       return [];
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index f1cc78c3..fac2ed20 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -34,7 +34,7 @@ export class DbCache {
     return false;
   }
 
-  static async getRoutineColumns(schema: string, name: string, resolveName?: boolean) {
+  static async getRoutineResultColumns(schema: string, name: string, resolveName?: boolean) {
     const key = getKey(`routine`, schema, name);
     
     if (!this.routineColumns.has(key) || this.shouldReset(name)) {

From f852197dcd8c097968e3694d23fa861f82bfeaf3 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 13:50:20 -0400
Subject: [PATCH 08/28] More detailed interfaces

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 global.d.ts              | 3 +++
 src/database/callable.ts | 4 ++++
 src/database/table.ts    | 2 ++
 3 files changed, 9 insertions(+)

diff --git a/global.d.ts b/global.d.ts
index ceca16a3..e5762a4f 100644
--- a/global.d.ts
+++ b/global.d.ts
@@ -1,4 +1,6 @@
 interface TableColumn {
+  TABLE_SCHEMA?: string,
+  TABLE_NAME?: string,
   COLUMN_NAME: string,
   SYSTEM_COLUMN_NAME: string,
   CONSTRAINT_NAME?: string,
@@ -14,6 +16,7 @@ interface TableColumn {
 }
 
 interface SQLParm {
+  SPECIFIC_SCHEMA: string,
   SPECIFIC_NAME: string,
   PARAMETER_NAME: string,
   PARAMETER_MODE: "IN" | "OUT" | "INOUT",
diff --git a/src/database/callable.ts b/src/database/callable.ts
index 4b17025d..bb158e6c 100644
--- a/src/database/callable.ts
+++ b/src/database/callable.ts
@@ -6,6 +6,8 @@ const {instance} = vscode.extensions.getExtension(`halcyontechltd.code-for-ibmi`
 
 export type CallableType = "PROCEDURE"|"FUNCTION";
 export interface CallableRoutine {
+  schema: string;
+  name: string;
   specificNames: string[];
   type: string;
 }
@@ -25,6 +27,8 @@ export default class Callable {
     );
 
     let routine: CallableRoutine = {
+      schema,
+      name,
       specificNames: [],
       type: forType
     }
diff --git a/src/database/table.ts b/src/database/table.ts
index c18b4899..e7928519 100644
--- a/src/database/table.ts
+++ b/src/database/table.ts
@@ -13,6 +13,8 @@ export default class Table {
   static async getItems(schema: string, name: string): Promise<TableColumn[]> {
     const sql = [
       `SELECT `,
+      `  column.TABLE_SCHEMA,`,
+      `  column.TABLE_NAME,`,
       `  column.COLUMN_NAME,`,
       `  key.CONSTRAINT_NAME,`,
       `  column.DATA_TYPE, `,

From b61f35b0a6b2dc0c37771919cb792637cfc8d8ea Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 14:05:20 -0400
Subject: [PATCH 09/28] Initial work on hover support

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/completionProvider.ts |  3 +-
 src/language/providers/hoverProvider.ts      | 47 ++++++++++++++++
 src/language/providers/index.ts              |  4 +-
 src/language/providers/logic/cache.ts        | 56 ++++++++++++++++----
 src/language/providers/logic/parse.ts        | 16 ++++++
 src/language/providers/parameterProvider.ts  |  3 +-
 6 files changed, 117 insertions(+), 12 deletions(-)
 create mode 100644 src/language/providers/hoverProvider.ts
 create mode 100644 src/language/providers/logic/parse.ts

diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index 0a307533..b19c9cc0 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -13,6 +13,7 @@ import { prepareParamType, createCompletionItem, getParmAttributes } from "./log
 import { isCallableType, getCallableParameters } from "./logic/callable";
 import { localAssistIsEnabled, remoteAssistIsEnabled } from "./logic/available";
 import { DbCache } from "./logic/cache";
+import { getSqlDocument } from "./logic/parse";
 
 export interface CompletionType {
   order: string;
@@ -596,7 +597,7 @@ export const completionProvider = languages.registerCompletionItemProvider(
         const content = document.getText();
         const offset = document.offsetAt(position);
 
-        const sqlDoc = new Document(content);
+        const sqlDoc = getSqlDocument(document);
         const currentStatement = sqlDoc.getStatementByOffset(offset);
 
         const allItems: CompletionItem[] = [];
diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
new file mode 100644
index 00000000..fcbbea4c
--- /dev/null
+++ b/src/language/providers/hoverProvider.ts
@@ -0,0 +1,47 @@
+import { Hover, languages, MarkdownString } from "vscode";
+import { getSqlDocument } from "./logic/parse";
+import { DbCache } from "./logic/cache";
+import { JobManager } from "../../config";
+import Statement from "../../database/statement";
+
+export const hoverProvider = languages.registerHoverProvider({ language: `sql` }, {
+  async provideHover(document, position, token) {
+    const defaultSchema = getDefaultSchema();
+    const sqlDoc = getSqlDocument(document);
+    const offset = document.offsetAt(position);
+
+    const tokAt = sqlDoc.getTokenByOffset(offset);
+    const statementAt = sqlDoc.getStatementByOffset(offset);
+
+    if (statementAt) {
+      const refs = statementAt.getObjectReferences();
+      for (const ref of refs) {
+        const atRef = offset >= ref.tokens[0].range.start && offset <= ref.tokens[ref.tokens.length - 1].range.end;
+
+        if (atRef) {
+          const result = lookupSymbol(ref.object.name, ref.object.schema || defaultSchema);
+          return new Hover(new MarkdownString().appendCodeblock(JSON.stringify(result, null, 2), `json`));
+        }
+      }
+    }
+
+    if (tokAt && tokAt.value) {
+      const result = lookupSymbol(tokAt.value, defaultSchema);
+      return new Hover(new MarkdownString().appendCodeblock(JSON.stringify(result, null, 2), `json`));
+    }
+
+    return;
+  }
+});
+
+function lookupSymbol(name: string, schema: string) {
+  name = Statement.noQuotes(Statement.delimName(name, true));
+  schema = Statement.noQuotes(Statement.delimName(schema, true));
+
+  return DbCache.lookupSymbol(name, schema);
+}
+
+const getDefaultSchema = (): string => {
+  const currentJob = JobManager.getSelection();
+  return currentJob && currentJob.job.options.libraries[0] ? currentJob.job.options.libraries[0] : `QGPL`;
+}
\ No newline at end of file
diff --git a/src/language/providers/index.ts b/src/language/providers/index.ts
index 8d20ba9d..e6febc13 100644
--- a/src/language/providers/index.ts
+++ b/src/language/providers/index.ts
@@ -1,5 +1,6 @@
 import { completionProvider } from "./completionProvider";
 import { formatProvider } from "./formatProvider";
+import { hoverProvider } from "./hoverProvider";
 import { signatureProvider } from "./parameterProvider";
 
 export function languageInit() {
@@ -8,7 +9,8 @@ export function languageInit() {
   functionality.push(
     completionProvider,
     formatProvider,
-    signatureProvider
+    signatureProvider,
+    hoverProvider
   );
   
   return functionality;
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index fac2ed20..7ba4d611 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -2,18 +2,22 @@ import Callable, { CallableRoutine, CallableSignature, CallableType } from "../.
 import Schemas, { PageData, SQLType } from "../../../database/schemas";
 import Table from "../../../database/table";
 
+interface RoutineDetail {
+  routine: CallableRoutine;
+  signatures: CallableSignature[];
+}
+
 export class DbCache {
-  private static routineColumns: Map<string, SQLParm[]> = new Map();
-  private static objectColumns: Map<string, TableColumn[]> = new Map();
   private static schemaObjects: Map<string, BasicSQLObject[]> = new Map();
+  private static routineResultColumns: Map<string, SQLParm[]> = new Map();
+  private static objectColumns: Map<string, TableColumn[]> = new Map();
   private static routines: Map<string, CallableRoutine|false> = new Map();
   private static routineSignatures: Map<string, CallableSignature[]> = new Map();
 
   private static toReset: string[] = [];
 
-  // TODO: call on connect
   static async resetCache() {
-    this.routineColumns.clear();
+    this.routineResultColumns.clear();
     this.objectColumns.clear();
     this.schemaObjects.clear();
     this.toReset = [];
@@ -24,7 +28,7 @@ export class DbCache {
   }
 
   private static shouldReset(name: string) {
-    const inx = this.toReset.indexOf(name);
+    const inx = this.toReset.indexOf(name.toLowerCase());
 
     if (inx > -1) {
       this.toReset.splice(inx, 1);
@@ -34,23 +38,57 @@ export class DbCache {
     return false;
   }
 
+  static lookupSymbol(name: string, schema: string) {
+    const routine = this.getCachedType(name, schema, `FUNCTION`) || this.getCachedType(name, schema, `PROCEDURE`);
+    if (routine) {
+      const signatures = this.getCachedSignatures(schema, name);
+      return { routine, signatures } as RoutineDetail;
+    }
+
+    // Search objects
+    for (const currentSchema of this.schemaObjects.values()) {
+      const chosenObject = currentSchema.find(column => column.name === name && column.schema === schema);
+      if (chosenObject) {
+        return chosenObject;
+      }
+    }
+
+    // Lookup by column
+
+    // First object columns
+    for (const currentObject of this.objectColumns.values()) {
+      const chosenColumn = currentObject.find(column => column.COLUMN_NAME.toLowerCase() === name.toLowerCase());
+      if (chosenColumn) {
+        return chosenColumn;
+      }
+    }
+
+    // Then by routine result columns
+    for (const currentRoutine of this.routineResultColumns.values()) {
+      const chosenColumn = currentRoutine.find(column => column.PARAMETER_NAME.toLowerCase() === name.toLowerCase());
+      if (chosenColumn) {
+        return chosenColumn;
+      }
+    }
+  }
+
   static async getRoutineResultColumns(schema: string, name: string, resolveName?: boolean) {
     const key = getKey(`routine`, schema, name);
     
-    if (!this.routineColumns.has(key) || this.shouldReset(name)) {
+    if (!this.routineResultColumns.has(key) || this.shouldReset(name)) {
       const result = await Callable.getResultColumns(schema, name, resolveName);
       if (result) {
-        this.routineColumns.set(key, result);
+        this.routineResultColumns.set(key, result);
       }
     }
 
-    return this.routineColumns.get(key) || [];
+    return this.routineResultColumns.get(key) || [];
   }
 
   static async getColumns(schema: string, name: string) {
     const key = getKey(`columns`, schema, name);
     
-    if (!this.routineColumns.has(key) || this.shouldReset(name)) {
+    if (!this.routineResultColumns.has(key) || this.shouldReset(name)) {
       const result = await Table.getItems(schema, name);
       if (result) {
         this.objectColumns.set(key, result);
diff --git a/src/language/providers/logic/parse.ts b/src/language/providers/logic/parse.ts
new file mode 100644
index 00000000..4df09c3b
--- /dev/null
+++ b/src/language/providers/logic/parse.ts
@@ -0,0 +1,16 @@
+import { TextDocument } from "vscode";
+import Document from "../../sql/document";
+
+let cachedAst: Document | undefined;
+let cachedVersion: number = -1;
+
+export function getSqlDocument(document: TextDocument) {
+  if (cachedAst && cachedVersion === document.version) {
+    return cachedAst;
+  }
+
+  cachedAst = new Document(document.getText());
+  cachedVersion = document.version;
+
+  return cachedAst;
+}
\ No newline at end of file
diff --git a/src/language/providers/parameterProvider.ts b/src/language/providers/parameterProvider.ts
index 624a360e..fbe2368c 100644
--- a/src/language/providers/parameterProvider.ts
+++ b/src/language/providers/parameterProvider.ts
@@ -7,6 +7,7 @@ import { CallableType } from "../../database/callable";
 import { StatementType } from "../sql/types";
 import { remoteAssistIsEnabled } from "./logic/available";
 import { DbCache } from "./logic/cache";
+import { getSqlDocument } from "./logic/parse";
 
 export const signatureProvider = languages.registerSignatureHelpProvider({ language: `sql` }, {
   async provideSignatureHelp(document, position, token, context) {
@@ -15,7 +16,7 @@ export const signatureProvider = languages.registerSignatureHelpProvider({ langu
 
     if (remoteAssistIsEnabled()) {
 
-      const sqlDoc = new Document(content);
+      const sqlDoc = getSqlDocument(document);
       const currentStatement = sqlDoc.getStatementByOffset(offset);
 
       if (currentStatement) {

From 5d3d2323c1f7614a8bd8fec90574db606ae55da9 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 15:20:54 -0400
Subject: [PATCH 10/28] Initial work for hover support

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts    | 63 ++++++++++++++++++++--
 src/language/providers/logic/cache.ts      | 27 +++++++---
 src/language/providers/logic/callable.ts   |  2 +-
 src/language/providers/logic/completion.ts | 12 +++--
 4 files changed, 87 insertions(+), 17 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index fcbbea4c..187a9374 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -1,8 +1,9 @@
 import { Hover, languages, MarkdownString } from "vscode";
 import { getSqlDocument } from "./logic/parse";
-import { DbCache } from "./logic/cache";
+import { DbCache, LookupResult } from "./logic/cache";
 import { JobManager } from "../../config";
 import Statement from "../../database/statement";
+import { getParmAttributes, prepareParamType } from "./logic/completion";
 
 export const hoverProvider = languages.registerHoverProvider({ language: `sql` }, {
   async provideHover(document, position, token) {
@@ -13,6 +14,8 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
     const tokAt = sqlDoc.getTokenByOffset(offset);
     const statementAt = sqlDoc.getStatementByOffset(offset);
 
+    const md = new MarkdownString();
+
     if (statementAt) {
       const refs = statementAt.getObjectReferences();
       for (const ref of refs) {
@@ -20,20 +23,70 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
         if (atRef) {
           const result = lookupSymbol(ref.object.name, ref.object.schema || defaultSchema);
-          return new Hover(new MarkdownString().appendCodeblock(JSON.stringify(result, null, 2), `json`));
+          if (result) {
+            addSymbol(md, result);
+            return new Hover(md);
+          }
         }
       }
     }
 
     if (tokAt && tokAt.value) {
       const result = lookupSymbol(tokAt.value, defaultSchema);
-      return new Hover(new MarkdownString().appendCodeblock(JSON.stringify(result, null, 2), `json`));
+      if (result) {
+        addSymbol(md, result);
+        return new Hover(md);
+      }
     }
-
-    return;
   }
 });
 
+const systemSchemas = [`QSYS`, `QSYS2`, `SYSTOOLS`];
+
+function addSearch (base: MarkdownString, value: string) {
+  base.appendMarkdown([
+    ``,
+    `---`,
+    `[Search on IBM Documentation](https://www.ibm.com/docs/en/search/${encodeURI(value)})`
+  ].join(`\n\n`));
+}
+
+function addList(base: MarkdownString, items: string[]) {
+  if (items.length) {
+    base.appendMarkdown(`\n` + items.map(item => `- ${item}`).join(`\n`) + `\n`);
+  }
+}
+
+function addSymbol(base: MarkdownString, symbol: LookupResult) {
+  base.isTrusted = true;
+
+  if ('routine' in symbol) {
+    const routineName = Statement.prettyName(symbol.routine.name);
+    for (const signature of symbol.signatures) {
+      base.appendCodeblock(`${routineName}(${signature.parms.map(p => `${Statement.prettyName(p.PARAMETER_NAME)}${p.DEFAULT !== undefined ? `?` : ``}`).join(', ')})`, `sql`);
+    }
+
+    if (systemSchemas.includes(symbol.routine.schema)) {
+      addSearch(base, symbol.routine.name);
+    }
+  }
+  else if ('PARAMETER_NAME' in symbol) {
+    base.appendCodeblock(prepareParamType(symbol) + `\n`, `sql`);
+  }
+  else if ('COLUMN_NAME' in symbol) {
+    base.appendCodeblock(prepareParamType(symbol) + `\n`, `sql`);
+  }
+  else if ('name' in symbol) {
+    addList(base, [
+      `**Description:** ${symbol.text}`,
+    ]);
+
+    if (systemSchemas.includes(symbol.schema)) {
+      addSearch(base, symbol.name);
+    }
+  }
+}
+
 function lookupSymbol(name: string, schema: string) {
   name = Statement.noQuotes(Statement.delimName(name, true));
   schema = Statement.noQuotes(Statement.delimName(schema, true));
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index 7ba4d611..e9fafa05 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -5,8 +5,11 @@ import Table from "../../../database/table";
 interface RoutineDetail {
   routine: CallableRoutine;
   signatures: CallableSignature[];
+  resultingColumns: SQLParm[];
 }
 
+export type LookupResult = RoutineDetail | SQLParm | BasicSQLObject | TableColumn;
+
 export class DbCache {
   private static schemaObjects: Map<string, BasicSQLObject[]> = new Map();
   private static routineResultColumns: Map<string, SQLParm[]> = new Map();
@@ -27,22 +30,25 @@ export class DbCache {
     this.toReset.push(name.toLowerCase());
   }
 
-  private static shouldReset(name: string) {
-    const inx = this.toReset.indexOf(name.toLowerCase());
+  private static shouldReset(name?: string) {
+    if (name) {
+      const inx = this.toReset.indexOf(name.toLowerCase());
 
-    if (inx > -1) {
-      this.toReset.splice(inx, 1);
-      return true;
+      if (inx > -1) {
+        this.toReset.splice(inx, 1);
+        return true;
+      }
     }
 
     return false;
   }
 
-  static lookupSymbol(name: string, schema: string) {
-    const routine = this.getCachedType(name, schema, `FUNCTION`) || this.getCachedType(name, schema, `PROCEDURE`);
+  static lookupSymbol(name: string, schema: string): LookupResult {
+    const routine = this.getCachedType(schema, name, `FUNCTION`) || this.getCachedType(schema, name, `PROCEDURE`);
     if (routine) {
       const signatures = this.getCachedSignatures(schema, name);
-      return { routine, signatures } as RoutineDetail;
+      const resultingColumns = this.getCachedRoutineResultColumns(schema, name);
+      return { routine, signatures, resultingColumns } as RoutineDetail;
     }
 
     // Search objects
@@ -85,6 +91,11 @@ export class DbCache {
     return this.routineResultColumns.get(key) || [];
   }
 
+  static getCachedRoutineResultColumns(schema: string, name: string) {
+    const key = getKey(`routine`, schema, name);
+    return this.routineResultColumns.get(key) || [];
+  }
+
   static async getColumns(schema: string, name: string) {
     const key = getKey(`columns`, schema, name);
     
diff --git a/src/language/providers/logic/callable.ts b/src/language/providers/logic/callable.ts
index c74d87ac..264a8bfc 100644
--- a/src/language/providers/logic/callable.ts
+++ b/src/language/providers/logic/callable.ts
@@ -17,7 +17,7 @@ export async function isCallableType(ref: ObjectRef, type: CallableType) {
     const callableRoutine = await DbCache.getType(ref.object.schema, ref.object.name, type);
 
     if (callableRoutine) {
-      DbCache.getSignaturesFor(ref.object.schema, ref.object.name, callableRoutine.specificNames);
+      await DbCache.getSignaturesFor(ref.object.schema, ref.object.name, callableRoutine.specificNames);
       return true;
     } else {
       // Not callable, let's just cache it as empty to stop spamming the db
diff --git a/src/language/providers/logic/completion.ts b/src/language/providers/logic/completion.ts
index 1aac7e20..504e750a 100644
--- a/src/language/providers/logic/completion.ts
+++ b/src/language/providers/logic/completion.ts
@@ -24,13 +24,19 @@ export function getParmAttributes(parm: SQLParm): string {
 }
 
 export function prepareParamType(param: TableColumn | SQLParm): string {
+  let baseType = param.DATA_TYPE;
+
   if (param.CHARACTER_MAXIMUM_LENGTH) {
-    return `${param.DATA_TYPE}(${param.CHARACTER_MAXIMUM_LENGTH})`;
+    baseType += `(${param.CHARACTER_MAXIMUM_LENGTH})`;
   }
 
   if (param.NUMERIC_PRECISION !== null && param.NUMERIC_SCALE !== null) {
-    return `${param.DATA_TYPE}(${param.NUMERIC_PRECISION}, ${param.NUMERIC_SCALE})`;
+    baseType += `(${param.NUMERIC_PRECISION}, ${param.NUMERIC_SCALE})`;
   }
 
-  return `${param.DATA_TYPE}`;
+  if ([`Y`, `YES`].includes(param.IS_NULLABLE)) {
+    baseType += ` nullable`;
+  };
+
+  return baseType;
 }

From 50688825257b2c4dbe3c867f84d5964cef8aa79d Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 15:23:57 -0400
Subject: [PATCH 11/28] Fix AST cache bug

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/logic/parse.ts | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/language/providers/logic/parse.ts b/src/language/providers/logic/parse.ts
index 4df09c3b..d9eeaecd 100644
--- a/src/language/providers/logic/parse.ts
+++ b/src/language/providers/logic/parse.ts
@@ -1,16 +1,21 @@
 import { TextDocument } from "vscode";
 import Document from "../../sql/document";
 
-let cachedAst: Document | undefined;
-let cachedVersion: number = -1;
+let cached: Map<string, {ast, version}> = new Map();
 
 export function getSqlDocument(document: TextDocument) {
-  if (cachedAst && cachedVersion === document.version) {
-    return cachedAst;
-  }
+  const uri = document.uri.toString();
+
+  if (cached.has(uri)) {
+    const { ast, version } = cached.get(uri)!;
 
-  cachedAst = new Document(document.getText());
-  cachedVersion = document.version;
+    if (version === document.version) {
+      return ast;
+    }
+  }
+  
+  const newAsp = new Document(document.getText());
+  cached.set(uri, { ast: newAsp, version: document.version });
 
-  return cachedAst;
+  return newAsp;
 }
\ No newline at end of file

From 3ab9959fd2632668cbba43cb0165b4f75e210461 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 16 Sep 2024 15:54:29 -0400
Subject: [PATCH 12/28] Open provider to fetch information

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts | 90 +++++++++++++++++++------
 src/language/providers/index.ts         |  5 +-
 src/language/providers/logic/parse.ts   |  5 +-
 3 files changed, 75 insertions(+), 25 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 187a9374..bfe59e5e 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -1,9 +1,60 @@
-import { Hover, languages, MarkdownString } from "vscode";
+import { Hover, languages, MarkdownString, workspace } from "vscode";
 import { getSqlDocument } from "./logic/parse";
 import { DbCache, LookupResult } from "./logic/cache";
 import { JobManager } from "../../config";
 import Statement from "../../database/statement";
 import { getParmAttributes, prepareParamType } from "./logic/completion";
+import { StatementType } from "../sql/types";
+import { remoteAssistIsEnabled } from "./logic/available";
+
+// =================================
+// We need to open provider to exist so symbols can be cached for hover support when opening files
+// =================================
+
+export const openProvider = workspace.onDidOpenTextDocument(async (document) => {
+  if (remoteAssistIsEnabled()) {
+    if (document.languageId === `sql`) {
+      const sqlDoc = getSqlDocument(document);
+      const defaultSchema = getDefaultSchema();
+
+      for (const statement of sqlDoc.statements) {
+        const refs = statement.getObjectReferences();
+        if (refs.length) {
+          if (statement.type === StatementType.Call) {
+            const first = refs[0];
+            if (first.object.name) {
+              const name = Statement.noQuotes(Statement.delimName(first.object.name, true));
+              const schema = Statement.noQuotes(Statement.delimName(first.object.schema || defaultSchema, true));
+              const result = await DbCache.getType(schema, name, `PROCEDURE`);
+              if (result) {
+                await DbCache.getRoutineResultColumns(schema, name);
+                await DbCache.getSignaturesFor(schema, name, result.specificNames);
+              }
+            }
+
+          } else {
+            for (const ref of refs) {
+              if (ref.object.name) {
+                const name = Statement.noQuotes(Statement.delimName(ref.object.name, true));
+                const schema = Statement.noQuotes(Statement.delimName(ref.object.schema || defaultSchema, true));
+                if (ref.isUDTF) {
+                  const result = await DbCache.getType(schema, name, `FUNCTION`);
+                  if (result) {
+                    await DbCache.getRoutineResultColumns(schema, name);
+                    await DbCache.getSignaturesFor(schema, name, result.specificNames);
+                  }
+                } else {
+                  await DbCache.getColumns(schema, name);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+  }
+});
 
 export const hoverProvider = languages.registerHoverProvider({ language: `sql` }, {
   async provideHover(document, position, token) {
@@ -22,31 +73,36 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
         const atRef = offset >= ref.tokens[0].range.start && offset <= ref.tokens[ref.tokens.length - 1].range.end;
 
         if (atRef) {
-          const result = lookupSymbol(ref.object.name, ref.object.schema || defaultSchema);
+          const schema = ref.object.schema || defaultSchema;
+          const result = lookupSymbol(ref.object.name, schema);
+          if (result) {
+            addSymbol(md, result);
+          }
+
+          if (systemSchemas.includes(schema.toUpperCase())) {
+            addSearch(md, ref.object.name, result !== undefined);
+          }
+        } else if (tokAt && tokAt.value) {
+          const result = lookupSymbol(tokAt.value, defaultSchema);
           if (result) {
             addSymbol(md, result);
-            return new Hover(md);
           }
         }
       }
     }
 
-    if (tokAt && tokAt.value) {
-      const result = lookupSymbol(tokAt.value, defaultSchema);
-      if (result) {
-        addSymbol(md, result);
-        return new Hover(md);
-      }
-    }
+    return md.value ? new Hover(md) : undefined;
   }
 });
 
 const systemSchemas = [`QSYS`, `QSYS2`, `SYSTOOLS`];
 
-function addSearch (base: MarkdownString, value: string) {
+function addSearch(base: MarkdownString, value: string, withGap = true) {
+  if (withGap) {
+    base.appendMarkdown([``, `---`, ``].join(`\n`));
+  }
+
   base.appendMarkdown([
-    ``,
-    `---`,
     `[Search on IBM Documentation](https://www.ibm.com/docs/en/search/${encodeURI(value)})`
   ].join(`\n\n`));
 }
@@ -65,10 +121,6 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
     for (const signature of symbol.signatures) {
       base.appendCodeblock(`${routineName}(${signature.parms.map(p => `${Statement.prettyName(p.PARAMETER_NAME)}${p.DEFAULT !== undefined ? `?` : ``}`).join(', ')})`, `sql`);
     }
-
-    if (systemSchemas.includes(symbol.routine.schema)) {
-      addSearch(base, symbol.routine.name);
-    }
   }
   else if ('PARAMETER_NAME' in symbol) {
     base.appendCodeblock(prepareParamType(symbol) + `\n`, `sql`);
@@ -80,10 +132,6 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
     addList(base, [
       `**Description:** ${symbol.text}`,
     ]);
-
-    if (systemSchemas.includes(symbol.schema)) {
-      addSearch(base, symbol.name);
-    }
   }
 }
 
diff --git a/src/language/providers/index.ts b/src/language/providers/index.ts
index e6febc13..07a54872 100644
--- a/src/language/providers/index.ts
+++ b/src/language/providers/index.ts
@@ -1,6 +1,6 @@
 import { completionProvider } from "./completionProvider";
 import { formatProvider } from "./formatProvider";
-import { hoverProvider } from "./hoverProvider";
+import { hoverProvider, openProvider } from "./hoverProvider";
 import { signatureProvider } from "./parameterProvider";
 
 export function languageInit() {
@@ -10,7 +10,8 @@ export function languageInit() {
     completionProvider,
     formatProvider,
     signatureProvider,
-    hoverProvider
+    hoverProvider,
+    openProvider
   );
   
   return functionality;
diff --git a/src/language/providers/logic/parse.ts b/src/language/providers/logic/parse.ts
index d9eeaecd..d95a70cd 100644
--- a/src/language/providers/logic/parse.ts
+++ b/src/language/providers/logic/parse.ts
@@ -3,10 +3,11 @@ import Document from "../../sql/document";
 
 let cached: Map<string, {ast, version}> = new Map();
 
-export function getSqlDocument(document: TextDocument) {
+export function getSqlDocument(document: TextDocument): Document {
   const uri = document.uri.toString();
+  const likelyNew = document.uri.scheme === `untitled` && document.version === 1;
 
-  if (cached.has(uri)) {
+  if (cached.has(uri) && !likelyNew) {
     const { ast, version } = cached.get(uri)!;
 
     if (version === document.version) {

From e773791efe583114d9ab133c7ae13d8d98a98938 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 10:07:23 -0400
Subject: [PATCH 13/28] Better type info

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index bfe59e5e..e1b42aed 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -27,7 +27,7 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
               const schema = Statement.noQuotes(Statement.delimName(first.object.schema || defaultSchema, true));
               const result = await DbCache.getType(schema, name, `PROCEDURE`);
               if (result) {
-                await DbCache.getRoutineResultColumns(schema, name);
+                await DbCache.getRoutineResultColumns(schema, name, true);
                 await DbCache.getSignaturesFor(schema, name, result.specificNames);
               }
             }
@@ -119,7 +119,7 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
   if ('routine' in symbol) {
     const routineName = Statement.prettyName(symbol.routine.name);
     for (const signature of symbol.signatures) {
-      base.appendCodeblock(`${routineName}(${signature.parms.map(p => `${Statement.prettyName(p.PARAMETER_NAME)}${p.DEFAULT !== undefined ? `?` : ``}`).join(', ')})`, `sql`);
+      base.appendCodeblock(`${routineName}(\n${signature.parms.map(p => `  ${Statement.prettyName(p.PARAMETER_NAME)}${p.DEFAULT !== undefined ? `?` : ``}: ${prepareParamType(p)}`).join(',\n')}\n)`, `sql`);
     }
   }
   else if ('PARAMETER_NAME' in symbol) {

From ef68c53e908378272525e966a95e521bb1293aab Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 10:13:42 -0400
Subject: [PATCH 14/28] Fix bug where columns were not being resolves

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index e1b42aed..3fb1578f 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -40,7 +40,7 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
                 if (ref.isUDTF) {
                   const result = await DbCache.getType(schema, name, `FUNCTION`);
                   if (result) {
-                    await DbCache.getRoutineResultColumns(schema, name);
+                    await DbCache.getRoutineResultColumns(schema, name, true);
                     await DbCache.getSignaturesFor(schema, name, result.specificNames);
                   }
                 } else {

From 4cc2126dea47307c65ceb015bf7c39c2a6ff37ff Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 10:19:16 -0400
Subject: [PATCH 15/28] Props aren't really optional

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 global.d.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/global.d.ts b/global.d.ts
index e5762a4f..3fd47376 100644
--- a/global.d.ts
+++ b/global.d.ts
@@ -1,6 +1,6 @@
 interface TableColumn {
-  TABLE_SCHEMA?: string,
-  TABLE_NAME?: string,
+  TABLE_SCHEMA: string,
+  TABLE_NAME: string,
   COLUMN_NAME: string,
   SYSTEM_COLUMN_NAME: string,
   CONSTRAINT_NAME?: string,

From 86fb7bda001ec7f0f5f36d2a52f702731a0b0d19 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 10:37:10 -0400
Subject: [PATCH 16/28] Improved formatting for hover assist

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts    | 2 +-
 src/language/providers/logic/completion.ts | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 3fb1578f..46aa85ce 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -119,7 +119,7 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
   if ('routine' in symbol) {
     const routineName = Statement.prettyName(symbol.routine.name);
     for (const signature of symbol.signatures) {
-      base.appendCodeblock(`${routineName}(\n${signature.parms.map(p => `  ${Statement.prettyName(p.PARAMETER_NAME)}${p.DEFAULT !== undefined ? `?` : ``}: ${prepareParamType(p)}`).join(',\n')}\n)`, `sql`);
+      base.appendCodeblock(`${routineName}(\n${signature.parms.map(p => `  ${Statement.prettyName(p.PARAMETER_NAME)} => ${prepareParamType(p)}`).join(',\n')}\n)`, `sql`);
     }
   }
   else if ('PARAMETER_NAME' in symbol) {
diff --git a/src/language/providers/logic/completion.ts b/src/language/providers/logic/completion.ts
index 504e750a..3eb4c399 100644
--- a/src/language/providers/logic/completion.ts
+++ b/src/language/providers/logic/completion.ts
@@ -24,7 +24,7 @@ export function getParmAttributes(parm: SQLParm): string {
 }
 
 export function prepareParamType(param: TableColumn | SQLParm): string {
-  let baseType = param.DATA_TYPE;
+  let baseType = param.DATA_TYPE.toLowerCase();
 
   if (param.CHARACTER_MAXIMUM_LENGTH) {
     baseType += `(${param.CHARACTER_MAXIMUM_LENGTH})`;
@@ -34,8 +34,8 @@ export function prepareParamType(param: TableColumn | SQLParm): string {
     baseType += `(${param.NUMERIC_PRECISION}, ${param.NUMERIC_SCALE})`;
   }
 
-  if ([`Y`, `YES`].includes(param.IS_NULLABLE)) {
-    baseType += ` nullable`;
+  if ([`Y`, `YES`].includes(param.IS_NULLABLE) || ('DEFAULT' in param && param.DEFAULT !== null)) {
+    baseType += ` optional`;
   };
 
   return baseType;

From 4d9dde6833b9802fb5b1c1159123b8a7460cc7fa Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 11:26:25 -0400
Subject: [PATCH 17/28] Improvements to signature support

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 global.d.ts                                  |  1 +
 src/database/callable.ts                     | 44 ++--------------
 src/language/providers/completionProvider.ts |  8 +--
 src/language/providers/hoverProvider.ts      | 29 +++++++----
 src/language/providers/logic/cache.ts        | 53 ++++++++------------
 5 files changed, 50 insertions(+), 85 deletions(-)

diff --git a/global.d.ts b/global.d.ts
index 3fd47376..2195ab13 100644
--- a/global.d.ts
+++ b/global.d.ts
@@ -28,6 +28,7 @@ interface SQLParm {
   DEFAULT?: string,
   LONG_COMMENT?: string,
   ORDINAL_POSITION: number,
+  ROW_TYPE: "P" | "R",
 }
 
 interface BasicSQLObject {
diff --git a/src/database/callable.ts b/src/database/callable.ts
index bb158e6c..60c17bf3 100644
--- a/src/database/callable.ts
+++ b/src/database/callable.ts
@@ -15,6 +15,7 @@ export interface CallableRoutine {
 export interface CallableSignature {
   specificName: string;
   parms: SQLParm[];
+  returns: SQLParm[];
 }
 
 export default class Callable {
@@ -45,7 +46,7 @@ export default class Callable {
     const results = await JobManager.runSQL<SQLParm>(
       [
         `SELECT * FROM QSYS2.SYSPARMS`,
-        `WHERE SPECIFIC_SCHEMA = ? AND ROW_TYPE = 'P' AND SPECIFIC_NAME in (${specificNames.map(n => `?`).join(`, `)})`,
+        `WHERE SPECIFIC_SCHEMA = ? AND ROW_TYPE in ('P', 'R') AND SPECIFIC_NAME in (${specificNames.map(n => `?`).join(`, `)})`,
         `ORDER BY ORDINAL_POSITION`
       ].join(` `),
       {
@@ -60,48 +61,11 @@ export default class Callable {
     const groupedResults: CallableSignature[] = uniqueSpecificNames.map(name => {
       return {
         specificName: name,
-        parms: results.filter(row => row.SPECIFIC_NAME === name)
+        parms: results.filter(row => row.SPECIFIC_NAME === name && row.ROW_TYPE === `P`),
+        returns: results.filter(row => row.SPECIFIC_NAME === name && row.ROW_TYPE === `R`)
       }
     });
 
     return groupedResults;
   }
-
-  /**
-   * @param schema Not user input
-   * @param specificName Not user input
-   * @returns 
-   */
-  static getParms(schema: string, specificName: string, resolveName: boolean = false): Promise<SQLParm[]> {
-    const rowType = `P`; // Parameter
-    return Callable.getFromSysParms(schema, specificName, rowType, resolveName);
-  }
-
-  static getResultColumns(schema: string, specificName: string, resolveName: boolean = false) {
-    const rowType = `R`; // Row
-    return Callable.getFromSysParms(schema, specificName, rowType, resolveName);
-  }
-
-  static getFromSysParms(schema: string, name: string, rowType: "P"|"R", resolveName: boolean = false): Promise<SQLParm[]> {
-    let parameters = [schema, rowType];
-
-    let specificNameClause = undefined;
-
-    if (resolveName) {
-      specificNameClause = `SPECIFIC_NAME = (select specific_name from qsys2.sysroutines where specific_schema = ? and routine_name = ?)`;
-      parameters.push(schema, name);
-    } else {
-      specificNameClause = `SPECIFIC_NAME = ?`;
-      parameters.push(name);
-    }
-
-    return JobManager.runSQL<SQLParm>(
-      [
-        `SELECT * FROM QSYS2.SYSPARMS`,
-        `WHERE SPECIFIC_SCHEMA = ? AND ROW_TYPE = ? AND ${specificNameClause}`,
-        `ORDER BY ORDINAL_POSITION`
-      ].join(` `),
-      { parameters }
-    );
-  }
 }
\ No newline at end of file
diff --git a/src/language/providers/completionProvider.ts b/src/language/providers/completionProvider.ts
index b19c9cc0..ae605a83 100644
--- a/src/language/providers/completionProvider.ts
+++ b/src/language/providers/completionProvider.ts
@@ -94,13 +94,15 @@ async function getObjectColumns(
     name = Statement.noQuotes(Statement.delimName(name, true));
     
   if (isUDTF) {
-    const resultSet = await DbCache.getRoutineResultColumns(schema, name, true);
+    const resultSet = await DbCache.getCachedSignatures(schema, name);
     
     if (!resultSet?.length ? true : false) {
       return [];
     }
+
+    const chosenSig = resultSet[resultSet.length-1];
     
-    completionItems = resultSet.map((i) =>
+    completionItems = chosenSig.returns.map((i) =>
       createCompletionItem(
         Statement.prettyName(i.PARAMETER_NAME),
         CompletionItemKind.Field,
@@ -111,7 +113,7 @@ async function getObjectColumns(
     );
 
   } else {
-    const columns = await DbCache.getColumns(schema, name);
+    const columns = await DbCache.getObjectColumns(schema, name);
 
     if (!columns?.length ? true : false) {
       return [];
diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 46aa85ce..d0cabdb5 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -27,7 +27,6 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
               const schema = Statement.noQuotes(Statement.delimName(first.object.schema || defaultSchema, true));
               const result = await DbCache.getType(schema, name, `PROCEDURE`);
               if (result) {
-                await DbCache.getRoutineResultColumns(schema, name, true);
                 await DbCache.getSignaturesFor(schema, name, result.specificNames);
               }
             }
@@ -40,11 +39,10 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
                 if (ref.isUDTF) {
                   const result = await DbCache.getType(schema, name, `FUNCTION`);
                   if (result) {
-                    await DbCache.getRoutineResultColumns(schema, name, true);
                     await DbCache.getSignaturesFor(schema, name, result.specificNames);
                   }
                 } else {
-                  await DbCache.getColumns(schema, name);
+                  await DbCache.getObjectColumns(schema, name);
                 }
               }
             }
@@ -69,12 +67,14 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
     if (statementAt) {
       const refs = statementAt.getObjectReferences();
+      const possibleNames = refs.map(ref => ref.object.name).filter(name => name);
+
       for (const ref of refs) {
         const atRef = offset >= ref.tokens[0].range.start && offset <= ref.tokens[ref.tokens.length - 1].range.end;
 
         if (atRef) {
           const schema = ref.object.schema || defaultSchema;
-          const result = lookupSymbol(ref.object.name, schema);
+          const result = lookupSymbol(ref.object.name, schema, possibleNames);
           if (result) {
             addSymbol(md, result);
           }
@@ -82,8 +82,8 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
           if (systemSchemas.includes(schema.toUpperCase())) {
             addSearch(md, ref.object.name, result !== undefined);
           }
-        } else if (tokAt && tokAt.value) {
-          const result = lookupSymbol(tokAt.value, defaultSchema);
+        } else if (tokAt && tokAt.type === `word` && tokAt.value) {
+          const result = lookupSymbol(tokAt.value, defaultSchema, possibleNames);
           if (result) {
             addSymbol(md, result);
           }
@@ -118,8 +118,17 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
 
   if ('routine' in symbol) {
     const routineName = Statement.prettyName(symbol.routine.name);
-    for (const signature of symbol.signatures) {
-      base.appendCodeblock(`${routineName}(\n${signature.parms.map(p => `  ${Statement.prettyName(p.PARAMETER_NAME)} => ${prepareParamType(p)}`).join(',\n')}\n)`, `sql`);
+    for (let i = 0; i < symbol.signatures.length; i++) {
+      const signature = symbol.signatures[i];
+      const returns = signature.returns.length > 0 ? `: ${signature.returns.length} column${signature.returns.length === 1 ? `` : `s`}` : '';
+
+      base.appendCodeblock(`${routineName}(\n${signature.parms.map((p, pI) => {
+        return `  ${Statement.prettyName(p.PARAMETER_NAME || `parm${pI}`)} => ${prepareParamType(p)}`
+      }).join(',\n')}\n)${returns}`, `sql`);
+
+      if (i < symbol.signatures.length - 1) {
+        base.appendMarkdown(`\n---\n`);
+      }
     }
   }
   else if ('PARAMETER_NAME' in symbol) {
@@ -135,11 +144,11 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
   }
 }
 
-function lookupSymbol(name: string, schema: string) {
+function lookupSymbol(name: string, schema: string, possibleNames: string[]) {
   name = Statement.noQuotes(Statement.delimName(name, true));
   schema = Statement.noQuotes(Statement.delimName(schema, true));
 
-  return DbCache.lookupSymbol(name, schema);
+  return DbCache.lookupSymbol(name, schema, possibleNames);
 }
 
 const getDefaultSchema = (): string => {
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index e9fafa05..72edbee7 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -5,14 +5,12 @@ import Table from "../../../database/table";
 interface RoutineDetail {
   routine: CallableRoutine;
   signatures: CallableSignature[];
-  resultingColumns: SQLParm[];
 }
 
 export type LookupResult = RoutineDetail | SQLParm | BasicSQLObject | TableColumn;
 
 export class DbCache {
   private static schemaObjects: Map<string, BasicSQLObject[]> = new Map();
-  private static routineResultColumns: Map<string, SQLParm[]> = new Map();
   private static objectColumns: Map<string, TableColumn[]> = new Map();
   private static routines: Map<string, CallableRoutine|false> = new Map();
   private static routineSignatures: Map<string, CallableSignature[]> = new Map();
@@ -20,7 +18,6 @@ export class DbCache {
   private static toReset: string[] = [];
 
   static async resetCache() {
-    this.routineResultColumns.clear();
     this.objectColumns.clear();
     this.schemaObjects.clear();
     this.toReset = [];
@@ -43,17 +40,25 @@ export class DbCache {
     return false;
   }
 
-  static lookupSymbol(name: string, schema: string): LookupResult {
+  static lookupSymbol(name: string, schema: string, objectFilter: string[]): LookupResult {
     const routine = this.getCachedType(schema, name, `FUNCTION`) || this.getCachedType(schema, name, `PROCEDURE`);
     if (routine) {
       const signatures = this.getCachedSignatures(schema, name);
-      const resultingColumns = this.getCachedRoutineResultColumns(schema, name);
-      return { routine, signatures, resultingColumns } as RoutineDetail;
+      return { routine, signatures } as RoutineDetail;
+    }
+
+    objectFilter = objectFilter.map(o => o.toLowerCase());
+
+    const included = (name: string) => {
+      if (objectFilter) {
+        return objectFilter.includes(name.toLowerCase());
+      }
+      return true;
     }
 
     // Search objects
     for (const currentSchema of this.schemaObjects.values()) {
-      const chosenObject = currentSchema.find(column => column.name === name && column.schema === schema);
+      const chosenObject = currentSchema.find(sqlObject => included(sqlObject.name) && sqlObject.name === name && sqlObject.schema === schema);
       if (chosenObject) {
         return chosenObject;
       }
@@ -63,43 +68,27 @@ export class DbCache {
 
     // First object columns
     for (const currentObject of this.objectColumns.values()) {
-      const chosenColumn = currentObject.find(column => column.COLUMN_NAME.toLowerCase() === name.toLowerCase());
+      const chosenColumn = currentObject.find(column => included(column.TABLE_NAME) && column.COLUMN_NAME.toLowerCase() === name.toLowerCase());
       if (chosenColumn) {
         return chosenColumn;
       }
     }
 
     // Then by routine result columns
-    for (const currentRoutine of this.routineResultColumns.values()) {
-      const chosenColumn = currentRoutine.find(column => column.PARAMETER_NAME.toLowerCase() === name.toLowerCase());
-      if (chosenColumn) {
-        return chosenColumn;
+    for (const currentRoutineSig of this.routineSignatures.values()) {
+      for (const signature of currentRoutineSig) {
+        const chosenColumn = signature.returns.find(column => column.PARAMETER_NAME.toLowerCase() === name.toLowerCase());
+        if (chosenColumn) {
+          return chosenColumn;
+        }
       }
     }
   }
 
-  static async getRoutineResultColumns(schema: string, name: string, resolveName?: boolean) {
-    const key = getKey(`routine`, schema, name);
-    
-    if (!this.routineResultColumns.has(key) || this.shouldReset(name)) {
-      const result = await Callable.getResultColumns(schema, name, resolveName);
-      if (result) {
-        this.routineResultColumns.set(key, result);
-      }
-    }
-
-    return this.routineResultColumns.get(key) || [];
-  }
-
-  static getCachedRoutineResultColumns(schema: string, name: string) {
-    const key = getKey(`routine`, schema, name);
-    return this.routineResultColumns.get(key) || [];
-  }
-
-  static async getColumns(schema: string, name: string) {
+  static async getObjectColumns(schema: string, name: string) {
     const key = getKey(`columns`, schema, name);
     
-    if (!this.routineResultColumns.has(key) || this.shouldReset(name)) {
+    if (!this.objectColumns.has(key) || this.shouldReset(name)) {
       const result = await Table.getItems(schema, name);
       if (result) {
         this.objectColumns.set(key, result);

From 1aa2718e8d2e246c815e491c31a12e86d2894dad Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 13:00:09 -0400
Subject: [PATCH 18/28] Only fallback when no initial lookup is done

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index d0cabdb5..8b63a65d 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -82,11 +82,14 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
           if (systemSchemas.includes(schema.toUpperCase())) {
             addSearch(md, ref.object.name, result !== undefined);
           }
-        } else if (tokAt && tokAt.type === `word` && tokAt.value) {
-          const result = lookupSymbol(tokAt.value, defaultSchema, possibleNames);
-          if (result) {
-            addSymbol(md, result);
-          }
+        }
+      }
+
+      // If no symbol found, check if we can find a symbol by name
+      if (md.value.length === 0 && tokAt && tokAt.type === `word` && tokAt.value) {
+        const result = lookupSymbol(tokAt.value, defaultSchema, possibleNames);
+        if (result) {
+          addSymbol(md, result);
         }
       }
     }

From f60ff4738281ffc700dafed9cb1fcb45e2c778c6 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 13:08:32 -0400
Subject: [PATCH 19/28] Swap if logic

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 8b63a65d..6cfc13f8 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -12,8 +12,8 @@ import { remoteAssistIsEnabled } from "./logic/available";
 // =================================
 
 export const openProvider = workspace.onDidOpenTextDocument(async (document) => {
-  if (remoteAssistIsEnabled()) {
-    if (document.languageId === `sql`) {
+  if (document.languageId === `sql`) {
+    if (remoteAssistIsEnabled()) {
       const sqlDoc = getSqlDocument(document);
       const defaultSchema = getDefaultSchema();
 

From ac4c7d64e963ab1c3a9af5a32e92cc46585b2dff Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 14:06:03 -0400
Subject: [PATCH 20/28] Better support for CTE references

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/statement.ts             | 26 ++++------
 src/language/sql/tests/statements.test.ts | 61 +++++++++++++++++++++--
 2 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts
index 55944910..f333f98b 100644
--- a/src/language/sql/statement.ts
+++ b/src/language/sql/statement.ts
@@ -26,9 +26,6 @@ export default class Statement {
 		}
 		
 		switch (this.type) {
-			case StatementType.With:
-				this.tokens = SQLTokeniser.createBlocks(this.tokens);
-				break;
 			case StatementType.Create:
 				// No scalar transformation here..
 				break;
@@ -213,23 +210,25 @@ export default class Statement {
 
 	getCTEReferences(): CTEReference[] {
 		if (this.type !== StatementType.With) return [];
+
+		const withBlocks = SQLTokeniser.createBlocks(this.tokens.slice(0));
 		
 		let cteList: CTEReference[] = [];
 
-		for (let i = 0; i < this.tokens.length; i++) {
-			if (tokenIs(this.tokens[i], `word`)) {
-				let cteName = this.tokens[i].value!;
+		for (let i = 0; i < withBlocks.length; i++) {
+			if (tokenIs(withBlocks[i], `word`) || tokenIs(withBlocks[i], `function`)) {
+				let cteName = withBlocks[i].value!;
 				let parameters: string[] = [];
 				let statementBlockI = i+1;
 
-				if (tokenIs(this.tokens[i+1], `block`) && tokenIs(this.tokens[i+2], `keyword`, `AS`)) {
-					parameters = this.tokens[i+1].block!.filter(blockToken => blockToken.type === `word`).map(blockToken => blockToken.value!)
+				if (tokenIs(withBlocks[i+1], `block`) && tokenIs(withBlocks[i+2], `keyword`, `AS`)) {
+					parameters = withBlocks[i+1].block!.filter(blockToken => blockToken.type === `word`).map(blockToken => blockToken.value!)
 					statementBlockI = i+3;
-				} else if (tokenIs(this.tokens[i+1], `keyword`, `AS`)) {
+				} else if (tokenIs(withBlocks[i+1], `keyword`, `AS`)) {
 					statementBlockI = i+2;
 				}
 
-				const statementBlock = this.tokens[statementBlockI];
+				const statementBlock = withBlocks[statementBlockI];
 				if (tokenIs(statementBlock, `block`)) {
 					cteList.push({
 						name: cteName,
@@ -241,7 +240,7 @@ export default class Statement {
 				i = statementBlockI;
 			}
 
-			if (tokenIs(this.tokens[i], `statementType`, `SELECT`)) {
+			if (tokenIs(withBlocks[i], `statementType`, `SELECT`)) {
 				break;
 			}
 		}
@@ -513,11 +512,8 @@ export default class Statement {
 
 		let endIndex = i;
 
-		let isUDTF = false;
+		let isUDTF = tokenIs(nextToken, `function`, `TABLE`);
 
-		if (tokenIs(nextToken, `function`, `TABLE`)) {
-			isUDTF = true;
-		}
 
 		if (isUDTF) {
 			sqlObj = this.getRefAtToken(i+2);
diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts
index a3d7ed4a..045ac062 100644
--- a/src/language/sql/tests/statements.test.ts
+++ b/src/language/sql/tests/statements.test.ts
@@ -943,6 +943,35 @@ describe(`Object references`, () => {
     expect(refs[0].object.name).toBe(`apiVersion`);
     expect(refs[0].createType).toBe(`varchar(10) ccsid 1208 default '2023-07-07'`);
   });
+
+  test(`SELECT, WITH & LATERAL`, () => {
+    const lines = [
+      `with qsysobjs (lib, obj, type) as (`,
+      `  select object_library, object_name, object_type`,
+      `    from table (qsys2.object_ownership(current_user))`,
+      `    where path_name is null`,
+      `)`,
+      `select lib concat '/' concat obj concat ' (' concat type concat ')' as label,`,
+      `       objsize as "Size"`,
+      `  from qsysobjs q, lateral (`,
+      `         select objcreated, last_used_timestamp, objsize`,
+      `           from table (qsys2.object_statistics(lib, type, obj))`,
+      `       ) z`,
+      `where objsize is not null`,
+      `order by OBJSIZE DESC`,
+      `limit 10;`,
+    ].join(`\n`);
+
+    const document = new Document(lines);
+
+    expect(document.statements.length).toBe(1);
+
+    const statement = document.statements[0];
+
+    expect(statement.type).toBe(StatementType.With);
+
+    const refs = statement.getObjectReferences();
+  });
 });
 
 describe(`Offset reference tests`, () => {
@@ -1138,8 +1167,28 @@ describe(`PL body tests`, () => {
     const refs = statement.getObjectReferences();
     const ctes = statement.getCTEReferences();
 
-    expect(refs.length).toBe(1);
-    expect(refs[0].object.name).toBe(`Temp02`);
+    console.log(refs);
+    expect(refs.length).toBe(7);
+    expect(refs[0].object.name).toBe(`shipments`);
+    expect(refs[0].alias).toBe(`s`);
+
+    expect(refs[1].object.name).toBe(`BillingDate`);
+    expect(refs[1].alias).toBeUndefined();
+
+    expect(refs[2].object.name).toBe(`Temp01`);
+    expect(refs[2].alias).toBe(`t1`);
+
+    expect(refs[3].object.name).toBe(`Temp01`);
+    expect(refs[3].alias).toBe(`t1`);
+
+    expect(refs[4].object.name).toBe(`Temp02`);
+    expect(refs[4].alias).toBe(`t2`);
+
+    expect(refs[5].object.name).toBe(`customers`);
+    expect(refs[5].alias).toBe(`c`);
+
+    expect(refs[6].object.name).toBe(`Temp02`);
+    expect(refs[6].alias).toBeUndefined();
 
     expect(ctes.length).toBe(3);
     expect(ctes[0].name).toBe(`Temp01`);
@@ -1174,9 +1223,11 @@ describe(`PL body tests`, () => {
     expect(statement.type).toBe(StatementType.With);
 
     const objs = statement.getObjectReferences();
-    expect(objs.length).toBe(1);
-    expect(objs[0].object.schema).toBe(undefined);
-    expect(objs[0].object.name).toBe(`cteme`);
+    expect(objs.length).toBe(2);
+    expect(objs[0].object.schema).toBe(`qsys2`);
+    expect(objs[0].object.name).toBe(`sysixadv`);
+    expect(objs[1].object.schema).toBe(undefined);
+    expect(objs[1].object.name).toBe(`cteme`);
 
     const ctes = statement.getCTEReferences();
     expect(ctes.length).toBe(1);

From ee4f495f65c80cdb7519ce6ac25c38fcde354c2c Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 14:19:04 -0400
Subject: [PATCH 21/28] Improvements for looking up pasted code

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts  | 14 ++++----
 src/language/providers/logic/cache.ts    | 46 ++++++++++++++----------
 src/language/providers/logic/callable.ts |  2 +-
 3 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 6cfc13f8..6a67412e 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -25,7 +25,7 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
             if (first.object.name) {
               const name = Statement.noQuotes(Statement.delimName(first.object.name, true));
               const schema = Statement.noQuotes(Statement.delimName(first.object.schema || defaultSchema, true));
-              const result = await DbCache.getType(schema, name, `PROCEDURE`);
+              const result = await DbCache.getRoutine(schema, name, `PROCEDURE`);
               if (result) {
                 await DbCache.getSignaturesFor(schema, name, result.specificNames);
               }
@@ -37,7 +37,7 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
                 const name = Statement.noQuotes(Statement.delimName(ref.object.name, true));
                 const schema = Statement.noQuotes(Statement.delimName(ref.object.schema || defaultSchema, true));
                 if (ref.isUDTF) {
-                  const result = await DbCache.getType(schema, name, `FUNCTION`);
+                  const result = await DbCache.getRoutine(schema, name, `FUNCTION`);
                   if (result) {
                     await DbCache.getSignaturesFor(schema, name, result.specificNames);
                   }
@@ -74,9 +74,11 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
         if (atRef) {
           const schema = ref.object.schema || defaultSchema;
-          const result = lookupSymbol(ref.object.name, schema, possibleNames);
+          const result = await lookupSymbol(ref.object.name, schema, possibleNames);
           if (result) {
             addSymbol(md, result);
+          } else {
+            
           }
 
           if (systemSchemas.includes(schema.toUpperCase())) {
@@ -87,7 +89,7 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
       // If no symbol found, check if we can find a symbol by name
       if (md.value.length === 0 && tokAt && tokAt.type === `word` && tokAt.value) {
-        const result = lookupSymbol(tokAt.value, defaultSchema, possibleNames);
+        const result = await lookupSymbol(tokAt.value, undefined, possibleNames);
         if (result) {
           addSymbol(md, result);
         }
@@ -147,9 +149,9 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
   }
 }
 
-function lookupSymbol(name: string, schema: string, possibleNames: string[]) {
+function lookupSymbol(name: string, schema: string|undefined, possibleNames: string[]) {
   name = Statement.noQuotes(Statement.delimName(name, true));
-  schema = Statement.noQuotes(Statement.delimName(schema, true));
+  schema = schema ? Statement.noQuotes(Statement.delimName(schema, true)) : undefined
 
   return DbCache.lookupSymbol(name, schema, possibleNames);
 }
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index 72edbee7..245c81bf 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -40,27 +40,37 @@ export class DbCache {
     return false;
   }
 
-  static lookupSymbol(name: string, schema: string, objectFilter: string[]): LookupResult {
-    const routine = this.getCachedType(schema, name, `FUNCTION`) || this.getCachedType(schema, name, `PROCEDURE`);
-    if (routine) {
-      const signatures = this.getCachedSignatures(schema, name);
-      return { routine, signatures } as RoutineDetail;
-    }
-
-    objectFilter = objectFilter.map(o => o.toLowerCase());
-
-    const included = (name: string) => {
+  static async lookupSymbol(name: string, schema: string|undefined, objectFilter: string[]): Promise<LookupResult> {
+    const included = (lookupName: string) => {
       if (objectFilter) {
-        return objectFilter.includes(name.toLowerCase());
+        return objectFilter.includes(lookupName.toLowerCase());
       }
       return true;
     }
 
-    // Search objects
-    for (const currentSchema of this.schemaObjects.values()) {
-      const chosenObject = currentSchema.find(sqlObject => included(sqlObject.name) && sqlObject.name === name && sqlObject.schema === schema);
-      if (chosenObject) {
-        return chosenObject;
+    if (schema) {
+      // Looking routine
+      const routine = this.getCachedRoutine(schema, name, `FUNCTION`) || this.getCachedRoutine(schema, name, `PROCEDURE`);
+      if (routine) {
+        const signatures = this.getCachedSignatures(schema, name);
+        return { routine, signatures } as RoutineDetail;
+      }
+
+      objectFilter = objectFilter.map(o => o.toLowerCase());
+
+      // Search objects
+      for (const currentSchema of this.schemaObjects.values()) {
+        const chosenObject = currentSchema.find(sqlObject => included(sqlObject.name) && sqlObject.name === name && sqlObject.schema === schema);
+        if (chosenObject) {
+          return chosenObject;
+        }
+      }
+
+      // Finally, let's do a last lookup
+      const lookupRoutine = await this.getRoutine(schema, name, `FUNCTION`) || await this.getRoutine(schema, name, `PROCEDURE`);
+      if (lookupRoutine) {
+        const signatures = await this.getSignaturesFor(schema, name, lookupRoutine.specificNames);
+        return { routine: lookupRoutine, signatures } as RoutineDetail;
       }
     }
 
@@ -111,7 +121,7 @@ export class DbCache {
     return this.schemaObjects.get(key) || [];
   }
 
-  static async getType(schema: string, name: string, type: CallableType) {
+  static async getRoutine(schema: string, name: string, type: CallableType) {
     const key = getKey(type, schema, name);
     
     if (!this.routines.has(key) || this.shouldReset(name)) {
@@ -127,7 +137,7 @@ export class DbCache {
     return this.routines.get(key) || undefined;
   }
 
-  static getCachedType(schema: string, name: string, type: CallableType) {
+  static getCachedRoutine(schema: string, name: string, type: CallableType) {
     const key = getKey(type, schema, name);
     return this.routines.get(key) || undefined
   }
diff --git a/src/language/providers/logic/callable.ts b/src/language/providers/logic/callable.ts
index 264a8bfc..d6f0ca8e 100644
--- a/src/language/providers/logic/callable.ts
+++ b/src/language/providers/logic/callable.ts
@@ -14,7 +14,7 @@ export async function isCallableType(ref: ObjectRef, type: CallableType) {
     ref.object.schema = Statement.noQuotes(Statement.delimName(ref.object.schema, true));
     ref.object.name = Statement.noQuotes(Statement.delimName(ref.object.name, true));
 
-    const callableRoutine = await DbCache.getType(ref.object.schema, ref.object.name, type);
+    const callableRoutine = await DbCache.getRoutine(ref.object.schema, ref.object.name, type);
 
     if (callableRoutine) {
       await DbCache.getSignaturesFor(ref.object.schema, ref.object.name, callableRoutine.specificNames);

From 71a75b6191b2043a304c72176d2f90969aae8d7e Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 16:04:55 -0400
Subject: [PATCH 22/28] Improved routine detail

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 global.d.ts                                |  2 +
 src/language/providers/hoverProvider.ts    | 78 ++++++++++++++++------
 src/language/providers/logic/cache.ts      |  2 +-
 src/language/providers/logic/completion.ts |  6 +-
 4 files changed, 63 insertions(+), 25 deletions(-)

diff --git a/global.d.ts b/global.d.ts
index 2195ab13..372da004 100644
--- a/global.d.ts
+++ b/global.d.ts
@@ -1,3 +1,4 @@
+// https://www.ibm.com/docs/en/i/7.4?topic=views-syscolumns2
 interface TableColumn {
   TABLE_SCHEMA: string,
   TABLE_NAME: string,
@@ -15,6 +16,7 @@ interface TableColumn {
   IS_IDENTITY: "YES" | "NO",
 }
 
+// https://www.ibm.com/docs/en/i/7.4?topic=views-sysparms
 interface SQLParm {
   SPECIFIC_SCHEMA: string,
   SPECIFIC_NAME: string,
diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 6a67412e..96d11d77 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -1,11 +1,13 @@
 import { Hover, languages, MarkdownString, workspace } from "vscode";
 import { getSqlDocument } from "./logic/parse";
-import { DbCache, LookupResult } from "./logic/cache";
+import { DbCache, LookupResult, RoutineDetail } from "./logic/cache";
 import { JobManager } from "../../config";
 import Statement from "../../database/statement";
 import { getParmAttributes, prepareParamType } from "./logic/completion";
 import { StatementType } from "../sql/types";
 import { remoteAssistIsEnabled } from "./logic/available";
+import { getPositionData } from "./logic/callable";
+import { CallableSignature } from "../../database/callable";
 
 // =================================
 // We need to open provider to exist so symbols can be cached for hover support when opening files
@@ -56,6 +58,8 @@ export const openProvider = workspace.onDidOpenTextDocument(async (document) =>
 
 export const hoverProvider = languages.registerHoverProvider({ language: `sql` }, {
   async provideHover(document, position, token) {
+    if (!remoteAssistIsEnabled()) return;
+    
     const defaultSchema = getDefaultSchema();
     const sqlDoc = getSqlDocument(document);
     const offset = document.offsetAt(position);
@@ -75,10 +79,23 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
         if (atRef) {
           const schema = ref.object.schema || defaultSchema;
           const result = await lookupSymbol(ref.object.name, schema, possibleNames);
+
+
           if (result) {
-            addSymbol(md, result);
-          } else {
-            
+            if ('routine' in result) {
+              const callableRef = statementAt.getCallableDetail(offset, false);
+              if (callableRef) {
+                const { currentCount } = getPositionData(callableRef, offset);
+                const signatures = await DbCache.getCachedSignatures(callableRef.parentRef.object.schema, callableRef.parentRef.object.name);
+                const possibleSignatures = signatures.filter((s) => s.parms.length >= currentCount).sort((a, b) => a.parms.length - b.parms.length);
+                const signature = possibleSignatures.find((signature) => currentCount <= signature.parms.length);
+                if (signature) {
+                  addRoutineMd(md, signature, result);
+                }
+              }
+            } else {
+              addSymbol(md, result);
+            }
           }
 
           if (systemSchemas.includes(schema.toUpperCase())) {
@@ -102,6 +119,39 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
 const systemSchemas = [`QSYS`, `QSYS2`, `SYSTOOLS`];
 
+function addRoutineMd(base: MarkdownString, signature: CallableSignature, result: RoutineDetail) {
+  const returns = signature.returns.length > 0 ? `: ${signature.returns.length} column${signature.returns.length === 1 ? `` : `s`}` : '';
+
+  let codeLines: string[] = [`${Statement.prettyName(result.routine.name)}(`];
+
+  for (let i = 0; i < signature.parms.length; i++) {
+    const parm = signature.parms[i];
+    let parmString = `  ${Statement.prettyName(parm.PARAMETER_NAME || `parm${i + 1}`)} => ${prepareParamType(parm)}`;
+
+    if (i < signature.parms.length - 1) {
+      parmString += `,`;
+    }
+
+    codeLines.push(parmString);
+  }
+
+  codeLines.push(`)${returns}`);
+
+  base.appendCodeblock(codeLines.join(`\n`), `sql`);
+
+  let parmDetail = [``, `---`];
+
+  for (let i = 0; i < signature.parms.length; i++) {
+    const parm = signature.parms[i];
+    const escapedAsterisk = parm.LONG_COMMENT ? parm.LONG_COMMENT.replace(/\*/g, `\\*`) : ``;
+    parmDetail.push(``, `*@param* \`${Statement.prettyName(parm.PARAMETER_NAME || `parm${i}`)}\` ${escapedAsterisk}`);
+  }
+
+  parmDetail.push(``);
+
+  base.appendMarkdown(parmDetail.join(`\n`));
+}
+
 function addSearch(base: MarkdownString, value: string, withGap = true) {
   if (withGap) {
     base.appendMarkdown([``, `---`, ``].join(`\n`));
@@ -120,23 +170,7 @@ function addList(base: MarkdownString, items: string[]) {
 
 function addSymbol(base: MarkdownString, symbol: LookupResult) {
   base.isTrusted = true;
-
-  if ('routine' in symbol) {
-    const routineName = Statement.prettyName(symbol.routine.name);
-    for (let i = 0; i < symbol.signatures.length; i++) {
-      const signature = symbol.signatures[i];
-      const returns = signature.returns.length > 0 ? `: ${signature.returns.length} column${signature.returns.length === 1 ? `` : `s`}` : '';
-
-      base.appendCodeblock(`${routineName}(\n${signature.parms.map((p, pI) => {
-        return `  ${Statement.prettyName(p.PARAMETER_NAME || `parm${pI}`)} => ${prepareParamType(p)}`
-      }).join(',\n')}\n)${returns}`, `sql`);
-
-      if (i < symbol.signatures.length - 1) {
-        base.appendMarkdown(`\n---\n`);
-      }
-    }
-  }
-  else if ('PARAMETER_NAME' in symbol) {
+  if ('PARAMETER_NAME' in symbol) {
     base.appendCodeblock(prepareParamType(symbol) + `\n`, `sql`);
   }
   else if ('COLUMN_NAME' in symbol) {
@@ -149,7 +183,7 @@ function addSymbol(base: MarkdownString, symbol: LookupResult) {
   }
 }
 
-function lookupSymbol(name: string, schema: string|undefined, possibleNames: string[]) {
+function lookupSymbol(name: string, schema: string | undefined, possibleNames: string[]) {
   name = Statement.noQuotes(Statement.delimName(name, true));
   schema = schema ? Statement.noQuotes(Statement.delimName(schema, true)) : undefined
 
diff --git a/src/language/providers/logic/cache.ts b/src/language/providers/logic/cache.ts
index 245c81bf..2417747b 100644
--- a/src/language/providers/logic/cache.ts
+++ b/src/language/providers/logic/cache.ts
@@ -2,7 +2,7 @@ import Callable, { CallableRoutine, CallableSignature, CallableType } from "../.
 import Schemas, { PageData, SQLType } from "../../../database/schemas";
 import Table from "../../../database/table";
 
-interface RoutineDetail {
+export interface RoutineDetail {
   routine: CallableRoutine;
   signatures: CallableSignature[];
 }
diff --git a/src/language/providers/logic/completion.ts b/src/language/providers/logic/completion.ts
index 3eb4c399..e2f1674d 100644
--- a/src/language/providers/logic/completion.ts
+++ b/src/language/providers/logic/completion.ts
@@ -34,8 +34,10 @@ export function prepareParamType(param: TableColumn | SQLParm): string {
     baseType += `(${param.NUMERIC_PRECISION}, ${param.NUMERIC_SCALE})`;
   }
 
-  if ([`Y`, `YES`].includes(param.IS_NULLABLE) || ('DEFAULT' in param && param.DEFAULT !== null)) {
-    baseType += ` optional`;
+  const usefulNull = 'COLUMN_NAME' in param || ('ROW_TYPE' in param && param.ROW_TYPE === 'R');
+
+  if (usefulNull && [`Y`, `YES`].includes(param.IS_NULLABLE)) {
+    baseType += ` nullable`;
   };
 
   return baseType;

From 103f43f4c90ba895bb18ec787bb21ba2615a7066 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 16:23:41 -0400
Subject: [PATCH 23/28] Better support for LATERAL

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/statement.ts             | 15 ++++++++++---
 src/language/sql/tests/statements.test.ts | 26 ++++++++++++++++++++++-
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts
index f333f98b..248da070 100644
--- a/src/language/sql/statement.ts
+++ b/src/language/sql/statement.ts
@@ -512,8 +512,8 @@ export default class Statement {
 
 		let endIndex = i;
 
-		let isUDTF = tokenIs(nextToken, `function`, `TABLE`);
-
+		const isUDTF = tokenIs(nextToken, `function`, `TABLE`);
+		const isLateral = tokenIs(nextToken, `function`, `LATERAL`);
 
 		if (isUDTF) {
 			sqlObj = this.getRefAtToken(i+2);
@@ -528,7 +528,16 @@ export default class Statement {
 				nextIndex = -1;
 				nextToken = undefined;
 			}
-
+		} else if (isLateral) {
+			console.log(`HI`);
+			const blockTokens = this.getBlockAt(nextToken.range.end+1);
+			console.log(blockTokens);
+			const newStatement = new Statement(blockTokens, {start: nextToken.range.start, end: blockTokens[blockTokens.length-1].range.end});
+			[sqlObj] = newStatement.getObjectReferences();
+			
+			nextIndex = i + 2 + blockTokens.length;
+			nextToken = this.tokens[nextIndex];
+		
 		} else {
 			if (nextToken && NameTypes.includes(nextToken.type)) {
 				nextIndex = i;
diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts
index 045ac062..00032e1e 100644
--- a/src/language/sql/tests/statements.test.ts
+++ b/src/language/sql/tests/statements.test.ts
@@ -971,7 +971,32 @@ describe(`Object references`, () => {
     expect(statement.type).toBe(StatementType.With);
 
     const refs = statement.getObjectReferences();
+    expect(refs.length).toBe(3);
+
+    expect(refs[0].object.name).toBe(`object_ownership`);
+    expect(refs[0].object.schema).toBe(`qsys2`);
+
+    expect(refs[1].object.name).toBe(`qsysobjs`);
+    expect(refs[1].object.schema).toBeUndefined();
+    expect(refs[1].alias).toBe(`q`);
+    
+    expect(refs[2].object.name).toBe(`object_statistics`);
+    expect(refs[2].object.schema).toBe(`qsys2`);
+    expect(refs[2].alias).toBe(`z`);
   });
+
+  test(`Multiple UDTFs`, () => {
+    const lines = [
+      `SELECT b.objlongschema, b.objname, b.objtype, b.objattribute, b.objcreated, b.objsize, b.objtext, b.days_used_count, b.last_used_timestamp,b.* FROM `,
+      `   TABLE (QSYS2.OBJECT_STATISTICS('*ALLUSRAVL ', '*LIB') ) as a, `,
+      `   TABLE (QSYS2.OBJECT_STATISTICS(a.objname, 'ALL')  ) AS b`,
+      `WHERE b.OBJOWNER = 'user-name'`,
+      `ORDER BY b.OBJSIZE DESC`,
+      `FETCH FIRST 100 ROWS ONLY;`,
+    ].join(`\n`);
+
+    const document = new Document(lines);
+  })
 });
 
 describe(`Offset reference tests`, () => {
@@ -1167,7 +1192,6 @@ describe(`PL body tests`, () => {
     const refs = statement.getObjectReferences();
     const ctes = statement.getCTEReferences();
 
-    console.log(refs);
     expect(refs.length).toBe(7);
     expect(refs[0].object.name).toBe(`shipments`);
     expect(refs[0].alias).toBe(`s`);

From 134586b7fa523efa053686b56de6e53fe723c331 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 16:37:34 -0400
Subject: [PATCH 24/28] Fix for brakcets not taken into account

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/statement.ts             |  9 ++++++---
 src/language/sql/tests/statements.test.ts | 18 ++++++++++++++++++
 src/language/sql/types.ts                 |  1 +
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts
index 248da070..e40a84a1 100644
--- a/src/language/sql/statement.ts
+++ b/src/language/sql/statement.ts
@@ -330,6 +330,9 @@ export default class Statement {
 					if (sqlObj) {
 						doAdd(sqlObj);
 						i += sqlObj.tokens.length;
+						if (sqlObj.isUDTF || sqlObj.fromLateral) {
+							i += 3; //For the brackets
+						}
 					}
 				}
 			}
@@ -520,6 +523,7 @@ export default class Statement {
 			if (sqlObj) {
 				sqlObj.isUDTF = true;
 				const blockTokens = this.getBlockAt(sqlObj.tokens[0].range.end);
+				
 				sqlObj.tokens = blockTokens;
 				nextIndex = i + 2 + blockTokens.length;
 				nextToken = this.tokens[nextIndex];
@@ -529,12 +533,11 @@ export default class Statement {
 				nextToken = undefined;
 			}
 		} else if (isLateral) {
-			console.log(`HI`);
 			const blockTokens = this.getBlockAt(nextToken.range.end+1);
-			console.log(blockTokens);
 			const newStatement = new Statement(blockTokens, {start: nextToken.range.start, end: blockTokens[blockTokens.length-1].range.end});
 			[sqlObj] = newStatement.getObjectReferences();
-			
+
+			sqlObj.fromLateral = true;
 			nextIndex = i + 2 + blockTokens.length;
 			nextToken = this.tokens[nextIndex];
 		
diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts
index 00032e1e..fc142137 100644
--- a/src/language/sql/tests/statements.test.ts
+++ b/src/language/sql/tests/statements.test.ts
@@ -996,6 +996,24 @@ describe(`Object references`, () => {
     ].join(`\n`);
 
     const document = new Document(lines);
+
+    expect(document.statements.length).toBe(1);
+
+    const statement = document.statements[0];
+
+    expect(statement.type).toBe(StatementType.Select);
+
+    const refs = statement.getObjectReferences();
+
+    console.log(refs);
+    expect(refs.length).toBe(2);
+    expect(refs[0].object.name).toBe(`OBJECT_STATISTICS`);
+    expect(refs[0].object.schema).toBe(`QSYS2`);
+    expect(refs[0].alias).toBe(`a`);
+
+    expect(refs[1].object.name).toBe(`OBJECT_STATISTICS`);
+    expect(refs[1].object.schema).toBe(`QSYS2`);
+    expect(refs[1].alias).toBe(`b`);
   })
 });
 
diff --git a/src/language/sql/types.ts b/src/language/sql/types.ts
index 75c69e20..b79f1479 100644
--- a/src/language/sql/types.ts
+++ b/src/language/sql/types.ts
@@ -84,6 +84,7 @@ export interface ObjectRef {
   alias?: string;
 
 	isUDTF?: boolean;
+	fromLateral?: boolean;
 
 	/** only used within create statements */
 	createType?: string;

From 6d7eb8a0486cbec7bfd54e3e0d383e46da71f7e2 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 17:04:47 -0400
Subject: [PATCH 25/28] Fix issue with signatures not showing up

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/providers/hoverProvider.ts   | 5 +++--
 src/language/sql/tests/statements.test.ts | 1 -
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts
index 96d11d77..8fd2b107 100644
--- a/src/language/providers/hoverProvider.ts
+++ b/src/language/providers/hoverProvider.ts
@@ -83,9 +83,10 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` }
 
           if (result) {
             if ('routine' in result) {
-              const callableRef = statementAt.getCallableDetail(offset, false);
+              const routineOffset = ref.tokens[ref.tokens.length-1].range.end+1;
+              const callableRef = statementAt.getCallableDetail(routineOffset, false);
               if (callableRef) {
-                const { currentCount } = getPositionData(callableRef, offset);
+                const { currentCount } = getPositionData(callableRef, routineOffset);
                 const signatures = await DbCache.getCachedSignatures(callableRef.parentRef.object.schema, callableRef.parentRef.object.name);
                 const possibleSignatures = signatures.filter((s) => s.parms.length >= currentCount).sort((a, b) => a.parms.length - b.parms.length);
                 const signature = possibleSignatures.find((signature) => currentCount <= signature.parms.length);
diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts
index fc142137..2d675051 100644
--- a/src/language/sql/tests/statements.test.ts
+++ b/src/language/sql/tests/statements.test.ts
@@ -1005,7 +1005,6 @@ describe(`Object references`, () => {
 
     const refs = statement.getObjectReferences();
 
-    console.log(refs);
     expect(refs.length).toBe(2);
     expect(refs[0].object.name).toBe(`OBJECT_STATISTICS`);
     expect(refs[0].object.schema).toBe(`QSYS2`);

From 5b6e17c7ce821bbc772068c73d6271c9f74e85cb Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 17:04:56 -0400
Subject: [PATCH 26/28] Correctly seperate statements when parsing document

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/document.ts  | 1 +
 src/language/sql/statement.ts | 2 +-
 src/language/sql/tokens.ts    | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts
index 2a51a348..e1303b22 100644
--- a/src/language/sql/document.ts
+++ b/src/language/sql/document.ts
@@ -51,6 +51,7 @@ export default class Document {
         case `keyword`:
           switch (tokens[i].value?.toUpperCase()) {
             case `BEGIN`:
+            case `DO`:
               // We include BEGIN in the current statement
               // then the next statement beings
               const statementTokens = tokens.slice(statementStart, i+1);
diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts
index e40a84a1..28ea6cff 100644
--- a/src/language/sql/statement.ts
+++ b/src/language/sql/statement.ts
@@ -43,7 +43,7 @@ export default class Statement {
 		// These statements can end with BEGIN, which signifies a block starter
 		if ([StatementType.Create, StatementType.Declare].includes(this.type)) {
 			const last = this.tokens[this.tokens.length-1];
-			if (tokenIs(last, `keyword`, `BEGIN`)) {
+			if (tokenIs(last, `keyword`, `BEGIN`) || tokenIs(last, `keyword`, `DO`)) {
 				return true;
 			}
 		}
diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts
index 19837829..a224f18f 100644
--- a/src/language/sql/tokens.ts
+++ b/src/language/sql/tokens.ts
@@ -74,7 +74,7 @@ export default class SQLTokeniser {
     {
       name: `KEYWORD`,
       match: [{ type: `word`, match: (value: string) => {
-        return [`AS`, `FOR`, `OR`, `REPLACE`, `BEGIN`, `END`, `CURSOR`, `DEFAULT`, `HANDLER`, `REFERENCES`, `ON`, `UNIQUE`, `SPECIFIC`, `EXTERNAL`].includes(value.toUpperCase());
+        return [`AS`, `FOR`, `OR`, `REPLACE`, `BEGIN`, `DO`, `END`, `CURSOR`, `DEFAULT`, `HANDLER`, `REFERENCES`, `ON`, `UNIQUE`, `SPECIFIC`, `EXTERNAL`].includes(value.toUpperCase());
       } }],
       becomes: `keyword`,
     },

From a271fe93452485a7d4388663c13a5c533ea4172d Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 23:22:03 -0400
Subject: [PATCH 27/28] Better support for THEN and LOOP

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/document.ts              |  7 +++
 src/language/sql/statement.ts             |  2 +-
 src/language/sql/tests/statements.test.ts | 70 +++++++++++++++++++++++
 src/language/sql/tokens.ts                |  2 +-
 4 files changed, 79 insertions(+), 2 deletions(-)

diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts
index e1303b22..2e69393e 100644
--- a/src/language/sql/document.ts
+++ b/src/language/sql/document.ts
@@ -50,8 +50,14 @@ export default class Document {
 
         case `keyword`:
           switch (tokens[i].value?.toUpperCase()) {
+            case `LOOP`:
+              // This handles the case that 'END LOOP' is supported.
+              if (currentStatementType === StatementType.End) {
+                break;
+              } 
             case `BEGIN`:
             case `DO`:
+            case `THEN`:
               // We include BEGIN in the current statement
               // then the next statement beings
               const statementTokens = tokens.slice(statementStart, i+1);
@@ -96,6 +102,7 @@ export default class Document {
     let depth = 0;
 
     for (const statement of this.statements) {
+      console.log(currentGroup);
         if (statement.isBlockEnder()) {
           if (depth > 0) {
             currentGroup.push(statement);
diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts
index 28ea6cff..e40a84a1 100644
--- a/src/language/sql/statement.ts
+++ b/src/language/sql/statement.ts
@@ -43,7 +43,7 @@ export default class Statement {
 		// These statements can end with BEGIN, which signifies a block starter
 		if ([StatementType.Create, StatementType.Declare].includes(this.type)) {
 			const last = this.tokens[this.tokens.length-1];
-			if (tokenIs(last, `keyword`, `BEGIN`) || tokenIs(last, `keyword`, `DO`)) {
+			if (tokenIs(last, `keyword`, `BEGIN`)) {
 				return true;
 			}
 		}
diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts
index 2d675051..f06008aa 100644
--- a/src/language/sql/tests/statements.test.ts
+++ b/src/language/sql/tests/statements.test.ts
@@ -1013,6 +1013,76 @@ describe(`Object references`, () => {
     expect(refs[1].object.name).toBe(`OBJECT_STATISTICS`);
     expect(refs[1].object.schema).toBe(`QSYS2`);
     expect(refs[1].alias).toBe(`b`);
+  });
+
+  test('LOOP statements', () => {
+    const lines = [
+      `CREATE OR REPLACE PROCEDURE KRAKEN917.Wait_For_Kraken(kraken_job_name varchar(10), delay_time bigint default 30)`,
+      `BEGIN`,
+      `    DECLARE v_sql_stmt CLOB(1M) CCSID 37;`,
+      ``,
+      `    DECLARE number_of_active_jobs INT;`,
+      ``,
+      `    CALL systools.lprintf('Waiting for job to finish...');`,
+      ``,
+      `    fetch_loop: LOOP`,
+      `        SET v_sql_stmt ='values(SELECT COUNT(*) FROM TABLE (qsys2.active_job_info(subsystem_list_filter => ''QBATCH'')) WHERE JOB_NAME_SHORT LIKE ''' CONCAT kraken_job_name CONCAT ''') into ?';`,
+      ``,
+      `        PREPARE values_st FROM v_sql_stmt;`,
+      ``,
+      `        EXECUTE values_st USING number_of_active_jobs;`,
+      ``,
+      `        IF number_of_active_jobs = 0 THEN`,
+      `            CALL SYSTOOLS.LPRINTF(kraken_job_name CONCAT ' JOB DONE');`,
+      ``,
+      `            LEAVE fetch_loop;`,
+      ``,
+      `        END IF;`,
+      ``,
+      `        CALL qsys2.qcmdexc('DLYJOB ' CONCAT delay_time);`,
+      ``,
+      `    END LOOP fetch_loop;`,
+      ``,
+      `END;`,
+    ].join(`\n`);
+
+    const document = new Document(lines);
+
+    const groups = document.getStatementGroups();
+    expect(groups.length).toBe(1);
+
+    const group = groups[0];
+
+    // console.log(group.statements.map((s, so) => `${so}  ` + s.type.padEnd(10) + ` ` + s.tokens.map(t => t.value).join(' ')));
+
+    expect(group.statements.length).toBe(16);
+    expect(group.statements.map(s => s.type)).toEqual([
+      'Create',  'Declare',
+      'Declare', 'Call',
+      'Unknown', 'Unknown',
+      'Unknown', 'Unknown',
+      'Unknown', 'Call',
+      'Unknown', 'End',
+      'Call',    'End',
+      'Unknown', 'End'
+    ]);
+
+    let refs;
+
+    const firstCall = group.statements[3];
+    refs = firstCall.getObjectReferences();
+    expect(refs.length).toBe(1);
+    expect(refs[0].object.name).toBe(`lprintf`);
+
+    const secondCall = group.statements[9];
+    refs = secondCall.getObjectReferences();
+    expect(refs.length).toBe(1);
+    expect(refs[0].object.name).toBe(`LPRINTF`);
+
+    const thirdCall = group.statements[12];
+    refs = thirdCall.getObjectReferences();
+    expect(refs.length).toBe(1);
+    expect(refs[0].object.name).toBe(`qcmdexc`);
   })
 });
 
diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts
index a224f18f..e462c064 100644
--- a/src/language/sql/tokens.ts
+++ b/src/language/sql/tokens.ts
@@ -74,7 +74,7 @@ export default class SQLTokeniser {
     {
       name: `KEYWORD`,
       match: [{ type: `word`, match: (value: string) => {
-        return [`AS`, `FOR`, `OR`, `REPLACE`, `BEGIN`, `DO`, `END`, `CURSOR`, `DEFAULT`, `HANDLER`, `REFERENCES`, `ON`, `UNIQUE`, `SPECIFIC`, `EXTERNAL`].includes(value.toUpperCase());
+        return [`AS`, `FOR`, `OR`, `REPLACE`, `BEGIN`, `DO`, `THEN`, `LOOP`, `END`, `CURSOR`, `DEFAULT`, `HANDLER`, `REFERENCES`, `ON`, `UNIQUE`, `SPECIFIC`, `EXTERNAL`].includes(value.toUpperCase());
       } }],
       becomes: `keyword`,
     },

From 9198377fe17971ea4e69c410f6097d181da05feb Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Tue, 17 Sep 2024 23:25:07 -0400
Subject: [PATCH 28/28] Remove log

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 src/language/sql/document.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts
index 2e69393e..2bbf41df 100644
--- a/src/language/sql/document.ts
+++ b/src/language/sql/document.ts
@@ -102,7 +102,6 @@ export default class Document {
     let depth = 0;
 
     for (const statement of this.statements) {
-      console.log(currentGroup);
         if (statement.isBlockEnder()) {
           if (depth > 0) {
             currentGroup.push(statement);