diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQConnection.java b/src/main/java/net/starschema/clouddb/jdbc/BQConnection.java index 6f62493a..afb7aae4 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQConnection.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQConnection.java @@ -59,9 +59,10 @@ public class BQConnection implements Connection { private String dataset = null; /** - * The projectid which needed for the queries. + * The ProjectId for the connection */ private String projectId = null; + /** Boolean to determine if the Connection is closed */ private boolean isclosed = false; @@ -114,16 +115,14 @@ public BQConnection(String url, Properties loginProp) throws SQLException { Matcher matchData = projectAndDatasetMatcher.matcher(pathParams); if (matchData.find()) { - this.projectId = matchData.group(1); + this.projectId = CatalogName.toProjectId(matchData.group(1)); this.dataset = matchData.group(2); } else { - this.projectId = pathParams; + this.projectId = CatalogName.toProjectId(pathParams); } } catch (UnsupportedEncodingException e1) { throw new BQSQLException(e1); } - // lets replace the : with __ and . with _ - this.projectId = this.projectId.replace(":", "__").replace(".", "_"); Properties caseInsensitiveLoginProps = new Properties(); @@ -227,7 +226,7 @@ else if (oAuthAccessToken != null) { else { throw new IllegalArgumentException("Must provide a valid mechanism to authenticate."); } - logger.debug("The project id for this connections is: " + this.projectId); + logger.debug("The project id for this connections is: " + projectId); } /** @@ -371,7 +370,7 @@ public Statement createStatement() throws SQLException { } logger.debug("Creating statement with resultsettype: forward only," + " concurrency: read only"); - return new BQStatement(this.projectId, this); + return new BQStatement(projectId, this); } /** {@inheritDoc} */ @@ -383,8 +382,7 @@ public Statement createStatement(int resultSetType, int resultSetConcurrency) } logger.debug("Creating statement with resultsettype: " + resultSetType + " concurrency: " + resultSetConcurrency); - return new BQStatement(this.projectId, this, resultSetType, - resultSetConcurrency); + return new BQStatement(projectId, this, resultSetType, resultSetConcurrency); } /** {@inheritDoc} */ @@ -417,8 +415,8 @@ public void setSchema(String schema) { } @Override - public String getSchema() { - return this.dataset; + public String getSchema() throws SQLException { + return getDataSet(); } public void abort(Executor executor) throws SQLException { @@ -464,7 +462,7 @@ public Bigquery getBigquery() { @Override public String getCatalog() throws SQLException { logger.debug("function call getCatalog returning projectId: " + projectId); - return this.projectId; + return projectId; } /** @@ -526,7 +524,7 @@ public DatabaseMetaData getMetaData() throws SQLException { * Getter method for projectId */ public String getProjectId() { - return this.projectId; + return projectId; } /** @@ -638,7 +636,7 @@ public boolean isValid(int timeout) throws SQLException { + String.valueOf(timeout)); } try { - this.bigquery.datasets().list(this.projectId.replace("__", ":").replace("_", ".")).execute(); + this.bigquery.datasets().list(projectId).execute(); } catch (IOException e) { return false; } @@ -729,10 +727,9 @@ public CallableStatement prepareCall(String sql, int resultSetType, @Override public PreparedStatement prepareStatement(String sql) throws SQLException { this.logger.debug("Creating Prepared Statement project id is: " - + this.projectId + " with parameters:"); + + projectId + " with parameters:"); this.logger.debug(sql); - PreparedStatement stm = new BQPreparedStatement(sql, this.projectId, - this); + PreparedStatement stm = new BQPreparedStatement(sql, projectId, this); return stm; } @@ -756,12 +753,12 @@ public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { this.logger.debug("Creating Prepared Statement" + - " project id is: " + this.projectId + + " project id is: " + projectId + ", resultSetType (int) is: " + String.valueOf(resultSetType) + ", resultSetConcurrency (int) is: " + String.valueOf(resultSetConcurrency) + " with parameters:"); this.logger.debug(sql); - PreparedStatement stm = new BQPreparedStatement(sql, this.projectId, + PreparedStatement stm = new BQPreparedStatement(sql, projectId, this, resultSetType, resultSetConcurrency); return stm; } diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQDatabaseMetadata.java b/src/main/java/net/starschema/clouddb/jdbc/BQDatabaseMetadata.java index 15dd2f79..46719708 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQDatabaseMetadata.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQDatabaseMetadata.java @@ -293,7 +293,7 @@ public ResultSet getCatalogs() throws SQLException { String[] Data = new String[Projects.size()]; String toLog = ""; for (int i = 0; i < Projects.size(); i++) { - Data[i] = Projects.get(i).getId().replace(":", "__").replace(".", "_"); + Data[i] = Projects.get(i).getId(); toLog += Data[i] + " , "; } logger.debug("Catalogs are: " + toLog); @@ -402,11 +402,17 @@ public ResultSet getColumns(String catalog, String schemaPattern, (columnNamePattern != null ? columnNamePattern : "null")); List Tables = null; try { - Tables = BQSupportFuncts.getTables(this.Connection, catalog, - schemaPattern, tableNamePattern); + Tables = BQSupportFuncts.getTables( + this.Connection, + CatalogName.toProjectId(catalog), + schemaPattern, + tableNamePattern); if (Tables == null) { //Because of Crystal Reports It's not elegant, but hey it works! - Tables = BQSupportFuncts.getTables(this.Connection, schemaPattern, - catalog, tableNamePattern); + Tables = BQSupportFuncts.getTables( + this.Connection, + CatalogName.toProjectId(schemaPattern), + catalog, + tableNamePattern); } } catch (IOException e) { throw new BQSQLException(e); @@ -415,13 +421,9 @@ public ResultSet getColumns(String catalog, String schemaPattern, if (Tables != null) { List data = new ArrayList(); for (int i = 0; i < Tables.size(); i++) { - String UparsedId = Tables.get(i).getId(); - String ProjectId = BQSupportFuncts - .getProjectIdFromAnyGetId(UparsedId).replace(":", "__").replace(".", "_"); - String DatasetId = BQSupportFuncts - .getDatasetIdFromTableDotGetId(UparsedId); - String TableId = BQSupportFuncts - .getTableIdFromTableDotGetId(UparsedId); + String ProjectId = Tables.get(i).getTableReference().getProjectId(); + String DatasetId = Tables.get(i).getTableReference().getDatasetId(); + String TableId = Tables.get(i).getTableReference().getTableId(); List tblfldschemas = Tables.get(i) .getSchema().getFields(); @@ -1404,10 +1406,8 @@ public ResultSet getSchemas() throws SQLException { data = new String[datasetlist.size()][2]; int i = 0; for (Datasets datasets : datasetlist) { - data[i][0] = datasets.getDatasetReference() - .getDatasetId(); - data[i][1] = datasets.getDatasetReference() - .getProjectId().replace(".", "_").replace(":", "__"); + data[i][0] = datasets.getDatasetReference().getDatasetId(); + data[i][1] = datasets.getDatasetReference().getProjectId(); i++; } } @@ -1482,10 +1482,8 @@ public ResultSet getSchemas(String catalog, String schemaPattern) data = new String[datasetlist.size()][2]; i = 0; for (Datasets datasets : datasetlist) { - String schema = datasets.getDatasetReference() - .getDatasetId(); - String projnm = datasets.getDatasetReference() - .getProjectId(); + String schema = datasets.getDatasetReference().getDatasetId(); + String projnm = datasets.getDatasetReference().getProjectId(); logger.debug("We search for catalog/project: " + catalog); if ((schema.equals(schemaPattern) || schemaPattern == null) && (projnm.equals(catalog) || catalog == null)) { @@ -1744,11 +1742,17 @@ public ResultSet getTables(String catalog, String schemaPattern, + ", types: " + typesToLog + ")"); List
tables = null; try { - tables = BQSupportFuncts.getTables(this.Connection, catalog, - schemaPattern, tableNamePattern); + tables = BQSupportFuncts.getTables( + this.Connection, + CatalogName.toProjectId(catalog), + schemaPattern, + tableNamePattern); if (tables == null) { //because of crystal reports, It's not elegant but hey, it works! - tables = BQSupportFuncts.getTables(this.Connection, tableNamePattern, - schemaPattern, catalog); + tables = BQSupportFuncts.getTables( + this.Connection, + CatalogName.toProjectId(tableNamePattern), + schemaPattern, + catalog); } } catch (IOException e) { throw new BQSQLException(e); @@ -1757,16 +1761,15 @@ public ResultSet getTables(String catalog, String schemaPattern, logger.debug("got result, size: " + tables.size()); String[][] data = new String[tables.size()][10]; for (int i = 0; i < tables.size(); i++) { - String UparsedId = tables.get(i).getId(); - data[i][0] = BQSupportFuncts.getProjectIdFromAnyGetId(UparsedId).replace(":", "__").replace(".", "_"); - data[i][1] = BQSupportFuncts.getDatasetIdFromTableDotGetId(UparsedId); - data[i][2] = BQSupportFuncts.getTableIdFromTableDotGetId(UparsedId); + data[i][0] = tables.get(i).getTableReference().getProjectId(); + data[i][1] = tables.get(i).getTableReference().getDatasetId(); + data[i][2] = tables.get(i).getTableReference().getTableId(); data[i][3] = "TABLE"; data[i][4] = tables.get(i).getDescription(); data[i][5] = null; - data[i][6] = BQSupportFuncts.getProjectIdFromAnyGetId(UparsedId).replace(":", "__").replace(".", "_"); - data[i][7] = BQSupportFuncts.getDatasetIdFromTableDotGetId(UparsedId); - data[i][8] = BQSupportFuncts.getTableIdFromTableDotGetId(UparsedId); + data[i][6] = tables.get(i).getTableReference().getProjectId(); + data[i][7] = tables.get(i).getTableReference().getDatasetId(); + data[i][8] = tables.get(i).getTableReference().getTableId(); data[i][9] = null; } return new DMDResultSet(data, new String[]{"TABLE_CAT", diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java b/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java index dd3edcd0..86045521 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQPreparedStatement.java @@ -72,7 +72,7 @@ public BQPreparedStatement(String querysql, String projectid, this.logger.debug("Constructor of PreparedStatement Running " + "projectid is:" + projectid + "sqlquery: " + querysql); - this.ProjectId = projectid; + this.projectId = projectid; this.connection = bqConnection; //this.resultSetType = ResultSet.TYPE_SCROLL_INSENSITIVE; //-scrollable this.resultSetType = ResultSet.TYPE_FORWARD_ONLY; @@ -116,7 +116,7 @@ public BQPreparedStatement(String querysql, String projectid, "The Resultset Concurrency can't be ResultSet.CONCUR_UPDATABLE"); } - this.ProjectId = projectid; + this.projectId = projectid; this.connection = bqConnection; this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; @@ -238,7 +238,7 @@ public ResultSet executeQuery() throws SQLException { // Gets the Job reference of the completed job with give Query referencedJob = BQSupportFuncts.startQuery( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + this.projectId, this.RunnableStatement, this.connection.getDataSet(), this.connection.getUseLegacySql(), @@ -252,17 +252,17 @@ public ResultSet executeQuery() throws SQLException { do { if (BQSupportFuncts.getQueryState(referencedJob, this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", ".")) + this.projectId) .equals("DONE")) { if (resultSetType == ResultSet.TYPE_SCROLL_INSENSITIVE) { return new BQScrollableResultSet(BQSupportFuncts.getQueryResults( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + this.projectId, referencedJob), this); } else { return new BQForwardOnlyResultSet( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + this.projectId, referencedJob, this); } } diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java b/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java index 6a7f9f77..d071d5a9 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQStatement.java @@ -59,12 +59,12 @@ public class BQStatement extends BQStatementRoot implements java.sql.Statement { /** * Constructor for BQStatement object just initializes local variables * - * @param projectid + * @param projectId * @param bqConnection */ - public BQStatement(String projectid, BQConnection bqConnection) { - logger.debug("Constructor of BQStatement is running projectid is: " + projectid); - this.ProjectId = projectid; + public BQStatement(String projectId, BQConnection bqConnection) { + logger.debug("Constructor of BQStatement is running projectId is: " + projectId); + this.projectId = projectId; this.connection = bqConnection; this.resultSetType = ResultSet.TYPE_FORWARD_ONLY; this.resultSetConcurrency = ResultSet.CONCUR_READ_ONLY; @@ -73,15 +73,15 @@ public BQStatement(String projectid, BQConnection bqConnection) { /** * Constructor for BQStatement object just initializes local variables * - * @param projectid + * @param projectId * @param bqConnection * @param resultSetType * @param resultSetConcurrency * @throws BQSQLException */ - public BQStatement(String projectid, BQConnection bqConnection, + public BQStatement(String projectId, BQConnection bqConnection, int resultSetType, int resultSetConcurrency) throws BQSQLException { - logger.debug("Constructor of BQStatement is running projectid is: " + projectid + + logger.debug("Constructor of BQStatement is running projectId is: " + projectId + ",resultSetType is: " + resultSetType + ",resutSetConcurrency is: " + resultSetConcurrency); if (resultSetConcurrency == ResultSet.CONCUR_UPDATABLE) { @@ -89,7 +89,7 @@ public BQStatement(String projectid, BQConnection bqConnection, "The Resultset Concurrency can't be ResultSet.CONCUR_UPDATABLE"); } - this.ProjectId = projectid; + this.projectId = projectId; this.connection = bqConnection; this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; @@ -146,14 +146,14 @@ private ResultSet executeQueryHelper(String querySql, boolean unlimitedBillingBy null : this.connection.getBigquery() .jobs() - .get(this.ProjectId, qr.getJobReference().getJobId()) + .get(projectId, qr.getJobReference().getJobId()) .setLocation(qr.getJobReference().getLocation()) .execute(); if (qr.getJobComplete()) { if (resultSetType != ResultSet.TYPE_SCROLL_INSENSITIVE) { return new BQForwardOnlyResultSet( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + projectId, referencedJob, this, qr.getRows(), fetchedAll, qr.getSchema()); } else if (fetchedAll) { // We can only return scrollable result sets here if we have all the rows: otherwise we'll @@ -180,7 +180,7 @@ private ResultSet executeQueryHelper(String querySql, boolean unlimitedBillingBy try { status = BQSupportFuncts.getQueryState(referencedJob, this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", ".")); + projectId); } catch (IOException e) { if (retries++ < MAX_IO_FAILURE_RETRIES) { continue; @@ -196,12 +196,12 @@ private ResultSet executeQueryHelper(String querySql, boolean unlimitedBillingBy if (resultSetType == ResultSet.TYPE_SCROLL_INSENSITIVE) { return new BQScrollableResultSet(BQSupportFuncts.getQueryResults( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + projectId, referencedJob), this); } else { return new BQForwardOnlyResultSet( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + projectId, referencedJob, this); } } @@ -241,7 +241,7 @@ protected QueryResponse runSyncQuery(String querySql, boolean unlimitedBillingBy try { QueryResponse resp = BQSupportFuncts.runSyncQuery( this.connection.getBigquery(), - this.ProjectId, + projectId, querySql, connection.getDataSet(), this.connection.getUseLegacySql(), @@ -324,7 +324,7 @@ public void cancel() throws SQLException { /** Wrap [BQSupportFuncts.cancelQuery] purely for testability purposes. */ protected void performQueryCancel(JobReference jobRefToCancel) throws IOException { - BQSupportFuncts.cancelQuery(jobRefToCancel, this.connection.getBigquery(), this.ProjectId.replace("__", ":").replace("_", ".")); + BQSupportFuncts.cancelQuery(jobRefToCancel, this.connection.getBigquery(), projectId); } @Override diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java b/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java index 6f5c4108..eb70a938 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQStatementRoot.java @@ -50,7 +50,7 @@ public abstract class BQStatementRoot { ResultSet resset = null; /** String containing the context of the Project */ - String ProjectId = null; + String projectId = null; Logger logger = LoggerFactory.getLogger(BQStatementRoot.class); @@ -257,7 +257,7 @@ private int executeDML(String sql) throws SQLException { try { QueryResponse qr = BQSupportFuncts.runSyncQuery( this.connection.getBigquery(), - this.ProjectId, + projectId, sql, connection.getDataSet(), this.connection.getUseLegacySql(), @@ -271,7 +271,7 @@ private int executeDML(String sql) throws SQLException { return Math.toIntExact(qr.getNumDmlAffectedRows()); } - referencedJob = this.connection.getBigquery().jobs().get(this.ProjectId, qr.getJobReference().getJobId()).execute(); + referencedJob = this.connection.getBigquery().jobs().get(projectId, qr.getJobReference().getJobId()).execute(); } catch (IOException e) { throw new BQSQLException("Something went wrong with the query: " + sql, e); } @@ -279,11 +279,11 @@ private int executeDML(String sql) throws SQLException { try { do { if (BQSupportFuncts.getQueryState(referencedJob, - this.connection.getBigquery(), this.ProjectId).equals( + this.connection.getBigquery(), projectId).equals( "DONE")) { return Math.toIntExact( BQSupportFuncts.getQueryResults( - this.connection.getBigquery(), this.ProjectId, + this.connection.getBigquery(), projectId, referencedJob).getNumDmlAffectedRows() ); } @@ -314,7 +314,7 @@ public ResultSet executeQuery(String querySql, boolean unlimitedBillingBytes) th try { QueryResponse qr = BQSupportFuncts.runSyncQuery( this.connection.getBigquery(), - this.ProjectId, + projectId, querySql, connection.getDataSet(), this.connection.getUseLegacySql(), @@ -328,7 +328,7 @@ public ResultSet executeQuery(String querySql, boolean unlimitedBillingBytes) th } jobAlreadyCompleted = true; } - referencedJob = this.connection.getBigquery().jobs().get(this.ProjectId, qr.getJobReference().getJobId()).execute(); + referencedJob = this.connection.getBigquery().jobs().get(projectId, qr.getJobReference().getJobId()).execute(); this.logger.info("Executing Query: " + querySql); } catch (IOException e) { @@ -337,16 +337,16 @@ public ResultSet executeQuery(String querySql, boolean unlimitedBillingBytes) th try { do { if (jobAlreadyCompleted || BQSupportFuncts.getQueryState(referencedJob, - this.connection.getBigquery(), this.ProjectId).equals( + this.connection.getBigquery(), projectId).equals( "DONE")) { if (resultSetType == ResultSet.TYPE_SCROLL_INSENSITIVE) { return new BQScrollableResultSet(BQSupportFuncts.getQueryResults( - this.connection.getBigquery(), this.ProjectId, + this.connection.getBigquery(), projectId, referencedJob), this); } else { return new BQForwardOnlyResultSet( this.connection.getBigquery(), - this.ProjectId.replace("__", ":").replace("_", "."), + projectId, referencedJob, this); } } diff --git a/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java b/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java index 902471d9..822ee2d4 100644 --- a/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java +++ b/src/main/java/net/starschema/clouddb/jdbc/BQSupportFuncts.java @@ -73,8 +73,8 @@ public class BQSupportFuncts { */ public static String constructUrlFromPropertiesFile(Properties properties, boolean full, String dataset) throws UnsupportedEncodingException { - String ProjectId = properties.getProperty("projectid"); - logger.debug("projectId is: " + ProjectId); + String projectId = properties.getProperty("projectid"); + logger.debug("projectId is: " + projectId); String User = properties.getProperty("user"); String Password = properties.getProperty("password"); String path = properties.getProperty("path"); @@ -83,16 +83,16 @@ public static String constructUrlFromPropertiesFile(Properties properties, boole String forreturn = ""; if (properties.getProperty("type").equals("installed")) { - if (User != null && Password != null && ProjectId != null) { + if (User != null && Password != null && projectId != null) { forreturn = BQDriver.getURLPrefix() - + URLEncoder.encode(ProjectId, "UTF-8"); + + URLEncoder.encode(projectId, "UTF-8"); } else { return null; } } else if (properties.getProperty("type").equals("service")) { - if (User != null && Password != null && ProjectId != null) { + if (User != null && Password != null && projectId != null) { forreturn = BQDriver.getURLPrefix() - + URLEncoder.encode(ProjectId, "UTF-8") + + URLEncoder.encode(projectId, "UTF-8") + (dataset != null && full ? "/" + URLEncoder.encode(dataset, "UTF-8") : "") + "?withServiceAccount=true"; if (full) { @@ -106,9 +106,9 @@ public static String constructUrlFromPropertiesFile(Properties properties, boole } } else if (properties.getProperty("type").equals("oauth")) { String accessToken = properties.getProperty("oauthaccesstoken"); - if (accessToken != null && ProjectId != null) { + if (accessToken != null && projectId != null) { forreturn = BQDriver.getURLPrefix() - + URLEncoder.encode(ProjectId, "UTF-8") + + URLEncoder.encode(projectId, "UTF-8") + (dataset != null && full ? "/" + URLEncoder.encode(dataset, "UTF-8") : ""); if (full) { forreturn += "?oAuthAccessToken=" + URLEncoder.encode(accessToken, "UTF-8"); @@ -158,41 +158,10 @@ public static Properties getUrlQueryComponents(String url, Properties defaults) return components; } - /** - * Displays results of a Query on the console - * - * @param bigquery A valid authorized Bigquery instance - * @param projectId The exact Id of the Project associated with the completed - * BigQuery job - * @param completedJob The Job reference of the completed job - * @throws IOException

- * If the request fails to get the QueryResult of the specified - * job in the given ProjectId - *

- */ - public static void displayQueryResults(Bigquery bigquery, String projectId, - Job completedJob) throws IOException { - projectId = projectId.replace("__", ":").replace("_", "."); - JobReference completedJobReference = completedJob.getJobReference(); - GetQueryResultsResponse queryResult = bigquery - .jobs() - .getQueryResults(projectId, completedJobReference.getJobId()) - .setLocation(completedJobReference.getLocation()) - .execute(); - List rows = queryResult.getRows(); - System.out.print("\nQuery Results:\n------------\n"); - for (TableRow row : rows) { - for (TableCell field : row.getF()) { - System.out.printf("%-20s", field.getV()); - } - System.out.println(); - } - } - /** * Return a list of Projects which contains the String catalogname * - * @param catalogName The String which the id of the result Projects must contain + * @param projectId The String which the id of the result Projects must contain * @param Connection A valid BQConnection instance * @return a list of Projects which contains the String catalogname * @throws IOException

@@ -200,27 +169,18 @@ public static void displayQueryResults(Bigquery bigquery, String projectId, * (to be sorted from) fails *

*/ - private static List getCatalogs(String catalogName, + private static List getProjects(String projectIdFilter, BQConnection Connection) throws IOException { - logger.debug("Function call getCatalogs catalogName: " + - (catalogName != null ? catalogName : "null")); + logger.debug("Function call getProjects catalogName: " + + (projectIdFilter != null ? projectIdFilter : "null")); List projects = Connection.getBigquery().projects().list() .execute().getProjects(); - //since we'll use . -> _ : -> __ conversion, it's easier to - // replace all the . : before we do any compare if (projects != null && projects.size() != 0) { //we got projects! - for (Projects projects2 : projects) { - projects2.setId(projects2.getId().replace(".", "_").replace(":", "__")); - //updating the reference too - ProjectReference projRef = projects2.getProjectReference(); - projRef.setProjectId(projRef.getProjectId().replace(".", "_").replace(":", "__")); - projects2.setProjectReference(projRef); - } - if (catalogName != null) { + if (projectIdFilter != null) { List ProjectsSearch = new ArrayList(); for (Projects project : projects) { - if (project.getId().contains(catalogName)) { + if (project.getId().equals(projectIdFilter)) { ProjectsSearch.add(project); } } @@ -237,32 +197,6 @@ private static List getCatalogs(String catalogName, } } - /** - * Parses a (instance of dataset).getId() and gives back the id only for the - * dataset - * - * @param getId (instance of dataset).getId() - * @return the id only for the dataset - */ - public static String getDatasetIdFromDatasetDotGetId(String getId) { - /*logger.debug("Function call getDatasetIdFromDatasetDotGetId" + getId + - "return is: " + getId.substring(getId.lastIndexOf(":") + 1));*/ - return getId.substring(getId.lastIndexOf(":") + 1); - } - - /** - * Parses a (instance of table).getid() and gives back the id only for the - * dataset - * - * @param getId (instance of table).getid() - * @return the id only for the dataset - */ - public static String getDatasetIdFromTableDotGetId(String getId) { - /*logger.debug("Function call getDatasetIdFromTableDotGetId" + getId + - "return is: " + getId.substring(getId.lastIndexOf(":") + 1, getId.lastIndexOf(".")));*/ - return getId.substring(getId.lastIndexOf(":") + 1, getId.lastIndexOf(".")); - } - /** * Returns a list of Datasets, which are associated with the Project which's * id is exactly ProjectId, and their name matches datasetnamepattern @@ -277,12 +211,14 @@ public static String getDatasetIdFromTableDotGetId(String getId) { * fails *

*/ - private static List getDatasets(String datasetname, - String projectId, BQConnection connection) throws IOException { + private static List getDatasets( + String projectId, + String datasetname, + BQConnection connection + ) throws IOException { logger.debug("function call getDatasets, " + "datasetName: " + (datasetname != null ? datasetname : "null") + ", projectId: " + (projectId != null ? projectId : "null")); - projectId = projectId.replace("__", ":").replace("_", "."); List datasetcontainer = connection.getBigquery().datasets() .list(projectId).execute().getDatasets(); @@ -290,7 +226,7 @@ private static List getDatasets(String datasetname, if (datasetname != null) { List datasetsSearch = new ArrayList(); for (Datasets in : datasetcontainer) { - if (matchPattern(getDatasetIdFromDatasetDotGetId(in.getId()), + if (matchPattern(in.getDatasetReference().getDatasetId(), datasetname)) { datasetsSearch.add(in); } @@ -308,24 +244,6 @@ private static List getDatasets(String datasetname, } } - /** - * Parses a (instance of dataset)/(instance of table).getid() and gives back - * the id only for the Project - * - * @param getId (instance of dataset)/(instance of table).getid() - * @returnthe the id only for the Project - */ - public static String getProjectIdFromAnyGetId(String getId) { - int pos = getId.indexOf(":"); // The first appearance of ":" - if (getId.indexOf(":", pos + 1) != -1) { - // If there's a second ":" we'll use it - // (there must be a second ":" !!) - pos = getId.indexOf(":", pos + 1); - } - String ret = getId.substring(0, pos); // Cutting out the project id - return ret; - } - /** * Returns the result of a completed query * @@ -400,7 +318,6 @@ public static GetQueryResultsResponse getQueryResultsDivided(Bigquery bigquery, */ public static String getQueryState(Job myjob, Bigquery bigquery, String projectId) throws IOException { - projectId = projectId.replace("__", ":").replace("_", "."); JobReference myjobReference = myjob.getJobReference(); Job pollJob = bigquery.jobs() .get(projectId, myjobReference.getJobId()) @@ -427,17 +344,6 @@ public static JobCancelResponse cancelQuery(JobReference jobReference, Bigquery return bigquery.jobs().cancel(projectId, jobReference.getJobId()).setLocation(jobReference.getLocation()).execute(); } - /** - * Parses a (instance of table).getid() and gives back the id only for the - * table - * - * @param getId (instance of table).getid() - * @return the id only for the table - */ - public static String getTableIdFromTableDotGetId(String getId) { - return getId.substring(getId.lastIndexOf(".") + 1); - } - /** * Returns a list of Tables which's id matches TablenamePattern and are * exactly in the given Project and Dataset @@ -453,7 +359,7 @@ public static String getTableIdFromTableDotGetId(String getId) { * ProjectId, DatasetId fails *

*/ - public static List getTables(String tableNamePattern, + private static List getTables(String tableNamePattern, String projectId, String datasetId, BQConnection connection) throws IOException { logger.debug("Function call getTables : " + @@ -461,7 +367,6 @@ public static List getTables(String tableNamePattern, ", projectId: " + (projectId != null ? projectId : "null") + ", datasetID:" + (datasetId != null ? datasetId : "null") + "connection"); - projectId = projectId.replace("__", ":").replace("_", "."); Bigquery.Tables.List listCall = connection.getBigquery().tables() .list(projectId, datasetId).setMaxResults(10000000L); // Really big number that we'll never hit List tables = listCall.execute().getTables(); @@ -469,8 +374,7 @@ public static List getTables(String tableNamePattern, if (tableNamePattern != null) { List tablesSearch = new ArrayList(); for (Tables in : tables) { - if (matchPattern(getTableIdFromTableDotGetId(in.getId()), - tableNamePattern)) { + if (matchPattern(in.getTableReference().getTableId(), tableNamePattern)) { tablesSearch.add(in); } } @@ -507,47 +411,38 @@ public static List getTables(String tableNamePattern, *

*/ public static List

getTables(BQConnection connection, - String catalog, String schema, String tablename) throws IOException { + String projectIdFilter, String schema, String tablename) throws IOException { List
RET = new ArrayList
(); logger.debug("Function call getTables : " + - "catalog: " + (catalog != null ? catalog : "null") + + "projectIdFilter: " + (projectIdFilter != null ? projectIdFilter : "null") + ", schema: " + (schema != null ? schema : "null") + ", tablename:" + (tablename != null ? tablename : "null") + "connection"); //getting the projects for this connection - List Projects = BQSupportFuncts.getCatalogs(catalog, - connection); + List Projects = BQSupportFuncts.getProjects(projectIdFilter, connection); if (Projects != null && Projects.size() != 0) { for (Projects proj : Projects) { + String projectId = proj.getId(); List datasetlist = null; - datasetlist = BQSupportFuncts.getDatasets(schema, proj.getId(), - connection); + datasetlist = BQSupportFuncts.getDatasets(projectId, schema, connection); if (datasetlist != null && datasetlist.size() != 0) { for (Datasets dataset : datasetlist) { List tables = null; - tables = BQSupportFuncts.getTables(tablename, proj - .getId(), - BQSupportFuncts - .getDatasetIdFromDatasetDotGetId(dataset - .getId()), connection); + String datasetId = dataset.getDatasetReference().getDatasetId(); + tables = BQSupportFuncts.getTables( + tablename, projectId, datasetId, connection); if (tables != null && tables.size() != 0) { for (Tables table : tables) { - //TODO replace projID __ -> : _ -> . - String datasetString = BQSupportFuncts - .getDatasetIdFromDatasetDotGetId(dataset.getId()); - String tableString = BQSupportFuncts - .getTableIdFromTableDotGetId(table.getId()); + String tableString = table.getTableReference().getTableId(); logger.debug("Calling connection.getBigquery().tables() " + - "dataset is: " + datasetString + + "dataset is: " + datasetId + ", table is: " + tableString + - ", project is: " + proj.getId().replace("__", ":").replace("_", ".")); + ", project is: " + projectId); Table tbl = connection.getBigquery().tables() - .get(proj.getId().replace("__", ":").replace("_", "."), - datasetString, - tableString) + .get(projectId, datasetId, tableString) .execute(); if (tbl != null) { RET.add(tbl); @@ -676,8 +571,6 @@ static QueryResponse runSyncQuery(Bigquery bigquery, String projectId, String querySql, String dataSet, Boolean useLegacySql, Long maxBillingBytes, Long queryTimeoutMs, Long maxResults ) throws IOException { - projectId = projectId.replace("__", ":").replace("_", "."); - QueryRequest qr = new QueryRequest() .setTimeoutMs(queryTimeoutMs) .setQuery(querySql) @@ -711,7 +604,6 @@ static QueryResponse runSyncQuery(Bigquery bigquery, String projectId, public static Job startQuery(Bigquery bigquery, String projectId, String querySql, String dataSet, Boolean useLegacySql, Long maxBillingBytes) throws IOException { - projectId = projectId.replace("__", ":").replace("_", "."); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationQuery queryConfig = new JobConfigurationQuery(); diff --git a/src/main/java/net/starschema/clouddb/jdbc/CatalogName.java b/src/main/java/net/starschema/clouddb/jdbc/CatalogName.java new file mode 100644 index 00000000..ed8a0507 --- /dev/null +++ b/src/main/java/net/starschema/clouddb/jdbc/CatalogName.java @@ -0,0 +1,23 @@ +package net.starschema.clouddb.jdbc; + +/** + * Utility class to translate between JDBC Catalog names and BigQuery Project Ids. + * + *

This is primarily here to maintain compatibility with older versions of this driver which + * accepted an encoded version of the ProjectId as the Catalog name. So as not to break existing + * clients, we will still accept this encoding as part of the JDBC URL identifier. + * + *

The encoding affects Projects which are prefixed with a domain, such as + * "looker.com:sample-project". These may be selected with a Catalog name of + * "looker_com__sample-project", where the period (`.`) is replaced wih a single underscore (`_`) + * and the colon (`:`) is replaced with two underscores (`__`). + */ +public final class CatalogName { + public static String toProjectId(String encodedCatalogName) { + if (encodedCatalogName == null) { + return null; + } else { + return encodedCatalogName.replace("__", ":").replace("_", "."); + } + } +} diff --git a/src/test/java/BQJDBC/QueryResultTest/CancelTest.java b/src/test/java/BQJDBC/QueryResultTest/CancelTest.java index f10c0073..cad0b4e2 100644 --- a/src/test/java/BQJDBC/QueryResultTest/CancelTest.java +++ b/src/test/java/BQJDBC/QueryResultTest/CancelTest.java @@ -154,8 +154,8 @@ public void connectionCloseWithSyncQueryApi() throws SQLException, InterruptedEx } private static class TestableBQStatement extends BQStatement { - public TestableBQStatement(String projectid, BQConnection bqConnection) { - super(projectid, bqConnection); + public TestableBQStatement(String projectId, BQConnection bqConnection) { + super(projectId, bqConnection); } private Condition testPoint; private Lock testLock; diff --git a/src/test/java/BQJDBC/QueryResultTest/JdbcUrlTest.java b/src/test/java/BQJDBC/QueryResultTest/JdbcUrlTest.java index 81f02780..abec453c 100644 --- a/src/test/java/BQJDBC/QueryResultTest/JdbcUrlTest.java +++ b/src/test/java/BQJDBC/QueryResultTest/JdbcUrlTest.java @@ -40,11 +40,23 @@ public void urlWithDefaultDatasetShouldWork() throws SQLException { @Test public void projectWithColons() throws SQLException { - String urlWithColonContainingProject = URL.replace(bq.getProjectId(), "before:after"); + String urlWithColonContainingProject = URL.replace(bq.getProjectId(), "example.com:project"); try { - BQConnection bq_with_colons = new BQConnection(urlWithColonContainingProject, new Properties()); - // Some day we'll get rid of the whacky subbing in and out of colons with double underscores, but today is not that day - Assert.assertEquals("before__after", bq_with_colons.getProjectId()); + BQConnection bqWithColons = new BQConnection(urlWithColonContainingProject, new Properties()); + Assert.assertEquals("example.com:project", bqWithColons.getProjectId()); + Assert.assertEquals("example.com:project", bqWithColons.getCatalog()); + } catch (SQLException e){ + fail("failed to get or parse url: " + e.getMessage()); + } + } + + @Test + public void mungedProjectName() throws SQLException { + String urlWithUnderscoreContainingProject = URL.replace(bq.getProjectId(), "example_com__project"); + try { + BQConnection bqWithUnderscores = new BQConnection(urlWithUnderscoreContainingProject, new Properties()); + Assert.assertEquals("example.com:project", bqWithUnderscores.getProjectId()); + Assert.assertEquals("example.com:project", bqWithUnderscores.getCatalog()); } catch (SQLException e){ fail("failed to get or parse url: " + e.getMessage()); }