HAWQ-1802. analyze inherits resource if possible

apache · Aug 27, 2021 · fac6938 · fac6938
1 parent b136a8e
commit fac6938
Show file tree

Hide file tree

Showing 18 changed files with 512 additions and 60 deletions.
diff --git a/contrib/magma/monitor_install.sql b/contrib/magma/monitor_install.sql
@@ -1,7 +1,7 @@
 ------------------------------------------------------------------
 -- hawq status
 ------------------------------------------------------------------
-CREATE FUNCTION hawq_magma_status() RETURNS SETOF record LANGUAGE internal VOLATILE STRICT AS 'hawq_magma_status' WITH (OID=5085, DESCRIPTION="Return magma node information");
+CREATE FUNCTION pg_catalog.hawq_magma_status() RETURNS SETOF record LANGUAGE internal VOLATILE STRICT AS 'hawq_magma_status' WITH (OID=5085, DESCRIPTION="Return magma node information");
 
 CREATE OR REPLACE FUNCTION pg_catalog.magma_getstatus() RETURNS void
 AS '$libdir/magma.so', 'magma_getstatus'

diff --git a/contrib/orc/hive_install.sql b/contrib/orc/hive_install.sql
@@ -6,10 +6,10 @@
 --
 -- --------------------------------------------------------------------
 
-CREATE OR REPLACE FUNCTION hive_validate() RETURNS void
+CREATE OR REPLACE FUNCTION pg_catalog.hive_validate() RETURNS void
 AS '$libdir/exthive.so', 'hiveprotocol_validate'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION hive_blocklocation() RETURNS void
+CREATE OR REPLACE FUNCTION pg_catalog.hive_blocklocation() RETURNS void
 AS '$libdir/exthive.so', 'hiveprotocol_blocklocation'
 LANGUAGE C STABLE;
diff --git a/contrib/orc/orc.c b/contrib/orc/orc.c
@@ -643,7 +643,7 @@ Datum orc_beginscan(PG_FUNCTION_ARGS)
         ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 	if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 	{
-		ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+		ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
 	}
 
 	/* 5. Save file_scan_desc */
@@ -772,7 +772,7 @@ Datum orc_getnext(PG_FUNCTION_ARGS) {
     ORCFormatEndORCFormatC(user_data->fmt);
     e = ORCFormatGetErrorORCFormatC(user_data->fmt);
     if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION) {
-      ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+      ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
     }
 
     ORCFormatFreeORCFormatC(&(user_data->fmt));
@@ -802,7 +802,7 @@ Datum orc_getnext(PG_FUNCTION_ARGS) {
     slot->PRIVATE_tts_values = NULL;
     ExecClearTuple(slot);
   } else {
-    ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+    ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
   }
   PG_RETURN_BOOL(false);
 }
@@ -865,14 +865,14 @@ Datum orc_rescan(PG_FUNCTION_ARGS)
 		ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 		if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 		{
-			ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+			ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
 		}
 	} else {
 		ORCFormatRescanORCFormatC(user_data->fmt);
 		ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 		if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 		{
-			ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+			ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
 		}
 	}
 
@@ -993,7 +993,7 @@ Datum orc_stopscan(PG_FUNCTION_ARGS)
     ORCFormatEndORCFormatC(user_data->fmt);
     e = ORCFormatGetErrorORCFormatC(user_data->fmt);
     if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION) {
-      ereport(LOG, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+      ereport(LOG, (errcode(e->errCode), errmsg("%s", e->errMessage)));
     }
 
     ORCFormatFreeORCFormatC(&(user_data->fmt));
@@ -1029,7 +1029,7 @@ Datum orc_stopscan(PG_FUNCTION_ARGS)
     tts->PRIVATE_tts_isnull = NULL;
     ExecClearTuple(tts);
   } else {
-    ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+    ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
   }
 
   PG_RETURN_VOID();
@@ -1261,7 +1261,7 @@ Datum orc_insert_init(PG_FUNCTION_ARGS)
 	ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 	if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 	{
-		ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+		ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
 	}
 
 	/* 4. Save the result */
@@ -1515,7 +1515,7 @@ Datum orc_insert(PG_FUNCTION_ARGS)
 	ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 	if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 	{
-		ereport(ERROR,(errcode(e->errCode),errmsg("ORC::%s", e->errMessage)));
+		ereport(ERROR,(errcode(e->errCode),errmsg("%s", e->errMessage)));
 	}
 
 	ps->ps_tuple_oid = InvalidOid;
@@ -1540,7 +1540,7 @@ Datum orc_insert_finish(PG_FUNCTION_ARGS)
 	ORCFormatCatchedError *e = ORCFormatGetErrorORCFormatC(user_data->fmt);
 	if (e->errCode != ERRCODE_SUCCESSFUL_COMPLETION)
 	{
-		ereport(ERROR, (errcode(e->errCode), errmsg("ORC:%s", e->errMessage)));
+		ereport(ERROR, (errcode(e->errCode), errmsg("%s", e->errMessage)));
 	}
 
 	ORCFormatFreeORCFormatC(&(user_data->fmt));

diff --git a/contrib/orc/orc_install.sql b/contrib/orc/orc_install.sql
@@ -60,26 +60,26 @@ CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void
 AS '$libdir/orc.so', 'orc_insert_finish'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION hdfs_validate() RETURNS void
+CREATE OR REPLACE FUNCTION pg_catalog.hdfs_validate() RETURNS void
 AS '$libdir/exthdfs.so', 'hdfsprotocol_validate'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION hdfs_blocklocation() RETURNS void
+CREATE OR REPLACE FUNCTION pg_catalog.hdfs_blocklocation() RETURNS void
 AS '$libdir/exthdfs.so', 'hdfsprotocol_blocklocation'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION csv_in() RETURNS record
+CREATE OR REPLACE FUNCTION pg_catalog.csv_in() RETURNS record
 AS '$libdir/extfmtcsv.so', 'extfmtcsv_in'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION csv_out(record) RETURNS bytea
+CREATE OR REPLACE FUNCTION pg_catalog.csv_out(record) RETURNS bytea
 AS '$libdir/extfmtcsv.so', 'extfmtcsv_out'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION text_in() RETURNS record
+CREATE OR REPLACE FUNCTION pg_catalog.text_in() RETURNS record
 AS '$libdir/extfmtcsv.so', 'extfmttext_in'
 LANGUAGE C STABLE;
 
-CREATE OR REPLACE FUNCTION text_out(record) RETURNS bytea
+CREATE OR REPLACE FUNCTION pg_catalog.text_out(record) RETURNS bytea
 AS '$libdir/extfmtcsv.so', 'extfmttext_out'
 LANGUAGE C STABLE;
diff --git a/src/Makefile b/src/Makefile
@@ -46,16 +46,18 @@ install-local: installdirs-local
 	$(INSTALL_DATA) $(srcdir)/nls-global.mk '$(DESTDIR)$(pgxsdir)/$(subdir)/nls-global.mk'
 
 installdirs: installdirs-local
-ifeq ($(with_oushudb_extension), yes)
-	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_* $(DESTDIR)${bindir}
-	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat* $(DESTDIR)${bindir}
-	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_magma_tool $(DESTDIR)${bindir}
-	${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/template-magma-site.xml $(DESTDIR)${sysconfdir}
-	${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/magma-site.xml $(DESTDIR)${sysconfdir}
-endif
 
 installdirs-local:
 	$(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(subdir)'
+ifeq ($(with_oushudb_extension), yes)
+	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_ctl '$(DESTDIR)${bindir}/magma_ctl'
+	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/magma_server '$(DESTDIR)${bindir}/magma_server'
+	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat_monitor '$(DESTDIR)${bindir}/hawq_heartbeat_monitor'
+	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_heartbeat_sender '$(DESTDIR)${bindir}/hawq_heartbeat_sender'
+	${INSTALL_PROGRAM} $(DEPENDENCY_PATH)/bin/hawq_magma_tool '$(DESTDIR)${bindir}/hawq_magma_tool'
+	${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/template-magma-site.xml '$(DESTDIR)${sysconfdir}/template-magma-site.xml'
+	${INSTALL_DATA} $(DEPENDENCY_PATH)/include/magma/utils/magma-site.xml '$(DESTDIR)${sysconfdir}/magma-site.xml'
+endif
 
 uninstall: uninstall-local
 

diff --git a/src/backend/access/orc/orcam.c b/src/backend/access/orc/orcam.c
@@ -122,7 +122,7 @@ static void checkOrcError(OrcFormatData *orcFormatData) {
     ORCFormatFreeORCFormatC(&orcFormatData->fmt);
     ORCFormatFreeStorageFormatC(&orcFormatData->updateDeleteFmt);
     ereport(ERROR,
-            (errcode(errBuf.errCode), errmsg("ORC: %s", errBuf.errMessage)));
+            (errcode(errBuf.errCode), errmsg("%s", errBuf.errMessage)));
   }
 }
 

diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
@@ -144,6 +144,7 @@ install-data: $(BKIFILES) installdirs
 	$(INSTALL_DATA) $(call vpathsearch,filesystem.sql) '$(DESTDIR)$(datadir)/filesystem.sql'
 	$(INSTALL_DATA) $(srcdir)/../utils/adt/array_distance_install.sql '$(DESTDIR)$(datadir)/array_distance_install.sql'
 	$(INSTALL_DATA) $(srcdir)/../utils/adt/array_distance_uninstall.sql '$(DESTDIR)$(datadir)/array_distance_uninstall.sql'
+	$(INSTALL_DATA) $(srcdir)/../utils/adt/json_install.sql '$(DESTDIR)$(datadir)/json_install.sql'
 
 ifeq ($(with_orc), yes)
 	$(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_install.sql '$(DESTDIR)$(datadir)/orc_install.sql'

diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c
@@ -970,6 +970,9 @@ relationId == PgResQueueToastIndex ||
 Oid
 GetNewOid(Relation relation)
 {
+  if (gp_upgrade_mode && gen_new_oid_value != InvalidOid) {
+    return gen_new_oid_value;
+  }
 	Oid			newOid;
 	Oid			oidIndex;
 	Relation	indexrel;

diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
@@ -790,6 +790,21 @@ void CheckAttributeForOrc(TupleDesc desc) {
                       errmsg("array type of column \"%s\" is not supported yet",
                              NameStr(desc->attrs[i]->attname))));
     }
+
+    int32_t datatype =
+        (int32_t)(((Form_pg_attribute)(desc->attrs[i]))->atttypid);
+    int4 typmod = ((Form_pg_attribute)(desc->attrs[i]))->atttypmod;
+    if (HAWQ_TYPE_NUMERIC == datatype) {
+      int4 tmp_typmod = typmod - VARHDRSZ;
+      int precision = (tmp_typmod >> 16) & 0xffff;
+      int scale = tmp_typmod & 0xffff;
+      if (precision < 1 || 38 < precision)
+        ereport(ERROR,
+                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+                 errmsg("ORC DECIMAL precision must be between 1 and 38")));
+      if (scale == 0)
+        ereport(NOTICE, (errmsg("Using a scale of zero for ORC DECIMAL")));
+    }
   }
 }
 

diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
@@ -522,15 +522,24 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 	CurrentResourceOwner = asOwner;
 
 	/**
-	 *  we use preferred_seg_num as default and
-	 *  compute target_seg_num based on data size and distributed type
-	 *  if there is no preferred_seg_num.
+	 * We allocate query resource for analyze if there is no active resource
+	 * otherwise, we inherit instead
 	 */
+	QueryResource *savedResource = NULL;
+	savedResource = GetActiveQueryResource();
+
+	QueryResource *resource = NULL;
 	int target_seg_num = preferred_seg_num;
-	if (target_seg_num <= 0) {
-		target_seg_num = calculate_virtual_segment_number(lFullRelOids);
+	if (!savedResource) {
+		/**
+		 *  we use preferred_seg_num as default and
+		 *  compute target_seg_num based on data size and distributed type
+		 *  if there is no preferred_seg_num.
+		 */
+		if (target_seg_num <= 0) {
+			target_seg_num = calculate_virtual_segment_number(lFullRelOids);
+		}
 	}
-	elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
 
 	/**
 	 * We open relations with appropreciate locks
@@ -559,13 +568,12 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 		}
 	}
 
-	/**
-	 * We allocate query resource for analyze
-	 */
-	QueryResource *resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
-	QueryResource *savedResource = NULL;
-
-	savedResource = GetActiveQueryResource();
+	if (!savedResource) {
+		elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
+		resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
+	} else {
+		resource = AllocateResource(QRL_INHERIT, 0, 0, 0, 0, NULL, 0);
+	}
 	SetActiveQueryResource(resource);
 
 	/**

diff --git a/src/backend/utils/adt/array_distance_install.sql b/src/backend/utils/adt/array_distance_install.sql
@@ -6,10 +6,10 @@
 --
 -- --------------------------------------------------------------------
 
-CREATE OR REPLACE FUNCTION euclidean_metric_float4array(anyarray, anyarray) RETURNS float4 LANGUAGE internal IMMUTABLE AS 'euclidean_metric_float4array';
+CREATE OR REPLACE FUNCTION pg_catalog.euclidean_metric_float4array(anyarray, anyarray) RETURNS float4 LANGUAGE internal IMMUTABLE AS 'euclidean_metric_float4array';
 
-CREATE OR REPLACE FUNCTION euclidean_metric_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal STABLE AS 'euclidean_metric_float8array';
+CREATE OR REPLACE FUNCTION pg_catalog.euclidean_metric_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal STABLE AS 'euclidean_metric_float8array';
 
-CREATE OR REPLACE FUNCTION cosine_distance_float4array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float4array';
+CREATE OR REPLACE FUNCTION pg_catalog.cosine_distance_float4array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float4array';
 
-CREATE OR REPLACE FUNCTION cosine_distance_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float8array';
+CREATE OR REPLACE FUNCTION pg_catalog.cosine_distance_float8array(anyarray, anyarray) RETURNS float8 LANGUAGE internal IMMUTABLE AS 'cosine_distance_float8array';