Adds PG17beta2 support

.devcontainer/Dockerfile

@@ -103,6 +103,18 @@ RUN mkdir .pgenv-staging/
 RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
 RUN rm .pgenv-staging/config/default.conf
 
+FROM base AS pg17
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 17beta2
+RUN rm .pgenv/src/*.tar*
+RUN make -C .pgenv/src/postgresql-*/ clean
+RUN make -C .pgenv/src/postgresql-*/src/include install
+
+# create a staging directory with all files we want to copy from our pgenv build
+# we will copy the contents of the staged folder into the final image at once
+RUN mkdir .pgenv-staging/
+RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
+RUN rm .pgenv-staging/config/default.conf
+
 FROM base AS uncrustify-builder
 
 RUN sudo apt update && sudo apt install -y cmake tree

.gitattributes

@@ -29,6 +29,7 @@ src/backend/distributed/deparser/ruleutils_13.c -citus-style
 src/backend/distributed/deparser/ruleutils_14.c -citus-style
 src/backend/distributed/deparser/ruleutils_15.c -citus-style
 src/backend/distributed/deparser/ruleutils_16.c -citus-style
+src/backend/distributed/deparser/ruleutils_17.c -citus-style
 src/backend/distributed/commands/index_pg_source.c -citus-style
 
 src/include/distributed/citus_nodes.h -citus-style

.github/workflows/build_and_test.yml

@@ -32,11 +32,12 @@ jobs:
       style_checker_image_name: "ghcr.io/citusdata/stylechecker"
       style_checker_tools_version: "0.8.18"
       sql_snapshot_pg_version: "16.3"
-      image_suffix: "-v13fd57c"
+      image_suffix: "-dev-16d4616"
       pg14_version: '{ "major": "14", "full": "14.12" }'
       pg15_version: '{ "major": "15", "full": "15.7" }'
       pg16_version: '{ "major": "16", "full": "16.3" }'
-      upgrade_pg_versions: "14.12-15.7-16.3"
+      pg17_version: '{ "major": "17", "full": "17beta2" }'
+      upgrade_pg_versions: "14.12-15.7-16.3-17beta2"
     steps:
       # Since GHA jobs needs at least one step we use a noop step here.
       - name: Set up parameters
@@ -113,6 +114,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
@@ -144,6 +146,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
         make:
           - check-split
           - check-multi
@@ -173,6 +176,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-failure
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-enterprise-failure
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -185,6 +192,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-enterprise-failure
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: check-pytest
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -197,6 +208,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-pytest
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
           - make: installcheck
             suite: cdc
             image_name: ${{ needs.params.outputs.test_image_name }}
@@ -205,6 +220,10 @@ jobs:
             suite: cdc
             image_name: ${{ needs.params.outputs.test_image_name }}
             pg_version: ${{ needs.params.outputs.pg16_version }}
+          - make: installcheck
+            suite: cdc
+            image_name: ${{ needs.params.outputs.test_image_name }}
+            pg_version: ${{ needs.params.outputs.pg17_version }}
           - make: check-query-generator
             pg_version: ${{ needs.params.outputs.pg14_version }}
             suite: regress
@@ -217,6 +236,10 @@ jobs:
             pg_version: ${{ needs.params.outputs.pg16_version }}
             suite: regress
             image_name: ${{ needs.params.outputs.fail_test_image_name }}
+          - make: check-query-generator
+            pg_version: ${{ needs.params.outputs.pg17_version }}
+            suite: regress
+            image_name: ${{ needs.params.outputs.fail_test_image_name }}
     runs-on: ubuntu-20.04
     container:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ needs.params.outputs.image_suffix }}"
@@ -260,6 +283,7 @@ jobs:
           - ${{ needs.params.outputs.pg14_version }}
           - ${{ needs.params.outputs.pg15_version }}
           - ${{ needs.params.outputs.pg16_version }}
+          - ${{ needs.params.outputs.pg17_version }}
         parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
     steps:
     - uses: actions/checkout@v4
@@ -308,6 +332,10 @@ jobs:
             new_pg_major: 16
           - old_pg_major: 14
             new_pg_major: 16
+          - old_pg_major: 16
+            new_pg_major: 17
+          - old_pg_major: 15
+            new_pg_major: 17
     env:
       old_pg_major: ${{ matrix.old_pg_major }}
       new_pg_major: ${{ matrix.new_pg_major }}
@@ -391,7 +419,7 @@ jobs:
       CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.test_image_name }}:${{ fromJson(needs.params.outputs.pg17_version).full }}${{ needs.params.outputs.image_suffix }}
     needs:
       - params
       - test-citus
@@ -502,7 +530,7 @@ jobs:
     name: Test flakyness
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg16_version).full }}${{ needs.params.outputs.image_suffix }}
+      image: ${{ needs.params.outputs.fail_test_image_name }}:${{ fromJson(needs.params.outputs.pg17_version).full }}${{ needs.params.outputs.image_suffix }}
       options: --user root
     env:
       runs: 8

configure

@@ -2588,7 +2588,7 @@ fi
 if test "$with_pg_version_check" = no; then
     { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num (skipped compatibility check)" >&5
 $as_echo "$as_me: building against PostgreSQL $version_num (skipped compatibility check)" >&6;}
-elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16' -a  "$version_num" != '17'; then
    as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5
 else
    { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5

configure.ac

@@ -80,7 +80,7 @@ AC_SUBST(with_pg_version_check)
 
 if test "$with_pg_version_check" = no; then
     AC_MSG_NOTICE([building against PostgreSQL $version_num (skipped compatibility check)])
-elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16'; then
+elif test "$version_num" != '14' -a "$version_num" != '15' -a  "$version_num" != '16' -a  "$version_num" != '17'; then
    AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.])
 else
    AC_MSG_NOTICE([building against PostgreSQL $version_num])

src/backend/columnar/columnar_customscan.c

@@ -363,7 +363,7 @@ ColumnarGetRelationInfoHook(PlannerInfo *root, Oid relationObjectId,
 
 		/* disable index-only scan */
 		IndexOptInfo *indexOptInfo = NULL;
-		foreach_ptr(indexOptInfo, rel->indexlist)
+		foreach_declared_ptr(indexOptInfo, rel->indexlist)
 		{
 			memset(indexOptInfo->canreturn, false, indexOptInfo->ncolumns * sizeof(bool));
 		}
@@ -381,7 +381,7 @@ RemovePathsByPredicate(RelOptInfo *rel, PathPredicate removePathPredicate)
 	List *filteredPathList = NIL;
 
 	Path *path = NULL;
-	foreach_ptr(path, rel->pathlist)
+	foreach_declared_ptr(path, rel->pathlist)
 	{
 		if (!removePathPredicate(path))
 		{
@@ -428,7 +428,7 @@ static void
 CostColumnarPaths(PlannerInfo *root, RelOptInfo *rel, Oid relationId)
 {
 	Path *path = NULL;
-	foreach_ptr(path, rel->pathlist)
+	foreach_declared_ptr(path, rel->pathlist)
 	{
 		if (IsA(path, IndexPath))
 		{
@@ -783,7 +783,7 @@ ExtractPushdownClause(PlannerInfo *root, RelOptInfo *rel, Node *node)
 		List *pushdownableArgs = NIL;
 
 		Node *boolExprArg = NULL;
-		foreach_ptr(boolExprArg, boolExpr->args)
+		foreach_declared_ptr(boolExprArg, boolExpr->args)
 		{
 			Expr *pushdownableArg = ExtractPushdownClause(root, rel,
 														  (Node *) boolExprArg);
@@ -1550,7 +1550,7 @@ ColumnarPerStripeScanCost(RelOptInfo *rel, Oid relationId, int numberOfColumnsRe
 	uint32 maxColumnCount = 0;
 	uint64 totalStripeSize = 0;
 	StripeMetadata *stripeMetadata = NULL;
-	foreach_ptr(stripeMetadata, stripeList)
+	foreach_declared_ptr(stripeMetadata, stripeList)
 	{
 		totalStripeSize += stripeMetadata->dataLength;
 		maxColumnCount = Max(maxColumnCount, stripeMetadata->columnCount);
@@ -1924,11 +1924,6 @@ ColumnarScan_EndCustomScan(CustomScanState *node)
 	 */
 	TableScanDesc scanDesc = node->ss.ss_currentScanDesc;
 
-	/*
-	 * Free the exprcontext
-	 */
-	ExecFreeExprContext(&node->ss.ps);
-
 	/*
 	 * clean out the tuple table
 	 */

src/backend/columnar/columnar_metadata.c

@@ -2041,7 +2041,7 @@ GetHighestUsedRowNumber(uint64 storageId)
 	List *stripeMetadataList = ReadDataFileStripeList(storageId,
 													  GetTransactionSnapshot());
 	StripeMetadata *stripeMetadata = NULL;
-	foreach_ptr(stripeMetadata, stripeMetadataList)
+	foreach_declared_ptr(stripeMetadata, stripeMetadataList)
 	{
 		highestRowNumber = Max(highestRowNumber,
 							   StripeGetHighestRowNumber(stripeMetadata));

src/backend/columnar/columnar_reader.c

@@ -880,7 +880,7 @@ ReadChunkGroupNextRow(ChunkGroupReadState *chunkGroupReadState, Datum *columnVal
 	memset(columnNulls, true, sizeof(bool) * chunkGroupReadState->columnCount);
 
 	int attno;
-	foreach_int(attno, chunkGroupReadState->projectedColumnList)
+	foreach_declared_int(attno, chunkGroupReadState->projectedColumnList)
 	{
 		const ChunkData *chunkGroupData = chunkGroupReadState->chunkGroupData;
 		const int rowIndex = chunkGroupReadState->currentRow;
@@ -1489,7 +1489,7 @@ ProjectedColumnMask(uint32 columnCount, List *projectedColumnList)
 	bool *projectedColumnMask = palloc0(columnCount * sizeof(bool));
 	int attno;
 
-	foreach_int(attno, projectedColumnList)
+	foreach_declared_int(attno, projectedColumnList)
 	{
 		/* attno is 1-indexed; projectedColumnMask is 0-indexed */
 		int columnIndex = attno - 1;

src/backend/columnar/columnar_tableam.c

@@ -1424,15 +1424,32 @@ ConditionalLockRelationWithTimeout(Relation rel, LOCKMODE lockMode, int timeout,
 
 
 static bool
-columnar_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno,
+columnar_scan_analyze_next_block(TableScanDesc scan,
+#if PG_VERSION_NUM >= PG_VERSION_17
+								 ReadStream *stream)
+#else
+								 BlockNumber blockno,
 								 BufferAccessStrategy bstrategy)
+#endif
 {
 	/*
 	 * Our access method is not pages based, i.e. tuples are not confined
 	 * to pages boundaries. So not much to do here. We return true anyway
 	 * so acquire_sample_rows() in analyze.c would call our
 	 * columnar_scan_analyze_next_tuple() callback.
+	 * In PG17, we return false in case there is no buffer left, since
+	 * the outer loop changed in acquire_sample_rows(), and it is
+	 * expected for the scan_analyze_next_block function to check whether
+	 * there are any blocks left in the block sampler.
 	 */
+#if PG_VERSION_NUM >= PG_VERSION_17
+	Buffer buf = read_stream_next_buffer(stream, NULL);
+	if (!BufferIsValid(buf))
+	{
+		return false;
+	}
+	ReleaseBuffer(buf);
+#endif
 	return true;
 }
 
@@ -2239,7 +2256,9 @@ ColumnarProcessAlterTable(AlterTableStmt *alterTableStmt, List **columnarOptions
 									"Specify SET ACCESS METHOD before storage parameters, or use separate ALTER TABLE commands.")));
 			}
 
-			destIsColumnar = (strcmp(alterTableCmd->name, COLUMNAR_AM_NAME) == 0);
+			destIsColumnar = (strcmp(alterTableCmd->name ? alterTableCmd->name :
+									 default_table_access_method,
+									 COLUMNAR_AM_NAME) == 0);
 
 			if (srcIsColumnar && !destIsColumnar)
 			{
@@ -3083,7 +3102,7 @@ DefElem *
 GetExtensionOption(List *extensionOptions, const char *defname)
 {
 	DefElem *defElement = NULL;
-	foreach_ptr(defElement, extensionOptions)
+	foreach_declared_ptr(defElement, extensionOptions)
 	{
 		if (IsA(defElement, DefElem) &&
 			strncmp(defElement->defname, defname, NAMEDATALEN) == 0)

src/backend/distributed/cdc/cdc_decoder.c

@@ -22,6 +22,8 @@
 #include "utils/rel.h"
 #include "utils/typcache.h"
 
+#include "pg_version_constants.h"
+
 PG_MODULE_MAGIC;
 
 extern void _PG_output_plugin_init(OutputPluginCallbacks *cb);
@@ -435,6 +437,74 @@ TranslateChangesIfSchemaChanged(Relation sourceRelation, Relation targetRelation
 		return;
 	}
 
+#if PG_VERSION_NUM >= PG_VERSION_17
+
+	/* Check the ReorderBufferChange's action type and handle them accordingly.*/
+	switch (change->action)
+	{
+		case REORDER_BUFFER_CHANGE_INSERT:
+		{
+			/* For insert action, only new tuple should always be translated*/
+			HeapTuple sourceRelationNewTuple = change->data.tp.newtuple;
+			HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc(
+				sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+			change->data.tp.newtuple = targetRelationNewTuple;
+			break;
+		}
+
+		/*
+		 * For update changes both old and new tuples need to be translated for target relation
+		 * if the REPLICA IDENTITY is set to FULL. Otherwise, only the new tuple needs to be
+		 * translated for target relation.
+		 */
+		case REORDER_BUFFER_CHANGE_UPDATE:
+		{
+			/* For update action, new tuple should always be translated*/
+			/* Get the new tuple from the ReorderBufferChange, and translate it to target relation. */
+			HeapTuple sourceRelationNewTuple = change->data.tp.newtuple;
+			HeapTuple targetRelationNewTuple = GetTupleForTargetSchemaForCdc(
+				sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+			change->data.tp.newtuple = targetRelationNewTuple;
+
+			/*
+			 * Format oldtuple according to the target relation. If the column values of replica
+			 * identiy change, then the old tuple is non-null and needs to be formatted according
+			 * to the target relation schema.
+			 */
+			if (change->data.tp.oldtuple != NULL)
+			{
+				HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple;
+				HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc(
+					sourceRelationOldTuple,
+					sourceRelationDesc,
+					targetRelationDesc);
+
+				change->data.tp.oldtuple = targetRelationOldTuple;
+			}
+			break;
+		}
+
+		case REORDER_BUFFER_CHANGE_DELETE:
+		{
+			/* For delete action, only old tuple should be translated*/
+			HeapTuple sourceRelationOldTuple = change->data.tp.oldtuple;
+			HeapTuple targetRelationOldTuple = GetTupleForTargetSchemaForCdc(
+				sourceRelationOldTuple,
+				sourceRelationDesc,
+				targetRelationDesc);
+
+			change->data.tp.oldtuple = targetRelationOldTuple;
+			break;
+		}
+
+		default:
+		{
+			/* Do nothing for other action types. */
+			break;
+		}
+	}
+#else
+
 	/* Check the ReorderBufferChange's action type and handle them accordingly.*/
 	switch (change->action)
 	{
@@ -499,4 +569,5 @@ TranslateChangesIfSchemaChanged(Relation sourceRelation, Relation targetRelation
 			break;
 		}
 	}
+#endif
 }