Nr 280084 add support for postgre sql v17 (#176)

* Modified integration tests for Postgres 17

The values

These inventory variables are no longer present in Postgres 17 as per the release docs:
https://www.postgresql.org/docs/current/release-17.html

* Modified instance definitions to add support for Postgres17

* Modified Instance Definitions to reflect Postgres 17 changes

* Modified latest inventory to include new fields

* Made instance definition queries more extensible for newer versions

* Added checkpointer table metrics to the schema

* Added query to get dropped metrics for pg_stat_io

* Tests checking version defs are now table driven

* schema json shows metrics from pg_stat_io

* Modified test to check for correct instance definition order and elements

* Added seperate test for out of order
Simplified version specific tests

* Modified changelog to reflect Postgresv17 support

* Misspelling

* Fixed changelog format

* Fixed changelog for correct output and removed breaking changes

* Fixed unrelease changelog

CHANGELOG.md

@@ -9,8 +9,10 @@ Unreleased section should follow [Release Toolkit](https://github.com/newrelic/r
 
 ## Unreleased
 
-### Enhancements
+### enhancements
 - Added support for pgbouncer v1.23 with new columns in `STATS` table.
+- Added support for PostgreSQL v17
+- Metrics are updated to reflect PostgreSQL v17 metrics origin (metrics collected from `pg_stat_bgwriter` are now collected from `pg_stat_checkpointer` and `pg_stat_io`) when PostgreSQL v17 is used. Specifically: `bgwriter.buffersWrittenForCheckpointsPerSecond` → `checkpointer.buffersWrittenForCheckpointsPerSecond`, `bgwriter.checkpointSyncTimeInMillisecondsPerSecond` → `checkpointer.checkpointSyncTimeInMillisecondsPerSecond`, `bgwriter.checkpointWriteTimeInMillisecondsPerSecond` → `checkpointer.checkpointWriteTimeInMillisecondsPerSecond`, `bgwriter.checkpointsRequestedPerSecond` → `checkpointer.checkpointsRequestedPerSecond`, `bgwriter.checkpointsScheduledPerSecond` → `checkpointer.checkpointsScheduledPerSecond`, `bgwriter.backendFsyncCallsPerSecond` → `io.backendFsyncCallsPerSecond`, `bgwriter.buffersWrittenByBackendPerSecond` → `io.buffersWrittenByBackendPerSecond`.
 
 ## v2.15.0 - 2024-10-07
 

src/metrics/instance_definitions.go

@@ -4,22 +4,46 @@ import (
 	"github.com/blang/semver/v4"
 )
 
-func generateInstanceDefinitions(version *semver.Version) []*QueryDefinition {
-	queryDefinitions := make([]*QueryDefinition, 1, 3)
-	v91 := semver.MustParse("9.1.0")
-	v92 := semver.MustParse("9.2.0")
-
-	queryDefinitions[0] = instanceDefinitionBase
+type VersionDefinition struct {
+	minVersion       semver.Version
+	queryDefinitions []*QueryDefinition
+}
 
-	if version.GE(v91) {
-		queryDefinitions = append(queryDefinitions, instanceDefinition91)
-	}
+var versionDefinitions = []VersionDefinition{
+	{
+		minVersion: semver.MustParse("17.0.0"),
+		queryDefinitions: []*QueryDefinition{
+			instanceDefinitionBase170,
+			instanceDefinition170,
+			instanceDefinitionInputOutput170,
+		},
+	},
+	{
+		minVersion: semver.MustParse("9.2.0"),
+		queryDefinitions: []*QueryDefinition{
+			instanceDefinitionBase,
+			instanceDefinition91,
+			instanceDefinition92,
+		},
+	},
+	{
+		minVersion: semver.MustParse("9.1.0"),
+		queryDefinitions: []*QueryDefinition{
+			instanceDefinitionBase,
+			instanceDefinition91,
+		},
+	},
+}
 
-	if version.GE(v92) {
-		queryDefinitions = append(queryDefinitions, instanceDefinition92)
+func generateInstanceDefinitions(version *semver.Version) []*QueryDefinition {
+	// Find the first version definition that's applicable
+	for _, versionDef := range versionDefinitions {
+		if version.GE(versionDef.minVersion) {
+			return versionDef.queryDefinitions
+		}
 	}
 
-	return queryDefinitions
+	return []*QueryDefinition{instanceDefinitionBase}
 }
 
 var instanceDefinitionBase = &QueryDefinition{
@@ -62,6 +86,50 @@ var instanceDefinition92 = &QueryDefinition{
 
 	dataModels: []struct {
 		CheckpointWriteTime *int64 `db:"time_writing_checkpoint_files_to_disk"       metric_name:"bgwriter.checkpointWriteTimeInMillisecondsPerSecond" source_type:"rate"`
-		CheckpointSynTime   *int64 `db:"time_synchronizing_checkpoint_files_to_disk" metric_name:"bgwriter.checkpointSyncTimeInMillisecondsPerSecond"  source_type:"rate"`
+		CheckpointSyncTime  *int64 `db:"time_synchronizing_checkpoint_files_to_disk" metric_name:"bgwriter.checkpointSyncTimeInMillisecondsPerSecond"  source_type:"rate"`
+	}{},
+}
+
+var instanceDefinitionBase170 = &QueryDefinition{
+	query: `SELECT
+		BG.buffers_clean AS buffers_written_by_background_writer,
+		BG.maxwritten_clean AS background_writer_stops,
+		BG.buffers_alloc AS buffers_allocated
+		FROM pg_stat_bgwriter BG;`,
+
+	dataModels: []struct {
+		BuffersWrittenByBackgroundWriter *int64 `db:"buffers_written_by_background_writer" metric_name:"bgwriter.buffersWrittenByBackgroundWriterPerSecond" source_type:"rate"`
+		BackgroundWriterStops            *int64 `db:"background_writer_stops"              metric_name:"bgwriter.backgroundWriterStopsPerSecond"            source_type:"rate"`
+		BuffersAllocated                 *int64 `db:"buffers_allocated"                    metric_name:"bgwriter.buffersAllocatedPerSecond"                 source_type:"rate"`
+	}{},
+}
+
+var instanceDefinition170 = &QueryDefinition{
+	query: `SELECT 
+		CP.num_timed AS scheduled_checkpoints_performed,
+		CP.num_requested AS requested_checkpoints_performed,
+		CP.buffers_written AS buffers_written_during_checkpoint,
+		cast(CP.write_time AS bigint) AS time_writing_checkpoint_files_to_disk,
+		cast(CP.sync_time AS bigint) AS time_synchronizing_checkpoint_files_to_disk
+		FROM pg_stat_checkpointer CP;`,
+
+	dataModels: []struct {
+		ScheduledCheckpointsPerformed  *int64 `db:"scheduled_checkpoints_performed"             metric_name:"checkpointer.checkpointsScheduledPerSecond"                  source_type:"rate"`
+		RequestedCheckpointsPerformed  *int64 `db:"requested_checkpoints_performed"             metric_name:"checkpointer.checkpointsRequestedPerSecond"              source_type:"rate"`
+		BuffersWrittenDuringCheckpoint *int64 `db:"buffers_written_during_checkpoint"           metric_name:"checkpointer.buffersWrittenForCheckpointsPerSecond"      source_type:"rate"`
+		CheckpointWriteTime            *int64 `db:"time_writing_checkpoint_files_to_disk"       metric_name:"checkpointer.checkpointWriteTimeInMillisecondsPerSecond" source_type:"rate"`
+		CheckpointSyncTime             *int64 `db:"time_synchronizing_checkpoint_files_to_disk" metric_name:"checkpointer.checkpointSyncTimeInMillisecondsPerSecond"  source_type:"rate"`
+	}{},
+}
+
+var instanceDefinitionInputOutput170 = &QueryDefinition{
+	query: `SELECT 
+		SUM(IO.writes) AS buffers_written_by_backend, 
+		SUM(IO.fsyncs) AS times_backend_executed_own_fsync 
+		FROM pg_stat_io IO;`,
+
+	dataModels: []struct {
+		BuffersWrittenByBackend *int64 `db:"buffers_written_by_backend"       metric_name:"io.buffersWrittenByBackendPerSecond"  source_type:"rate"`
+		BackendExecutedOwnFsync *int64 `db:"times_backend_executed_own_fsync" metric_name:"io.backendFsyncCallsPerSecond"        source_type:"rate"`
 	}{},
 }

src/metrics/instance_definitions_test.go

@@ -7,30 +7,62 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_generateInstanceDefinitions90(t *testing.T) {
-	v := semver.MustParse("9.0.0")
-	queryDefinitions := generateInstanceDefinitions(&v)
+func Test_generateInstanceDefinitions(t *testing.T) {
+	tests := []struct {
+		name            string
+		version         string
+		expectedQueries []*QueryDefinition
+	}{
+		{
+			name:            "PostgreSQL 9.0",
+			version:         "9.0.0",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase},
+		},
+		{
+			name:            "PostgreSQL 9.1",
+			version:         "9.1.0",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase, instanceDefinition91},
+		},
+		{
+			name:            "PostgreSQL 9.2",
+			version:         "9.2.0",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase, instanceDefinition91, instanceDefinition92},
+		},
+		{
+			name:            "PostgreSQL 10.2",
+			version:         "10.2.0",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase, instanceDefinition91, instanceDefinition92},
+		},
+		{
+			name:            "PostgreSQL 16.4",
+			version:         "16.4.2",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase, instanceDefinition91, instanceDefinition92},
+		},
+		{
+			name:            "PostgreSQL 17.0",
+			version:         "17.0.0",
+			expectedQueries: []*QueryDefinition{instanceDefinitionBase170, instanceDefinition170, instanceDefinitionInputOutput170},
+		},
+	}
 
-	assert.Equal(t, 1, len(queryDefinitions))
-}
-
-func Test_generateInstanceDefinitions91(t *testing.T) {
-	v := semver.MustParse("9.1.0")
-	queryDefinitions := generateInstanceDefinitions(&v)
-
-	assert.Equal(t, 2, len(queryDefinitions))
-}
-
-func Test_generateInstanceDefinitions92(t *testing.T) {
-	v := semver.MustParse("9.2.0")
-	queryDefinitions := generateInstanceDefinitions(&v)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			version := semver.MustParse(tt.version)
+			queryDefinitions := generateInstanceDefinitions(&version)
+			assert.Equal(t, len(tt.expectedQueries), len(queryDefinitions))
+			assert.Equal(t, tt.expectedQueries, queryDefinitions)
+		})
+	}
 
-	assert.Equal(t, 3, len(queryDefinitions))
 }
 
-func Test_generateInstanceDefinitions10(t *testing.T) {
-	v := semver.MustParse("10.2.0")
-	queryDefinitions := generateInstanceDefinitions(&v)
+func Test_generateInstanceDefinitionsOutOfOrder(t *testing.T) {
+	t.Run("PostgreSQL 17.5 order check", func(t *testing.T) {
+		version := semver.MustParse("17.5.0")
+		queryDefinitions := generateInstanceDefinitions(&version)
+		expectedQueries := []*QueryDefinition{instanceDefinitionInputOutput170, instanceDefinition170, instanceDefinitionBase170}
 
-	assert.Equal(t, 3, len(queryDefinitions))
+		// This fails because order is different
+		assert.False(t, assert.ObjectsAreEqual(expectedQueries, queryDefinitions), "Query definitions should be in the correct order")
+	})
 }

tests/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       POSTGRES_DB: demo
 
   postgres-latest-supported:
-    image: postgres:16
+    image: postgres:17.0
     restart: always
     environment:
       POSTGRES_USER: postgres