updated database resiliency

Ango/Get code prod droplet.bru

@@ -14,6 +14,6 @@ body:json {
   {
     "batchid": "11111111-1111-1111-1111-111111111111",
     "clientid": "217be7c8-679c-4e08-bffc-db3451bdcdbf",
-    "customerid": "50b0b41b-c665-4409-a2bb-a4fc18828dc2"
+    "customerid": "88e92f73-e6a9-4ab6-bb04-3861e715e9f8"
   }
 }

main.go

@@ -72,25 +72,18 @@ func connectToDB() (*pgxpool.Pool, error) {
 
 	for i := 0; i < maxRetries; i++ {
 		config, _ := pgxpool.ParseConfig(databaseURL)
-		config.MaxConns = 20 // Adjust based on expected workload
-		config.MaxConnIdleTime = 30 * time.Minute
-		config.MaxConnLifetime = 2 * time.Hour
-		config.HealthCheckPeriod = 1 * time.Minute // Add health check period
-		config.ConnConfig.ConnectTimeout = 5 * time.Second // Add connection timeout
+		config.MaxConns = 20
+		config.MaxConnIdleTime = 30 * time.Second // Reduced from 30 minutes
+		config.MaxConnLifetime = 1 * time.Hour // Reduced from 2 hours
+		config.HealthCheckPeriod = 1 * time.Minute
+		config.ConnConfig.ConnectTimeout = 5 * time.Second
 
 		db, err = pgxpool.ConnectConfig(context.Background(), config)
 		if err == nil {
-			// Test the connection by querying the "batches" table
 			err = testDBConnection(db)
 			if err == nil {
 				break
 			}
-			// Test the connection by querying the database
-			var testResult int
-			err = db.QueryRow(context.Background(), "SELECT 1").Scan(&testResult)
-			if err != nil {
-				return nil, fmt.Errorf("error testing database connection: %v", err)
-			}
 		}
 		log.Printf("Error connecting to database (attempt %d/%d): %v\nDatabase URL: %s", i+1, maxRetries, err, databaseURL)
 		time.Sleep(5 * time.Second)
@@ -100,7 +93,7 @@ func connectToDB() (*pgxpool.Pool, error) {
 }
 
 func monitorDBConnections(pool *pgxpool.Pool) {
-	ticker := time.NewTicker(1 * time.Minute)
+	ticker := time.NewTicker(30 * time.Second) // Increased frequency
 	defer ticker.Stop()
 
 	for range ticker.C {
@@ -131,7 +124,7 @@ func monitorDBConnections(pool *pgxpool.Pool) {
 		_, err = pool.Exec(ctx, `
 			SELECT pg_terminate_backend(pid)
 			FROM pg_stat_activity
-			WHERE state = 'active'
+			WHERE state = 'idle in transaction'
 			  AND state_change < NOW() - INTERVAL '30 seconds'
 			  AND query NOT LIKE '%pg_terminate_backend%'
 		`)

service.go

@@ -55,6 +55,9 @@ func getCode(ctx context.Context, req Request) (string, error) {
 		}
 	}
 
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
 	tx, err := db.Begin(ctx)
 	if err != nil {
 		return "", err
@@ -115,17 +118,17 @@ func getCode(ctx context.Context, req Request) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	if time.Now().Sub(updateCodesTime) > 100*time.Millisecond {
-		log.Printf("Query for updating codes took long (%v)ms", time.Now().Sub(updateCodesTime))
+	if time.Since(updateCodesTime) > 100*time.Millisecond {
+		log.Printf("Query for updating codes took long (%v)ms", time.Since(updateCodesTime))
 	}
 
 	insertCodeUsageTime := time.Now()
 	_, err = tx.Exec(ctx, "INSERT INTO code_usage (code, batch_id, client_id, customer_id, used_at) VALUES ($1, $2, $3, $4, $5)", code, req.BatchID, req.ClientID, req.CustomerID, time.Now())
 	if err != nil {
 		return "", err
 	}
-	if time.Now().Sub(insertCodeUsageTime) > 100*time.Millisecond {
-		log.Printf("Query for inserting codes took long (%v)ms", time.Now().Sub(insertCodeUsageTime))
+	if time.Since(insertCodeUsageTime) > 100*time.Millisecond {
+		log.Printf("Query for inserting codes took long (%v)ms", time.Since(insertCodeUsageTime))
 	}
 
 	if err = tx.Commit(ctx); err != nil {