Skip to content

Commit

Permalink
Change query log config format from JSON to TABLE (#257)
Browse files Browse the repository at this point in the history 
Change query log config format from JSON to TABLE & added support for json output format
  • Loading branch information
@bhupendray-yb
bhupendray-yb authored Oct 24, 2024
1 parent 272b666 commit 2e83931
Show file tree
Hide file tree
Showing 9 changed files with 361 additions and 100 deletions.
42 changes: 15 additions & 27 deletions cmd/cluster/log-exporter/query_log_exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package query_log_exporter

import (
"fmt"
"os"
"strconv"
"strings"

Expand Down Expand Up @@ -75,15 +74,7 @@ var enableDbQueryLoggingCmd = &cobra.Command{
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}

DbQueryLoggingContext := formatter.Context{
Output: os.Stdout,
Format: formatter.NewDbQueryLoggingFormat(),
}

err = formatter.DbQueryLoggingWrite(DbQueryLoggingContext, []ybmclient.PgLogExporterConfigData{resp.GetData()})
if err != nil {
fmt.Println(err.Error())
}
formatter.DbQueryLoggingWriteFull(resp.GetData(), integrationName)
},
}

Expand Down Expand Up @@ -115,15 +106,15 @@ var describeLogExporterCmd = &cobra.Command{
return
}

DbQueryLoggingContext := formatter.Context{
Output: os.Stdout,
Format: formatter.NewDbQueryLoggingFormat(),
}
pgLogExporterConfigData := resp.GetData()[0]
integrationName, integrationId := "", pgLogExporterConfigData.Spec.ExporterId

err = formatter.DbQueryLoggingWrite(DbQueryLoggingContext, resp.GetData())
integrationName, err = authApi.GetIntegrationNameFromId(integrationId)
if err != nil {
fmt.Println(err.Error())
logrus.Debugf("could not fetch associated name for integration id: %s", integrationId)
}

formatter.DbQueryLoggingWriteFull(resp.GetData()[0], integrationName)
},
}

Expand Down Expand Up @@ -220,15 +211,20 @@ var updateLogExporterConfigCmd = &cobra.Command{
exporterConfigId := logExporterData.Info.Id

var integrationId string = ""
var integrationName string = ""
// use integration name if provided by user, else use existing one
if cmd.Flags().Changed("integration-name") {
integrationName, _ := cmd.Flags().GetString("integration-name")
integrationName, _ = cmd.Flags().GetString("integration-name")
integrationId, err = authApi.GetIntegrationIdFromName(integrationName)
if err != nil {
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}
} else {
integrationId = logExporterData.Spec.ExporterId
integrationName, err = authApi.GetIntegrationNameFromId(integrationId)
if err != nil {
logrus.Debugf("could not fetch associated name for integration id: %s", integrationId)
}
}

existingExportConfig := logExporterData.Spec.ExportConfig
Expand All @@ -243,16 +239,8 @@ var updateLogExporterConfigCmd = &cobra.Command{
logrus.Fatalf(ybmAuthClient.GetApiErrorDetails(err))
}

DbQueryLoggingContext := formatter.Context{
Output: os.Stdout,
Format: formatter.NewDbQueryLoggingFormat(),
}

err = formatter.DbQueryLoggingWrite(DbQueryLoggingContext,
[]ybmclient.PgLogExporterConfigData{pgLogExporterConfigResponse.GetData()})
if err != nil {
fmt.Println(err.Error())
}
fmt.Println("Request submitted to edit DB query log config for the cluster, this may take a few minutes...")
formatter.DbQueryLoggingWriteFull(pgLogExporterConfigResponse.GetData(), integrationName)
},
}

Expand Down
1 change: 1 addition & 0 deletions cmd/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ var _ = Describe("Cluster", func() {
Expect(err).ToNot(HaveOccurred())
os.Setenv("YBM_HOST", fmt.Sprintf("http://%s", server.Addr()))
os.Setenv("YBM_APIKEY", "test-token")
os.Setenv("YBM_FF_CONNECTION_POOLING", "true")
})

Describe("Pausing cluster", func() {
Expand Down
255 changes: 212 additions & 43 deletions cmd/db_query_logs_exporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,19 @@ var _ = Describe("DB Query Logging", func() {
Expect(err).NotTo(HaveOccurred())
session.Wait(2)
// Specials characters like $ % [ ] etc must be escaped using backslash to do exact matching
Expect(session.Out).Should(gbytes.Say(`State Integration ID Log Config
ENABLING 9e740000-331b-4dec-89b0-4e59b81e9019 {"debug_print_plan":false,"log_connections":false,"log_disconnections":false,"log_duration":false,"log_error_verbosity":"DEFAULT","log_line_prefix":"\%m :\%r :\%u @ \%d :\[\%p\] :","log_min_duration_statement":-1,"log_min_error_statement":"ERROR","log_statement":"NONE"}`))
Expect(session.Out).Should(gbytes.Say(`State Integration Name
ENABLING datadog-tp
Log Config Key Log Config Value
debug-print-plan false
log-min-duration-statement -1
log-connections false
log-disconnections false
log-duration false
log-error-verbosity DEFAULT
log-statement NONE
log-min-error-statement ERROR
log-line-prefix \%m :\%r :\%u @ \%d :\[\%p\] :`))
Expect(server.ReceivedRequests()).Should(HaveLen(5))
session.Kill()
})
Expand Down Expand Up @@ -155,8 +166,19 @@ ENABLING 9e740000-331b-4dec-89b0-4e59b81e9019 {"debug_print_plan":false,"log

Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`State Integration ID Log Config
ENABLING 9e740000-331b-4dec-89b0-4e59b81e9019 {"debug_print_plan":true,"log_connections":true,"log_disconnections":false,"log_duration":false,"log_error_verbosity":"TERSE","log_line_prefix":"\%m :\%r :\%u @ \%d :\[\%p\] : \%a :","log_min_duration_statement":50,"log_min_error_statement":"ERROR","log_statement":"MOD"}`))
Expect(session.Out).Should(gbytes.Say(`State Integration Name
ENABLING datadog-tp
Log Config Key Log Config Value
debug-print-plan true
log-min-duration-statement 50
log-connections true
log-disconnections false
log-duration false
log-error-verbosity TERSE
log-statement MOD
log-min-error-statement ERROR
log-line-prefix \%m :\%r :\%u @ \%d :\[\%p\] : \%a :`))
Expect(server.ReceivedRequests()).Should(HaveLen(5))
session.Kill()
})
Expand Down Expand Up @@ -206,63 +228,210 @@ You can check the status via \$ ybm cluster db-query-logging describe --cluster-
),
)

err = loadJson("./test/fixtures/list-telemetry-provider.json", &responseIntegrationList)
Expect(err).ToNot(HaveOccurred())

statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/telemetry-providers"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseIntegrationList),
),
)

cmd := exec.Command(compiledCLIPath, "cluster", "db-query-logging", "describe",
"--cluster-name", "stunning-sole",
)
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)

Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`State Integration ID Log Config
ACTIVE 9e740000-331b-4dec-89b0-4e59b81e9019 {"debug_print_plan":false,"log_connections":true,"log_disconnections":false,"log_duration":false,"log_error_verbosity":"DEFAULT","log_line_prefix":"\%m :\%r :\%u @ \%d :\[\%p\] : \%a :","log_min_duration_statement":30,"log_min_error_statement":"ERROR","log_statement":"MOD"}`))
Expect(server.ReceivedRequests()).Should(HaveLen(4))
Expect(session.Out).Should(gbytes.Say(`State Integration Name
ACTIVE datadog-tp
Log Config Key Log Config Value
debug-print-plan false
log-min-duration-statement 30
log-connections true
log-disconnections false
log-duration false
log-error-verbosity DEFAULT
log-statement MOD
log-min-error-statement ERROR
log-line-prefix \%m :\%r :\%u @ \%d :\[\%p\] : \%a :`))
Expect(server.ReceivedRequests()).Should(HaveLen(5))
session.Kill()
})
})

Describe("When updating query log exporter config", func() {
It("should update log config with provided args", func() {
// Load existing log export config
err := loadJson("./test/fixtures/db-query-log-exporter-describe-resp.json", &pgLogExporterConfigListResponse)
Expect(err).ToNot(HaveOccurred())
Context("without integration name", func() {
It("should update log config with provided args", func() {
// Load existing log export config
err := loadJson("./test/fixtures/db-query-log-exporter-describe-resp.json", &pgLogExporterConfigListResponse)
Expect(err).ToNot(HaveOccurred())

statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterConfigListResponse),
),
)
statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterConfigListResponse),
),
)

// Load updated log export config response
err = loadJson("./test/fixtures/db-query-log-exporter-update-config-resp.json", &pgLogExporterResponse)
Expect(err).ToNot(HaveOccurred())
err = loadJson("./test/fixtures/list-telemetry-provider.json", &responseIntegrationList)
Expect(err).ToNot(HaveOccurred())

statusCode = 202
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodPut, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs/388c41b9-81f7-4ed9-8239-ab22f4aaca91"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterResponse),
),
)
statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/telemetry-providers"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseIntegrationList),
),
)

cmd := exec.Command(compiledCLIPath, "cluster", "db-query-logging", "update",
"--cluster-name", "stunning-sole",
// Change few query log configs
"--debug-print-plan", "true",
"--log-connections", "false",
"--log-min-duration-statement", "60",
"--log-statement", "ALL",
)
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
// Load updated log export config response
err = loadJson("./test/fixtures/db-query-log-exporter-update-config-resp.json", &pgLogExporterResponse)
Expect(err).ToNot(HaveOccurred())

Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`State Integration ID Log Config
ACTIVE 9e740000-331b-4dec-89b0-4e59b81e9019 {"debug_print_plan":true,"log_connections":false,"log_disconnections":false,"log_duration":false,"log_error_verbosity":"DEFAULT","log_line_prefix":"\%m :\%r :\%u @ \%d :\[\%p\] : \%a :","log_min_duration_statement":60,"log_min_error_statement":"ERROR","log_statement":"ALL"}`))
Expect(server.ReceivedRequests()).Should(HaveLen(5))
session.Kill()
statusCode = 202
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodPut, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs/388c41b9-81f7-4ed9-8239-ab22f4aaca91"),
ghttp.VerifyJSON(`{
"export_config": {
"debug_print_plan": true,
"log_connections": false,
"log_disconnections": false,
"log_duration": false,
"log_error_verbosity": "DEFAULT",
"log_line_prefix": "%m :%r :%u @ %d :[%p] : %a :",
"log_min_duration_statement": 60,
"log_min_error_statement": "ERROR",
"log_statement": "ALL"
},
"exporter_id": "129f7c97-81ae-47c7-8f9e-40ab4390093f"
}`),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterResponse),
),
)

cmd := exec.Command(compiledCLIPath, "cluster", "db-query-logging", "update",
"--cluster-name", "stunning-sole",
// Change few query log configs
"--debug-print-plan", "true",
"--log-connections", "false",
"--log-min-duration-statement", "60",
"--log-statement", "ALL",
)
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)

Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`Request submitted to edit DB query log config for the cluster, this may take a few minutes...
State Integration Name
ACTIVE datadog-tp
Log Config Key Log Config Value
debug-print-plan true
log-min-duration-statement 60
log-connections false
log-disconnections false
log-duration false
log-error-verbosity DEFAULT
log-statement ALL
log-min-error-statement ERROR
log-line-prefix \%m :\%r :\%u @ \%d :\[\%p\] : \%a :`))
Expect(server.ReceivedRequests()).Should(HaveLen(6))
session.Kill()
})
})
Context("with new integration name", func() {
It("should update log config with given integration name", func() {
// Load existing log export config
err := loadJson("./test/fixtures/db-query-log-exporter-describe-resp.json", &pgLogExporterConfigListResponse)
Expect(err).ToNot(HaveOccurred())

statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterConfigListResponse),
),
)

err = loadJson("./test/fixtures/list-telemetry-provider.json", &responseIntegrationList)
Expect(err).ToNot(HaveOccurred())
// Update integration ID to a new value
responseIntegrationList.Data[0].Info.Id = "517c2d8c-d320-4e4c-97d8-e030f0047b0a"

statusCode = 200
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/telemetry-providers"),
ghttp.RespondWithJSONEncodedPtr(&statusCode, responseIntegrationList),
),
)

// Load updated log export config response
err = loadJson("./test/fixtures/db-query-log-exporter-update-config-resp.json", &pgLogExporterResponse)
Expect(err).ToNot(HaveOccurred())

statusCode = 202
server.AppendHandlers(
ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodPut, "/api/public/v1/accounts/340af43a-8a7c-4659-9258-4876fd6a207b/projects/78d4459c-0f45-47a5-899a-45ddf43eba6e/cluster/5f80730f-ba3f-4f7e-8c01-f8fa4c90dad8/db-query-log-exporter-configs/388c41b9-81f7-4ed9-8239-ab22f4aaca91"),
// notice: exporter id must be for same as new integration ID
ghttp.VerifyJSON(`{
"export_config": {
"debug_print_plan": true,
"log_connections": false,
"log_disconnections": false,
"log_duration": false,
"log_error_verbosity": "DEFAULT",
"log_line_prefix": "%m :%r :%u @ %d :[%p] : %a :",
"log_min_duration_statement": 60,
"log_min_error_statement": "ERROR",
"log_statement": "ALL"
},
"exporter_id": "517c2d8c-d320-4e4c-97d8-e030f0047b0a"
}`),
ghttp.RespondWithJSONEncodedPtr(&statusCode, pgLogExporterResponse),
),
)

cmd := exec.Command(compiledCLIPath, "cluster", "db-query-logging", "update",
"--cluster-name", "stunning-sole",
// pass new integration name
"--integration-name", "datadog-tp-new",
"--debug-print-plan", "true",
"--log-connections", "false",
"--log-min-duration-statement", "60",
"--log-statement", "ALL",
)
session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)

Expect(err).NotTo(HaveOccurred())
session.Wait(2)
Expect(session.Out).Should(gbytes.Say(`Request submitted to edit DB query log config for the cluster, this may take a few minutes...
State Integration Name
ACTIVE datadog-tp-new
Log Config Key Log Config Value
debug-print-plan true
log-min-duration-statement 60
log-connections false
log-disconnections false
log-duration false
log-error-verbosity DEFAULT
log-statement ALL
log-min-error-statement ERROR
log-line-prefix \%m :\%r :\%u @ \%d :\[\%p\] : \%a :`))
Expect(server.ReceivedRequests()).Should(HaveLen(6))
session.Kill()
})
})

})

AfterEach(func() {
Expand Down
Loading

0 comments on commit 2e83931

Please sign in to comment.