From e5ae4ea9e5fda849e04e1d9c7452d8e17e10c34b Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Tue, 12 Mar 2024 15:20:05 -0400 Subject: [PATCH 1/3] cost unit testing and fixes Signed-off-by: Alex Meijer --- datadog/cmd/main/main.go | 4 +-- datadog/cmd/main/main_test.go | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/datadog/cmd/main/main.go b/datadog/cmd/main/main.go index a1751c9..427b3a0 100644 --- a/datadog/cmd/main/main.go +++ b/datadog/cmd/main/main.go @@ -412,13 +412,13 @@ func scrapeDatadogPrices(url string) (*datadogplugin.PricingInformation, error) } res := datadogplugin.DatadogProJSON{} r := regexp.MustCompile(`var productDetailData = \s*(.*?)\s*;`) - log.Debugf("got response: %s", string(b)) + log.Tracef("got response: %s", string(b)) matches := r.FindAllStringSubmatch(string(b), -1) if len(matches) != 1 { return nil, fmt.Errorf("requires exactly 1 product detail data, got %d", len(matches)) } - log.Debugf("matches[0][1]:" + matches[0][1]) + log.Tracef("matches[0][1]:" + matches[0][1]) err = json.Unmarshal([]byte(matches[0][1]), &res) if err != nil { return nil, fmt.Errorf("failed to read pricing page body: %v", err) diff --git a/datadog/cmd/main/main_test.go b/datadog/cmd/main/main_test.go index d1e1d4e..3c3e647 100644 --- a/datadog/cmd/main/main_test.go +++ b/datadog/cmd/main/main_test.go @@ -2,7 +2,17 @@ package main import ( "fmt" + "os" "testing" + "time" + + datadogplugin "github.com/opencost/opencost-plugins/datadog/datadogplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/util/timeutil" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestPricingFetch(t *testing.T) { @@ -15,3 +25,56 @@ func TestPricingFetch(t *testing.T) { t.Fatalf("expected non zero pricing details") } } + +func TestGetCustomCosts(t *testing.T) { + // read necessary env vars. If any are missing, log warning and skip test + ddSite := os.Getenv("DD_SITE") + ddApiKey := os.Getenv("DD_API_KEY") + ddAppKey := os.Getenv("DD_APPLICATION_KEY") + + if ddSite == "" { + log.Warnf("DD_SITE undefined, this needs to have the URL of your DD instance, skipping test") + t.Skip() + return + } + + if ddApiKey == "" { + log.Warnf("DD_API_KEY undefined, skipping test") + t.Skip() + return + } + + if ddAppKey == "" { + log.Warnf("DD_APPLICATION_KEY undefined, skipping test") + t.Skip() + return + } + + // write out config to temp file using contents of env vars + config := datadogplugin.DatadogConfig{ + DDSite: ddSite, + DDAPIKey: ddApiKey, + DDAppKey: ddAppKey, + } + + rateLimiter := rate.NewLimiter(0.25, 5) + ddCostSrc := DatadogCostSource{ + rateLimiter: rateLimiter, + } + ddCostSrc.ddCtx, ddCostSrc.usageApi = getDatadogClients(config) + windowStart := time.Date(2024, 3, 11, 0, 0, 0, 0, time.UTC) + // query for qty 2 of 1 hour windows + windowEnd := time.Date(2024, 3, 12, 0, 0, 0, 0, time.UTC) + + req := &pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(timeutil.Day), + } + + resp := ddCostSrc.GetCustomCosts(req) + + if len(resp) == 0 { + t.Fatalf("empty response") + } +} From cc20b7905797448b7b994b4ff977a4dc431e4fb5 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Tue, 12 Mar 2024 17:50:14 -0400 Subject: [PATCH 2/3] addl debug fixes Signed-off-by: Alex Meijer --- datadog/cmd/main/main.go | 19 ++++++++++++++++++- datadog/cmd/main/main_test.go | 1 + datadog/datadogplugin/datadogconfig.go | 7 ++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/datadog/cmd/main/main.go b/datadog/cmd/main/main.go index 427b3a0..a2baa3e 100644 --- a/datadog/cmd/main/main.go +++ b/datadog/cmd/main/main.go @@ -90,7 +90,7 @@ func main() { if err != nil { log.Fatalf("error building DD config: %v", err) } - + log.SetLogLevel(ddConfig.DDLogLevel) // datadog usage APIs allow 10 requests every 30 seconds rateLimiter := rate.NewLimiter(0.25, 5) ddCostSrc := DatadogCostSource{ @@ -127,11 +127,13 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric ccResp := boilerplateDDCustomCost(window) nextPageId := "init" + nextPagePrevPeriodId := "init" for morepages := true; morepages; morepages = (nextPageId != "") { params := datadogV2.NewGetHourlyUsageOptionalParameters() if nextPageId != "init" { params.PageNextRecordId = &nextPageId } + if d.rateLimiter.Tokens() < 1.0 { log.Infof("datadog rate limit reached. holding request until rate capacity is back") } @@ -156,6 +158,9 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric // where needed params.FilterTimestampEnd = window.Start() toSub := window.End().Sub(*window.Start()) + if nextPagePrevPeriodId != "init" { + params.PageNextRecordId = &nextPagePrevPeriodId + } respPriorWindow, r, err := d.usageApi.GetHourlyUsage(d.ddCtx, (*window.Start()).Add(-toSub), "all", *params) if err != nil { log.Errorf("Error when calling `UsageMeteringApi.GetHourlyUsage`: %v\n", err) @@ -170,6 +175,8 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric if resp.Data[index].Attributes.Measurements[indexMeas].Value.IsSet() { var prior *datadogV2.HourlyUsageMeasurement if len(respPriorWindow.Data) > index { + log.Infof("getting prior window data from timeframe %v, and measurement %v", window, resp.Data[index].Attributes.Measurements[indexMeas]) + log.Infof("prior window data: %v", respPriorWindow.Data[index]) prior = &respPriorWindow.Data[index].Attributes.Measurements[indexMeas] } else { // then this is an out of bound access @@ -210,6 +217,12 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric } else { nextPageId = "" } + + if respPriorWindow.Meta != nil && respPriorWindow.Meta.Pagination != nil && respPriorWindow.Meta.Pagination.NextRecordId.IsSet() { + nextPagePrevPeriodId = *respPriorWindow.Meta.Pagination.NextRecordId.Get() + } else { + nextPagePrevPeriodId = "" + } } return &ccResp @@ -375,6 +388,10 @@ func getDatadogConfig(configFilePath string) (*datadogplugin.DatadogConfig, erro return nil, fmt.Errorf("error marshaling json into DD config %v", err) } + if result.DDLogLevel == "" { + result.DDLogLevel = "info" + } + return &result, nil } diff --git a/datadog/cmd/main/main_test.go b/datadog/cmd/main/main_test.go index 3c3e647..b6ec989 100644 --- a/datadog/cmd/main/main_test.go +++ b/datadog/cmd/main/main_test.go @@ -72,6 +72,7 @@ func TestGetCustomCosts(t *testing.T) { Resolution: durationpb.New(timeutil.Day), } + log.SetLogLevel("debug") resp := ddCostSrc.GetCustomCosts(req) if len(resp) == 0 { diff --git a/datadog/datadogplugin/datadogconfig.go b/datadog/datadogplugin/datadogconfig.go index c567ee6..eed3e7f 100644 --- a/datadog/datadogplugin/datadogconfig.go +++ b/datadog/datadogplugin/datadogconfig.go @@ -1,7 +1,8 @@ package datadog type DatadogConfig struct { - DDSite string `json:"datadog_site"` - DDAPIKey string `json:"datadog_api_key"` - DDAppKey string `json:"datadog_app_key"` + DDSite string `json:"datadog_site"` + DDAPIKey string `json:"datadog_api_key"` + DDAppKey string `json:"datadog_app_key"` + DDLogLevel string `json:"log_level"` } From 9b2c5dc5a0a33fe762e828f40b21b644d7b47e41 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Tue, 12 Mar 2024 20:17:29 -0400 Subject: [PATCH 3/3] remove previous window code Signed-off-by: Alex Meijer --- datadog/cmd/main/main.go | 58 ++-------------------------------------- 1 file changed, 2 insertions(+), 56 deletions(-) diff --git a/datadog/cmd/main/main.go b/datadog/cmd/main/main.go index a2baa3e..bc3186b 100644 --- a/datadog/cmd/main/main.go +++ b/datadog/cmd/main/main.go @@ -127,7 +127,6 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric ccResp := boilerplateDDCustomCost(window) nextPageId := "init" - nextPagePrevPeriodId := "init" for morepages := true; morepages; morepages = (nextPageId != "") { params := datadogV2.NewGetHourlyUsageOptionalParameters() if nextPageId != "init" { @@ -138,7 +137,7 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric log.Infof("datadog rate limit reached. holding request until rate capacity is back") } - err := d.rateLimiter.WaitN(context.TODO(), 2) + err := d.rateLimiter.WaitN(context.TODO(), 1) if err != nil { log.Errorf("error waiting on rate limiter`: %v\n", err) ccResp.Errors = append(ccResp.Errors, err.Error()) @@ -153,37 +152,12 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric ccResp.Errors = append(ccResp.Errors, err.Error()) } - // many datadog usages are given in terms of a cumulative month to date usage - // therefore, make a call for the hour before this hour to get a comparison - // where needed - params.FilterTimestampEnd = window.Start() - toSub := window.End().Sub(*window.Start()) - if nextPagePrevPeriodId != "init" { - params.PageNextRecordId = &nextPagePrevPeriodId - } - respPriorWindow, r, err := d.usageApi.GetHourlyUsage(d.ddCtx, (*window.Start()).Add(-toSub), "all", *params) - if err != nil { - log.Errorf("Error when calling `UsageMeteringApi.GetHourlyUsage`: %v\n", err) - log.Errorf("Full HTTP response: %v\n", r) - ccResp.Errors = append(ccResp.Errors, err.Error()) - } - for index := range resp.Data { for indexMeas := range resp.Data[index].Attributes.Measurements { usageQty := float32(0.0) if resp.Data[index].Attributes.Measurements[indexMeas].Value.IsSet() { - var prior *datadogV2.HourlyUsageMeasurement - if len(respPriorWindow.Data) > index { - log.Infof("getting prior window data from timeframe %v, and measurement %v", window, resp.Data[index].Attributes.Measurements[indexMeas]) - log.Infof("prior window data: %v", respPriorWindow.Data[index]) - prior = &respPriorWindow.Data[index].Attributes.Measurements[indexMeas] - } else { - // then this is an out of bound access - log.Warnf("could not get prior window data from timeframe %v, and measurement %v", window, resp.Data[index].Attributes.Measurements[indexMeas]) - log.Warnf("passing in nil prior window data") - } - usageQty = GetUsageQuantity(*resp.Data[index].Attributes.ProductFamily, &resp.Data[index].Attributes.Measurements[indexMeas], prior) + usageQty = float32(resp.Data[index].Attributes.Measurements[indexMeas].GetValue()) } if usageQty == 0.0 { @@ -217,39 +191,11 @@ func (d *DatadogCostSource) getDDCostsForWindow(window opencost.Window, listPric } else { nextPageId = "" } - - if respPriorWindow.Meta != nil && respPriorWindow.Meta.Pagination != nil && respPriorWindow.Meta.Pagination.NextRecordId.IsSet() { - nextPagePrevPeriodId = *respPriorWindow.Meta.Pagination.NextRecordId.Get() - } else { - nextPagePrevPeriodId = "" - } } return &ccResp } -// we have two basic types usages: cumulative and rate -// rate usages are e.g., number of infra hosts, that have fixed costs per hour -// cumulative usages are e.g., number of logs ingested, that have a fixed cost per unit -// if a usage is cumulative, then suptract the usage in the hour prior to get the incremental usage -// if a usage is a rate, then just return the usage -func GetUsageQuantity(productFamily string, currentPeriodUsage, previousPeriodUsage *datadogV2.HourlyUsageMeasurement) float32 { - curUsage := currentPeriodUsage.GetValue() - if _, found := rateFamilies[productFamily]; found { - // this family is a rate family, so just return the usage - return float32(curUsage) - } - - prevUsage := int64(0) - if previousPeriodUsage == nil { - log.Warnf("previous period usage was nil, assuming 0 usage for that timeframe for family %s", productFamily) - } else { - prevUsage = previousPeriodUsage.GetValue() - } - - return float32(curUsage - prevUsage) -} - // the public pricing used in the pricing list doesn't always match the usage reports // therefore, we maintain a list of aliases var usageToPricingMap = map[string]string{