Skip to content

Commit

Permalink
APIGOV-29054 - fixes for monitoring and query API calls (#281)
Browse files Browse the repository at this point in the history 
* APIGOV-29054 - fix to use API ID instead of asset id in monitoring archive call

* APIGOV-29054 - fix for time conversion while parsing  Monitoring acrchive

* APIGOV-29054 - fixes to use the current time for lookup during the initial execution of the agent

* APIGOV-29054 - Fixes

* APIGOV-29054 - fixes for timestamp when using query API for fetching metrics

* APIGOV-29054 - additional log entries
  • Loading branch information
@vivekschauhan
vivekschauhan authored Nov 1, 2024
1 parent af80efc commit efe6870
Show file tree
Hide file tree
Showing 16 changed files with 56 additions and 45 deletions.
2 changes: 1 addition & 1 deletion README_discovery.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ Along with all [common agent variables](https://docs.axway.com/bundle/amplify-ce
| MULESOFT_AUTH_USERNAME | mulesoft.auth.username | The Mulesoft Anypoint username created for this agent | |
| MULESOFT_AUTH_CLIENTID | mulesoft.auth.clientID | The client id of a defined connected app with all of the necessary permssions | |
| MULESOFT_AUTH_CLIENTSECRET | mulesoft.auth.clientSecret | The client secret of a defined connected app with all of the necessary permssions | |
| MULESOFT_CACHEPATH | mulesoft.cachePath | Path entry to store stateful cache between agent invocations | _/tmp_ |
| MULESOFT_CACHEPATH | mulesoft.cachePath | Path entry to store stateful cache between agent invocations | _/data_ |
| MULESOFT_DISCOVERYIGNORETAGS | mulesoft.discoveryIgnoreTags | Comma-separated black list of tags that, if any are present, will prevent an API being publised to Amplify Central. Take precedence over MULESOFT_DISCOVERYTAGS | (empty tag list) |
| MULESOFT_DISCOVERYTAGS | mulesoft.discoveryTags | Comma-separated list of tags that, if any are present, will allow an API to be publised to Amplify Central. All APIs are discovered if not tags are specified | (empty tag list) |
| MULESOFT_DISCOVERORIGINALRAML | mulesoft.discoverOriginalRAML | Set to true if the agent should discover the Assets that were created in RAML as RAML | _false_ |
Expand Down
4 changes: 2 additions & 2 deletions README_traceability.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Install the agent and provision Amplify Central access as described in the [READ
The MuleSoft AnyPoint Traceability Agent is built and distributed as a docker image. To execute the agent using the image, the following command may be executed. Update the version in the examples below.

```shell
docker run --env-file env_vars -v `pwd`/keys:/keys ghcr.io/axway/mulesoft_traceability_agent:v1.2.2
docker run --env-file env_vars -v `pwd`/keys:/keys -v `pwd`/data:/data ghcr.io/axway/mulesoft_traceability_agent:v1.2.6
```

## Configuration Variables
Expand All @@ -24,7 +24,7 @@ Along with all [common agent variables](https://docs.axway.com/bundle/amplify-ce
| MULESOFT_AUTH_USERNAME | mulesoft.auth.username | The MuleSoft Anypoint username created for this agent | |
| MULESOFT_AUTH_CLIENTID | mulesoft.auth.clientID | The client id of a defined connected app with all of the necessary permssions | |
| MULESOFT_AUTH_CLIENTSECRET | mulesoft.auth.clientSecret | The client secret of a defined connected app with all of the necessary permssions | |
| MULESOFT_CACHEPATH | mulesoft.cachePath | Path entry to store stateful cache between agent invocations | _/tmp_ |
| MULESOFT_CACHEPATH | mulesoft.cachePath | Path entry to store stateful cache between agent invocations | _/data_ |
| MULESOFT_ENVIRONMENT | mulesoft.environment | The MuleSoft Anypoint Exchange the agent connects to, e.g. Sandbox. | |
| MULESOFT_ORGNAME | mulesoft.orgName | The MuleSoft Anypoint Business Unit the agent connects to | |
| MULESOFT_POLLINTERVAL | mulesoft.pollInterval | The frequency in which MuleSoft API Manager is polled for new endpoints. | _60s_ |
Expand Down
2 changes: 1 addition & 1 deletion build/discovery.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@ COPY --from=builder $APP_HOME/bin/discovery /discovery
COPY --from=builder /etc/passwd /etc/passwd

USER $APP_USER
VOLUME ["/tmp"]
VOLUME ["/keys", "/data"]
HEALTHCHECK --retries=1 CMD /discovery --status || exit 1
ENTRYPOINT ["/discovery"]
2 changes: 1 addition & 1 deletion build/mulesoft_traceability_agent.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ mulesoft_traceability_agent:
mulesoft:
environment: "${MULESOFT_ENVIRONMENT}"
orgName: "${MULESOFT_ORGNAME}"
cachePath: "${MULESOFT_CACHEPATH:/tmp}"
cachePath: "${MULESOFT_CACHEPATH:/data}"
pollInterval: ${MULESOFT_POLLINTERVAL:20s}
useMonitoringAPI: "${MULESOFT_USEMONITORINGAPI}"
auth:
Expand Down
2 changes: 1 addition & 1 deletion build/traceability.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@ COPY --from=builder /etc/passwd /etc/passwd
COPY --from=builder /etc/group /etc/group

USER $APP_USER
VOLUME ["/tmp", "/logs"]
VOLUME ["/keys", "/data", "/logs"]
HEALTHCHECK --retries=1 CMD /traceability --status || exit 1
ENTRYPOINT ["/traceability"]
6 changes: 3 additions & 3 deletions deployment/discovery-deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,18 @@ spec:
name: muleauth
key: username
- name: MULESOFT_CACHEPATH
value: /tmp
value: /data
- name: MULESOFT_ENVIRONMENT
value:
- name: MULESOFT_ORGNAME
value:
- name: MULESOFT_POLLINTERVAL
value: 60s
- name: MULESOFT_DISCOVERORIGINALRAML
value: false
value: "false"
- name: STATUS_PORT
value: "8989"
image: <Path_To_Image>
image:
imagePullPolicy: Always
name: discovery
resources: {}
Expand Down
21 changes: 11 additions & 10 deletions deployment/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,24 @@ services:
# - "8089:8089"
volumes:
- /path/to/keys:/keys
- /path/to/data:/data
environment:
CENTRAL_ORGANIZATIONID: "xxxxxxxx"
CENTRAL_AUTH_PUBLICKEY: /keys/dosa-public.pem
CENTRAL_AUTH_PRIVATEKEY: /keys/dosa-key.pem
CENTRAL_AUTH_KEYPASSWORD:
CENTRAL_ENVIRONMENT:
CENTRAL_AUTH_CLIENTID: <DOSA>
CENTRAL_AUTH_CLIENTID:
LOG_LEVEL: debug
LOG_FORMAT: json
LOG_OUTPUT: stdout
LOG_PATH: logs
MULESOFT_ANYPOINTEXCHANGEURL: https://anypoint.mulesoft.com
MULESOFT_ENVIRONMENT: Sandbox
MULESOFT_ORGID: <ORG_ID>
MULESOFT_ORGID:
MULESOFT_POLLINTERVAL: 60s
MULESOFT_AUTH_USERNAME: <USERNAME>
MULESOFT_AUTH_PASSWORD: <PASSWORD>
MULESOFT_AUTH_CLIENTID:
MULESOFT_AUTH_CLIENTSECRET:
MULESOFT_DISCOVERORIGINALRAML: false
traceability:
image: mulesoft-traceability
Expand All @@ -33,23 +34,23 @@ services:
- /path/to/data:/data
environment:
STATUS_PORT: 8989
CENTRAL_ORGANIZATIONID: "xxxxxxx"
CENTRAL_ORGANIZATIONID:
CENTRAL_AUTH_PUBLICKEY: /keys/dosa-public.pem
CENTRAL_AUTH_PRIVATEKEY: /keys/dosa-key.pem
CENTRAL_AUTH_KEYPASSWORD:
CENTRAL_ENVIRONMENT:
CENTRAL_AUTH_CLIENTID: <DOSA>
CENTRAL_AUTH_CLIENTID:
LOG_LEVEL: debug
LOG_FORMAT: json
LOG_OUTPUT: stdout
LOG_PATH: logs
MULESOFT_ANYPOINTEXCHANGEURL: https://anypoint.mulesoft.com
MULESOFT_ENVIRONMENT:
MULESOFT_ORGNAME: <ORG_NAME>
MULESOFT_ORGNAME:
MULESOFT_POLLINTERVAL: 60s
MULESOFT_AUTH_USERNAME: <USER>
MULESOFT_AUTH_PASSWORD: <PASSWORD>
MULESOFT_CACHEPATH: "/tmp"
MULESOFT_AUTH_CLIENTID:
MULESOFT_AUTH_CLIENTSECRET:
MULESOFT_CACHEPATH: "/data"
TRACEABILITY_HOST: "ingestion-lumberjack.datasearch.axway.com:453"
TRACEABILITY_PROTOCOL: tcp
TRACEABILITY_COMPRESSIONLEVEL: 3
Expand Down
4 changes: 2 additions & 2 deletions deployment/traceability-deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ spec:
name: muleauth
key: username
- name: MULESOFT_CACHEPATH
value: /tmp
value: /data
- name: MULESOFT_ENVIRONMENT
value:
- name: MULESOFT_ORGNAME
Expand All @@ -79,7 +79,7 @@ spec:
value: tcp
- name: TRACEABILITY_WORKER
value: "1"
image: <Path_To_Image>
image:
imagePullPolicy: Always
name: traceability
resources: {}
Expand Down
10 changes: 5 additions & 5 deletions pkg/anypoint/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (

const (
HealthCheckEndpoint = "mulesoft"
monitoringURITemplate = "%s/monitoring/archive/api/v1/organizations/%s/environments/%s/apis/%s/summary/%d/%02d/%02d"
monitoringURITemplate = "%s/monitoring/archive/api/v1/organizations/%s/environments/%s/apis/%s/summary/%4d/%02d/%02d"
metricSummaryURITemplate = "%s/monitoring/archive/api/v1/organizations/%s/environments/%s/apis/%s/summary/%d/%02d/%02d/%s"
queryTemplate = `SELECT sum("request_size.count") as request_count, max("response_time.max") as response_max, min("response_time.min") as response_min
FROM "rp_general"."api_summary_metric"
Expand Down Expand Up @@ -419,12 +419,12 @@ func (c *AnypointClient) GetMonitoringBootstrap() (*MonitoringBootInfo, error) {
}

// GetMonitoringMetrics returns monitoring data from InfluxDb
func (c *AnypointClient) GetMonitoringMetrics(dataSourceName string, dataSourceID int, apiID, apiVersionID string, startDate, endTime time.Time) ([]APIMonitoringMetric, error) {
func (c *AnypointClient) GetMonitoringMetrics(dataSourceName string, dataSourceID int, apiID, apiVersionID string, startTime, endTime time.Time) ([]APIMonitoringMetric, error) {
headers := map[string]string{
"Authorization": c.getAuthString(c.auth.GetToken()),
}

query := fmt.Sprintf(queryTemplate, apiID, apiVersionID, startDate.UnixMilli(), endTime.UnixMilli())
query := fmt.Sprintf(queryTemplate, apiID, apiVersionID, startTime.UnixMilli(), endTime.UnixMilli())
url := fmt.Sprintf("%s/monitoring/api/visualizer/api/datasources/proxy/%d/query", c.baseURL, dataSourceID)
request := coreapi.Request{
Method: coreapi.GET,
Expand All @@ -446,7 +446,7 @@ func (c *AnypointClient) GetMonitoringMetrics(dataSourceName string, dataSourceI
for _, mr := range metricResponse.Results {
for _, ms := range mr.Series {
m := APIMonitoringMetric{
Time: ms.Time,
Time: endTime,
Events: []APISummaryMetricEvent{
{
ClientID: ms.Tags.ClientID,
Expand Down Expand Up @@ -537,7 +537,7 @@ func (c *AnypointClient) parseMetricSummaries(metricDataStream []byte) ([]APIMon
}
break
}
metricTime := time.Unix(metricData.Time, 0)
metricTime := time.UnixMilli(metricData.Time)
metric := APIMonitoringMetric{
Time: metricTime,
Events: metricData.Events,
Expand Down
2 changes: 1 addition & 1 deletion pkg/anypoint/monitoring.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (ms *MetricSeries) UnmarshalJSON(data []byte) error {
}

tm := ms.getValue("time")
ms.Time = time.Unix(tm, 0)
ms.Time = time.UnixMilli(tm)
ms.Count = ms.getValue("request_count")
ms.ResponseMax = ms.getValue("response_max")
ms.ResponseMin = ms.getValue("response_min")
Expand Down
2 changes: 1 addition & 1 deletion pkg/anypoint/testdata/query-response.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
],
"values": [
[
1729589392,
1729589392000,
4,
3243,
74
Expand Down
1 change: 1 addition & 0 deletions pkg/cmd/discovery/root_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ func Test_initConfig(t *testing.T) {
cfg.MulesoftConfig.Environment = "env1"
cfg.MulesoftConfig.PollInterval = 5 * time.Second
cfg.MulesoftConfig.OrgName = "dev"
cfg.MulesoftConfig.CachePath = "/tmp"
err = cfg.MulesoftConfig.ValidateCfg()
assert.Nil(t, err)
}
2 changes: 1 addition & 1 deletion pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func AddConfigProperties(rootProps props) {
rootProps.AddDurationProperty(pathAuthLifetime, 60*time.Minute, "Mulesoft session lifetime.")
rootProps.AddStringProperty(pathDiscoveryTags, "", "APIs containing any of these tags are selected for discovery.")
rootProps.AddStringProperty(pathDiscoveryIgnoreTags, "", "APIs containing any of these tags are ignored. Takes precedence over "+pathDiscoveryIgnoreTags+".")
rootProps.AddStringProperty(pathCachePath, "/tmp", "Mulesoft Cache Path")
rootProps.AddStringProperty(pathCachePath, "/data", "Mulesoft Cache Path")
rootProps.AddDurationProperty(pathPollInterval, time.Minute, "The interval at which Mulesoft is checked for updates.", properties.WithLowerLimit(30*time.Second))
rootProps.AddStringProperty(pathProxyURL, "", "Proxy URL")

Expand Down
2 changes: 1 addition & 1 deletion pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func TestKongProperties(t *testing.T) {
assert.Equal(t, corecfg.TLSVersionAsValue("0"), cfg.TLS.GetMaxVersion())
assert.Equal(t, time.Minute, cfg.PollInterval)
assert.Equal(t, "", cfg.ProxyURL)
assert.Equal(t, "/tmp", cfg.CachePath)
assert.Equal(t, "/data", cfg.CachePath)
assert.Equal(t, false, cfg.DiscoverOriginalRaml)

// validate changed values
Expand Down
5 changes: 4 additions & 1 deletion pkg/discovery/servicehandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,10 @@ func (s *serviceHandler) getServiceDetail(asset *anypoint.Asset, api *anypoint.A
}

parser := apic.NewSpecResourceParser(rawSpec, "")
parser.Parse()
err = parser.Parse()
if err != nil {
return nil, err
}

modifiedSpec, err := updateSpec(parser.GetSpecProcessor(), api.EndpointURI, configuration)
if err != nil {
Expand Down
Loading

0 comments on commit efe6870

Please sign in to comment.