Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SYNTH-16456] Postpone reporting results on 404 #1480

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[SYNTH-16456] Postpone reporting results on 404 #1480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 143 additions & 1 deletion src/commands/synthetics/__tests__/utils/public.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,7 @@ describe('utils', () => {

describe('waitForResults', () => {
beforeEach(() => {
jest.useFakeTimers()
jest.useFakeTimers({now: 123})
jest.spyOn(utils, 'wait').mockImplementation(async () => jest.advanceTimersByTime(5000))
})

Expand Down Expand Up @@ -1336,6 +1336,148 @@ describe('utils', () => {
expect(mockReporter.testsWait).toHaveBeenCalledTimes(3)
})

test('should wait for incomplete results caused by 404', async () => {
jest.spyOn(utils, 'wait').mockImplementation(async () => waiter.resolve())

const tests = [result.test, {...result.test, public_id: 'other-public-id'}]

// === STEP 1 === (batch 'in_progress')
waiter.start()
mockApi({
getBatchImplementation: async () => ({
status: 'in_progress',
results: [
// First test
{...getInProgressResultInBatch()},
// Second test
{...getInProgressResultInBatch(), test_public_id: 'other-public-id', result_id: 'rid-2'},
],
}),
pollResultsImplementation: async () => [],
})

const resultsPromise = utils.waitForResults(
api,
trigger,
tests,
{
batchTimeout: 120000,
datadogSite: DEFAULT_COMMAND_CONFIG.datadogSite,
failOnCriticalErrors: false,
subdomain: DEFAULT_COMMAND_CONFIG.subdomain,
},
mockReporter
)

// Wait for the 2 tests (initial)
expect(mockReporter.testsWait).toHaveBeenNthCalledWith(1, [tests[0], tests[1]], MOCK_BASE_URL, trigger.batch_id)

await waiter.promise

// Still waiting for 2 tests
expect(mockReporter.testsWait).toHaveBeenNthCalledWith(
2,
[tests[0], tests[1]],
MOCK_BASE_URL,
trigger.batch_id,
0
)

// === STEP 2 === (batch 'in_progress')
waiter.start()
mockApi({
getBatchImplementation: async () => ({
status: 'in_progress',
results: [
// First test
{...getPassedResultInBatch()},
// Second test
{...getInProgressResultInBatch(), test_public_id: 'other-public-id', result_id: 'rid-2'},
],
}),
pollResultsImplementation: async () => {
throw getAxiosError(404, {message: 'Test results not found'})
},
})

await waiter.promise

// One result received
expect(mockReporter.resultReceived).toHaveBeenNthCalledWith(1, {
...batch.results[0],
status: 'passed',
})
// But not available
expect(mockReporter.resultEnd).not.toHaveBeenCalled()
// Now waiting for 1 test
expect(mockReporter.testsWait).toHaveBeenNthCalledWith(3, [tests[1]], MOCK_BASE_URL, trigger.batch_id, 0)

// === STEP 3 === (batch 'in_progress')
waiter.start()
mockApi({
getBatchImplementation: async () => ({
status: 'in_progress',
results: [
// First test
{...getPassedResultInBatch()},
// Second test
{...getInProgressResultInBatch(), test_public_id: 'other-public-id', result_id: 'rid-2'},
],
}),
pollResultsImplementation: async () => [
deepExtend({}, pollResult), // became available
],
})

await waiter.promise

// Result 1 just became available, so it should be reported
expect(mockReporter.resultEnd).toHaveBeenNthCalledWith(1, result, MOCK_BASE_URL, 'bid')
// Still waiting for 1 test
expect(mockReporter.testsWait).toHaveBeenNthCalledWith(3, [tests[1]], MOCK_BASE_URL, trigger.batch_id, 0)

mockApi({
getBatchImplementation: async () => ({
status: 'passed',
results: [
// First test
{...getPassedResultInBatch()},
// Second test
{...getPassedResultInBatch(), test_public_id: 'other-public-id', result_id: 'rid-2'},
],
}),
pollResultsImplementation: async () => {
throw getAxiosError(404, {message: 'Test results not found'})
},
})

expect(await resultsPromise).toEqual([
result,
{...result, resultId: 'rid-2', result: {}, timestamp: 123, test: tests[1]},
])

// One result received
expect(mockReporter.resultReceived).toHaveBeenNthCalledWith(2, {
...batch.results[0],
status: 'passed',
test_public_id: 'other-public-id',
result_id: 'rid-2',
})
// Last result is reported without a poll result
expect(mockReporter.resultEnd).toHaveBeenNthCalledWith(
2,
{...result, resultId: 'rid-2', result: {}, test: tests[1], timestamp: 123},
MOCK_BASE_URL,
'bid'
)
expect(mockReporter.error).toHaveBeenCalledWith(
'The information for result rid-2 of test other-public-id was incomplete at the end of the batch.\n\n'
)

// Do not report when there are no tests to wait anymore
expect(mockReporter.testsWait).toHaveBeenCalledTimes(4)
})

test('object in each result should be different even if they share the same public ID (config overrides)', async () => {
mockApi({
getBatchImplementation: async () => ({
Expand Down
2 changes: 1 addition & 1 deletion src/commands/synthetics/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ export const determineRetryDelay = (

const isEndpointError = (error: Error): error is EndpointError => error instanceof EndpointError

const getErrorHttpStatus = (error: Error): number | undefined =>
export const getErrorHttpStatus = (error: Error): number | undefined =>
isEndpointError(error) ? error.status : isAxiosError(error) ? error.response?.status : undefined

export const isForbiddenError = (error: Error): boolean => getErrorHttpStatus(error) === 403
Expand Down
74 changes: 62 additions & 12 deletions src/commands/synthetics/batch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
import deepExtend from 'deep-extend'

import {APIHelper, EndpointError, formatBackendErrors} from './api'
import {APIHelper, EndpointError, formatBackendErrors, getErrorHttpStatus} from './api'
import {BatchTimeoutRunawayError} from './errors'
import {
BaseResultInBatch,
Batch,
MainReporter,
PollResultMap,
PollResult,
Result,
ResultDisplayInfo,
ResultInBatch,
ServerResult,
Test,
} from './interfaces'
import {
Expand All @@ -32,6 +33,8 @@ export const waitForBatchToFinish = async (
): Promise<Result[]> => {
const safeDeadline = Date.now() + batchTimeout + 3 * POLLING_INTERVAL
const emittedResultIds = new Set<string>()
const backupPollResultMap = new Map<string, PollResult>()

let oldIncompleteResultIds = new Set<string>()

while (true) {
Expand All @@ -51,7 +54,7 @@ export const waitForBatchToFinish = async (
oldIncompleteResultIds
)

const {pollResultMap, incompleteResultIds} = await getPollResultMap(api, resultIdsToFetch)
const {pollResultMap, incompleteResultIds} = await getPollResultMap(api, resultIdsToFetch, backupPollResultMap)

const resultsToReport = getResultsToReport(
shouldContinuePolling,
Expand Down Expand Up @@ -143,7 +146,7 @@ export const reportReceivedResults = (batch: Batch, emittedResultIds: Set<string
const receivedResults: ResultInBatch[] = []

for (const [index, result] of batch.results.entries()) {
// Skipped results aren't reported in detail in the terminal output, but they are still reported by `resultReceived()`.
// Skipped results are only reported by `resultReceived()`, then they are excluded everywhere with `excludeSkipped()`.
const resultId = result.status === 'skipped' ? `skipped-${index}` : result.result_id

// The result is reported if it has a final status, or if it's a non-final result.
Expand All @@ -160,7 +163,7 @@ export const reportReceivedResults = (batch: Batch, emittedResultIds: Set<string
const reportResults = (
batchId: string,
results: ResultInBatch[],
pollResultMap: PollResultMap,
pollResultMap: Map<string, PollResult>,
resultDisplayInfo: ResultDisplayInfo,
safeDeadlineReached: boolean,
reporter: MainReporter
Expand Down Expand Up @@ -214,7 +217,7 @@ const reportWaitingTests = (

const getResultFromBatch = (
resultInBatch: ResultInBatch,
pollResultMap: PollResultMap,
pollResultMap: Map<string, PollResult>,
resultDisplayInfo: ResultDisplayInfo,
safeDeadlineReached = false
): Result => {
Expand All @@ -236,7 +239,10 @@ const getResultFromBatch = (
}
}

const pollResult = pollResultMap[resultInBatch.result_id]
const pollResult = pollResultMap.get(resultInBatch.result_id)
if (!pollResult) {
return getResultWithoutPollResult(resultInBatch, test, hasTimedOut, resultDisplayInfo)
}

if (safeDeadlineReached) {
pollResult.result.failure = new BatchTimeoutRunawayError().toJson()
Expand Down Expand Up @@ -268,6 +274,31 @@ const getResultFromBatch = (
}
}

const getResultWithoutPollResult = (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given the similarity between this function and the return statement from the above, could we merge the two, and add only the difference if pollResult is not available?

const resultFromBatch = {
    executionRule: resultInBatch.execution_rule,
    initialResultId: resultInBatch.initial_result_id,
    isNonFinal: isNonFinalResult(resultInBatch),
    location: getLocation(resultInBatch.location, test),
    passed: hasResultPassed(resultInBatch, isUnhealthy, hasTimedOut, options),
    resultId: getResultIdOrLinkedResultId(resultInBatch),
    retries: resultInBatch.retries || 0,
    maxRetries: resultInBatch.max_retries || 0,
    selectiveRerun: resultInBatch.selective_rerun,
    timedOut: hasTimedOut,
}

if (pollResult) {
  return {
    ...resultFromBatch,
    result: {
      ...pollResult.result,
      ...(safeDeadlineReached ? {
        failure: new BatchTimeoutRunawayError().toJson()
        passed: false
        } : timedOutRetry || hasTimedOut ? {
          failure: new {code: 'TIMEOUT', message: 'The batch timed out before receiving the result.'}
          passed: false
        } : {})
    },
    test: deepExtend({}, test, pollResult),
    timestamp: pollResult.timestamp
  }
} else {
  return {
    ...resultFromBatch,
    test: deepExtend({}, test),
    timestamp: Date.now()
  }
}

This code might not be the best solution, but will hopefully give you an idea.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This way would make it more concise, but i'm not sure if having the two functions is not actually clearer and easier to understand 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer the 2 functions as well tbh 😁

resultInBatch: BaseResultInBatch,
test: Test,
hasTimedOut: boolean,
resultDisplayInfo: ResultDisplayInfo
): Result => {
const {getLocation, options} = resultDisplayInfo

return {
executionRule: resultInBatch.execution_rule,
initialResultId: resultInBatch.initial_result_id,
isNonFinal: isNonFinalResult(resultInBatch),
location: getLocation(resultInBatch.location, test),
passed: hasResultPassed(resultInBatch, false, hasTimedOut, options),
result: {} as ServerResult,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't need this, could we rather make it optional?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm afraid of how it could break customer's code. I'll check internally (web-ui and CI integrations)

resultId: getResultIdOrLinkedResultId(resultInBatch),
retries: resultInBatch.retries || 0,
maxRetries: resultInBatch.max_retries || 0,
selectiveRerun: resultInBatch.selective_rerun,
test: deepExtend({}, test),
timedOut: hasTimedOut,
timestamp: Date.now(),
}
}

const getBatch = async (api: APIHelper, batchId: string): Promise<Batch> => {
try {
const batch = await api.getBatch(batchId)
Expand All @@ -278,23 +309,42 @@ const getBatch = async (api: APIHelper, batchId: string): Promise<Batch> => {
}
}

const getPollResultMap = async (api: APIHelper, resultIds: string[]) => {
/**
* Returns fresh poll results, or reads the backup map in case of 404.
*/
const getPollResultMap = async (api: APIHelper, resultIds: string[], backupPollResultMap: Map<string, PollResult>) => {
const pollResultMap = new Map<string, PollResult>()
const incompleteResultIds = new Set<string>()

try {
const pollResults = await api.pollResults(resultIds)

const pollResultMap: PollResultMap = {}
const incompleteResultIds = new Set<string>()

pollResults.forEach((r) => {
// When they are initialized in the backend, results only contain an `eventType: created` property.
if ('eventType' in r.result && r.result.eventType === 'created') {
incompleteResultIds.add(r.resultID)
}
pollResultMap[r.resultID] = r
pollResultMap.set(r.resultID, r)
backupPollResultMap.set(r.resultID, r)
})

return {pollResultMap, incompleteResultIds}
} catch (e) {
if (getErrorHttpStatus(e) === 404) {
// If some results have latency and retries were not enough, the whole request fails with "Test results not found".
// In that case, we mark results IDs that were never polled before as incomplete so they are fetched in the next polling cycles.
resultIds.forEach((resultId) => {
const backupPollResult = backupPollResultMap.get(resultId)
if (backupPollResult) {
pollResultMap.set(resultId, backupPollResult)
} else {
incompleteResultIds.add(resultId)
}
})

return {pollResultMap, incompleteResultIds}
}

throw new EndpointError(`Failed to poll results: ${formatBackendErrors(e)}\n`, e.response?.status)
}
}
Expand Down
2 changes: 0 additions & 2 deletions src/commands/synthetics/interfaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ export interface PollResult {
timestamp: number
}

export type PollResultMap = {[resultId: string]: PollResult}

/**
* Information required to convert a `PollResult` to a `Result`.
*/
Expand Down
Loading