diff --git a/.changeset/silly-starfishes-breathe.md b/.changeset/silly-starfishes-breathe.md new file mode 100644 index 000000000..37680dbb5 --- /dev/null +++ b/.changeset/silly-starfishes-breathe.md @@ -0,0 +1,5 @@ +--- +"@monokle/synchronizer": minor +--- + +introduced ProjectSynchronzier to allow fetching all project data at once diff --git a/packages/synchronizer/src/__tests__/projectSynchronizer.spec.ts b/packages/synchronizer/src/__tests__/projectSynchronizer.spec.ts new file mode 100644 index 000000000..0a514bbf9 --- /dev/null +++ b/packages/synchronizer/src/__tests__/projectSynchronizer.spec.ts @@ -0,0 +1,445 @@ +import {dirname, resolve} from 'path'; +import {fileURLToPath} from 'url'; +import {rm, mkdir, cp} from 'fs/promises'; +import sinon, { SinonStub } from 'sinon'; +import {assert} from 'chai'; +import {createMonokleProjectSynchronizerFromConfig} from '../createMonokleProjectSynchronizer.js'; +import {StorageHandlerPolicy} from '../handlers/storageHandlerPolicy.js'; +import {StorageHandlerJsonCache} from '../handlers/storageHandlerJsonCache.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +describe('ProjectSynchronizer Tests', () => { + const stubs: sinon.SinonStub[] = []; + + before(async () => { + await cleanupTmpConfigDir(); + }); + + afterEach(async () => { + if (stubs.length) { + stubs.forEach(stub => stub.restore()); + } + + await cleanupTmpConfigDir(); + }); + + it('returns empty data if not synchronized', async () => { + const storagePath = await createTmpConfigDir(); + const synchronizer = createSynchronizer(storagePath, stubs); + + const info = synchronizer.getProjectInfo(storagePath); + assert.isUndefined(info); + + const permissions = synchronizer.getProjectPermissions(storagePath); + assert.isUndefined(permissions); + + const policy = synchronizer.getProjectPolicy(storagePath); + assert.isObject(policy); + assert.isFalse(policy.valid); + assert.isEmpty(policy.path); + assert.isEmpty(policy.policy); + + const suppressions = synchronizer.getRepositorySuppressions(storagePath); + assert.isArray(suppressions); + assert.isEmpty(suppressions); + }); + + it('throws error when no access token provided for fetch', async () => { + try { + const storagePath = await createTmpConfigDir(); + const synchronizer = createSynchronizer(storagePath, stubs); + + await synchronizer.synchronize({} as any, storagePath); + + assert.fail('Should have thrown error.'); + } catch (err: any) { + assert.match(err.message, /Cannot fetch data without access token/); + } + }); + + it('refetches whole data when force synchronization used', async () => { + const storagePath = await createTmpConfigDir(); + const synchronizer = createSynchronizer(storagePath, stubs); + + const callsCounter = stubApi(synchronizer, stubs); + + await synchronizer.synchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + assert.equal(callsCounter.getUser, 1); + assert.equal(callsCounter.getProject, 1); + assert.equal(callsCounter.getPolicy, 1); + assert.equal(callsCounter.getSuppression, 1); + + await synchronizer.synchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + assert.equal(callsCounter.getUser, 2); + assert.equal(callsCounter.getProject, 2); + assert.equal(callsCounter.getPolicy, 1); + assert.equal(callsCounter.getSuppression, 2); + + await synchronizer.forceSynchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + assert.equal(callsCounter.getUser, 3); + assert.equal(callsCounter.getProject, 3); + assert.equal(callsCounter.getPolicy, 2); + assert.equal(callsCounter.getSuppression, 3); + }); + + it('synchronizes correctly', async () => { + const storagePath = await createTmpConfigDir(); + const synchronizer = createSynchronizer(storagePath, stubs); + + const callsCounter = stubApi(synchronizer, stubs); + + await synchronizer.synchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + const info1 = synchronizer.getProjectInfo(storagePath); + assert.isNotEmpty(info1); + assert.equal(info1?.name, 'User1 Project') + + const permissions1 = synchronizer.getProjectPermissions(storagePath); + assert.isObject(permissions1); + assert.isTrue(permissions1?.repositories.write); + + const policy1 = synchronizer.getProjectPolicy(storagePath); + assert.isObject(policy1); + assert.isTrue(policy1.valid); + assert.isNotEmpty(policy1.path); + assert.isNotEmpty(policy1.policy); + + const suppressions1 = synchronizer.getRepositorySuppressions(storagePath); + assert.isArray(suppressions1); + assert.equal(suppressions1.length, 1); + assert.isTrue(suppressions1[0].isAccepted); + + await synchronizer.synchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + const info2 = synchronizer.getProjectInfo(storagePath); + assert.isNotEmpty(info2); + assert.equal(info2?.name, 'User1 Project') + + const permissions2 = synchronizer.getProjectPermissions(storagePath); + assert.isObject(permissions2); + assert.isTrue(permissions2?.repositories.write); + + const policy2 = synchronizer.getProjectPolicy(storagePath); + assert.isObject(policy2); + assert.isTrue(policy2.valid); + assert.isNotEmpty(policy2.path); + assert.isNotEmpty(policy2.policy); + + const suppressions2 = synchronizer.getRepositorySuppressions(storagePath); + assert.isArray(suppressions2); + assert.equal(suppressions2.length, 2); + assert.isTrue(suppressions2[0].isAccepted); + assert.isTrue(suppressions2[1].isUnderReview); + + await synchronizer.synchronize({ + accessToken: 'SAMPLE_TOKEN' + } as any, storagePath); + + const info3 = synchronizer.getProjectInfo(storagePath); + assert.isNotEmpty(info3); + assert.equal(info3?.name, 'User1 Project') + + const permissions3 = synchronizer.getProjectPermissions(storagePath); + assert.isObject(permissions3); + assert.isTrue(permissions3?.repositories.write); + + const policy3 = synchronizer.getProjectPolicy(storagePath); + assert.isObject(policy3); + assert.isTrue(policy3.valid); + assert.isNotEmpty(policy3.path); + assert.isNotEmpty(policy3.policy); + + const suppressions3 = synchronizer.getRepositorySuppressions(storagePath); + assert.isArray(suppressions3); + assert.equal(suppressions3.length, 2); + assert.isTrue(suppressions3[0].isAccepted); + assert.isTrue(suppressions3[1].isUnderReview); + + assert.equal(callsCounter.getUser, 3); + assert.equal(callsCounter.getProject, 3); + assert.equal(callsCounter.getPolicy, 1); + assert.equal(callsCounter.getSuppression, 3); + }); +}); + +async function createTmpConfigDir(copyPolicyFixture = '') { + const testDir = resolve(__dirname, './'); + const testTmpDir = resolve(testDir, './tmp'); + const fixturesSourceDir = resolve(testDir, '../../../src/__tests__/fixtures'); + + await mkdir(testTmpDir, {recursive: true}); + + if (copyPolicyFixture) { + await cp(resolve(fixturesSourceDir, copyPolicyFixture), resolve(testTmpDir, copyPolicyFixture)); + } + + return testTmpDir; +} + +async function cleanupTmpConfigDir() { + const testDir = resolve(__dirname, './'); + const testTmpDir = resolve(testDir, './tmp'); + + await rm(testTmpDir, {recursive: true, force: true}); +} + +function createSynchronizer(storagePath: string, stubs: sinon.SinonStub[]) { + const synchronizer = createMonokleProjectSynchronizerFromConfig( + { + name: 'Tests', + version: 'unknown', + }, + { + origin: 'https://monokle.com', + apiOrigin: 'https://api.monokle.com', + authOrigin: 'https://auth.monokle.com', + schemasOrigin: 'https://schemas.monokle.com', + }, + new StorageHandlerPolicy(storagePath), + new StorageHandlerJsonCache(storagePath), + ); + + const getRootGitDataStub = sinon.stub((synchronizer as any), 'getRootGitData').callsFake(async () => { + return { + provider: 'GITHUB', + remote: 'origin', + owner: 'kubeshop', + name: 'monokle-demo', + }; + }); + + stubs.push(getRootGitDataStub); + + return synchronizer; +} + +function stubApi(synchronizer: any, stubs: SinonStub[]) { + const calls = { + getUser: 0, + getPolicy: 0, + getProject: 0, + getSuppression: 0, + } + + const queryApiStub = sinon.stub(synchronizer._apiHandler, 'queryApi').callsFake(async (...args) => { + const query = args[0] as string; + + if (query.includes('query getUser')) { + calls.getUser++; + return { + data: { + me: { + id: 1, + email: 'user1@kubeshop.io', + projects: [ + { + project: { + id: 1000, + slug: 'user1-proj', + name: 'User1 Project', + repositories: [ + { + id: 'user1-proj-policy-id', + projectId: 1000, + provider: 'GITHUB', + owner: 'kubeshop', + name: 'monokle-demo', + prChecks: false, + canEnablePrChecks: true, + }, + ], + }, + }, + ], + }, + }, + }; + } + + if (query.includes('query getPolicy')) { + calls.getPolicy++; + return { + data: { + getProject: { + id: 1000, + name: 'User1 Project', + policy: { + id: 'policy1', + json: { + plugins: { + 'pod-security-standards': true, + 'yaml-syntax': false, + 'resource-links': false, + 'kubernetes-schema': false, + practices: true, + }, + rules: { + 'pod-security-standards/host-process': 'err', + }, + settings: { + 'kubernetes-schema': { + schemaVersion: 'v1.27.1', + }, + }, + }, + }, + } + } + } + } + + if (query.includes('query getProject')) { + calls.getProject++; + return { + data: { + getProject: { + id: 1000, + slug: 'user1-proj', + name: 'User1 Project', + projectRepository: { + id: 'repo1', + projectId: 1000, + provider: 'GITHUB', + owner: 'kubeshop', + name: 'monokle-demo', + }, + permissions: { + project: { + view: true, + update: true, + delete: true + }, + members: { + view: true, + update: true, + delete: true + }, + repositories: { + read: true, + write: true + } + }, + policy: { + id: 'policy1', + updatedAt: '2024-02-08T12:15:10.298Z', + } + } + } + }; + } + + if (query.includes('query getSuppressions')) { + calls.getSuppression++; + + if (calls.getSuppression === 1) { + return { + data: { + getSuppressions: { + data: [{ + id: 'supp-1', + fingerprint: '16587e60761329', + description: 'K8S001 - Value at /spec/replicas should be integer', + location: 'blue-cms.deployment@bundles/simple.yaml@c9cf721b174f5-0', + status: 'ACCEPTED', + justification: null, + expiresAt: null, + updatedAt: '2024-02-01T13:32:10.445Z', + createdAt: '2024-02-01T13:32:10.445Z', + isUnderReview: false, + isAccepted: true, + isRejected: false, + isExpired: true, + isDeleted: false, + repositoryId: 'repo1', + }] + } + } + } + } else if (calls.getSuppression === 2) { + return { + data: { + getSuppressions: { + data: [{ + id: 'supp-1', + fingerprint: '16587e60761329', + description: 'K8S001 - Value at /spec/replicas should be integer', + location: 'blue-cms.deployment@bundles/simple.yaml@c9cf721b174f5-0', + status: 'REJECTED', + justification: null, + expiresAt: null, + updatedAt: '2024-02-01T13:32:10.445Z', + createdAt: '2024-02-01T13:32:10.445Z', + isUnderReview: false, + isAccepted: false, + isRejected: true, + isExpired: true, + isDeleted: true, + repositoryId: 'repo1', + }, { + id: 'supp-2', + fingerprint: '16587e607613292', + description: 'K8S001 - Value at /spec/replicas should be integer', + location: 'blue-cms.deployment@bundles/simple.yaml@c9cf721b174f5-0', + status: 'ACCEPTED', + justification: null, + expiresAt: null, + updatedAt: '2024-02-01T13:32:10.445Z', + createdAt: '2024-02-01T13:32:10.445Z', + isUnderReview: false, + isAccepted: true, + isRejected: false, + isExpired: true, + isDeleted: false, + repositoryId: 'repo1', + }, { + id: 'supp-3', + fingerprint: '16587e607613292', + description: 'K8S001 - Value at /spec/replicas should be integer', + location: 'blue-cms.deployment@bundles/simple.yaml@c9cf721b174f5-0', + status: 'UNDER_REVIEW', + justification: null, + expiresAt: null, + updatedAt: '2024-02-01T13:32:10.445Z', + createdAt: '2024-02-01T13:32:10.445Z', + isUnderReview: true, + isAccepted: false, + isRejected: false, + isExpired: true, + isDeleted: false, + repositoryId: 'repo1', + }] + } + } + } + } else { + return { + data: { + getSuppressions: { + data: [] + } + } + } + } + } + + return {}; + }); + stubs.push(queryApiStub); + + return calls; +} \ No newline at end of file diff --git a/packages/synchronizer/src/createMonokleProjectSynchronizer.ts b/packages/synchronizer/src/createMonokleProjectSynchronizer.ts new file mode 100644 index 000000000..5a92eaefd --- /dev/null +++ b/packages/synchronizer/src/createMonokleProjectSynchronizer.ts @@ -0,0 +1,37 @@ +import {DEFAULT_ORIGIN} from './constants.js'; +import {ApiHandler, ClientConfig} from './handlers/apiHandler.js'; +import {OriginConfig, fetchOriginConfig} from './handlers/configHandler.js'; +import {GitHandler} from './handlers/gitHandler.js'; +import {StorageHandlerJsonCache} from './handlers/storageHandlerJsonCache.js'; +import {StorageHandlerPolicy} from './handlers/storageHandlerPolicy.js'; +import {ProjectSynchronizer} from './utils/projectSynchronizer.js'; + +export async function createMonokleProjectSynchronizerFromOrigin( + clientConfig: ClientConfig, + origin: string = DEFAULT_ORIGIN, + storageHandlerPolicy: StorageHandlerPolicy = new StorageHandlerPolicy(), + storageHandlerJsonCache: StorageHandlerJsonCache = new StorageHandlerJsonCache(), + gitHandler: GitHandler = new GitHandler() +) { + try { + const originConfig = await fetchOriginConfig(origin); + + return createMonokleProjectSynchronizerFromConfig(clientConfig, originConfig, storageHandlerPolicy, storageHandlerJsonCache, gitHandler); + } catch (err: any) { + throw err; + } +} + +export function createMonokleProjectSynchronizerFromConfig( + clientConfig: ClientConfig, + originConfig: OriginConfig, + storageHandlerPolicy: StorageHandlerPolicy = new StorageHandlerPolicy(), + storageHandlerJsonCache: StorageHandlerJsonCache = new StorageHandlerJsonCache(), + gitHandler: GitHandler = new GitHandler() +) { + if (!originConfig?.apiOrigin) { + throw new Error(`No api origin found in origin config from ${origin}.`); + } + + return new ProjectSynchronizer(storageHandlerPolicy, storageHandlerJsonCache, new ApiHandler(originConfig, clientConfig), gitHandler); +} diff --git a/packages/synchronizer/src/handlers/apiHandler.ts b/packages/synchronizer/src/handlers/apiHandler.ts index 97c733084..8f400d890 100644 --- a/packages/synchronizer/src/handlers/apiHandler.ts +++ b/packages/synchronizer/src/handlers/apiHandler.ts @@ -49,6 +49,66 @@ const getProjectQuery = ` } `; +const getProjectPermissionsQuery = ` + query getProjectPermissions($slug: String!) { + getProject(input: { slug: $slug }) { + permissions { + project { + view + update + delete + } + members { + view + update + delete + } + repositories { + read + write + } + } + } + } +`; + +const getProjectDetailsQuery = ` + query getProjectDetails($slug: String!, $owner: String!, $name: String!, $provider: String!) { + getProject(input: { slug: $slug }) { + id + name + slug + projectRepository: repository(input: { owner: $owner, name: $name, provider: $provider }) { + id + projectId + provider + owner + name + } + permissions { + project { + view + update + delete + } + members { + view + update + delete + } + repositories { + read + write + } + } + policy { + id + updatedAt + } + } + } +`; + const getPolicyQuery = ` query getPolicy($slug: String!) { getProject(input: { slug: $slug }) { @@ -64,23 +124,32 @@ const getPolicyQuery = ` const getSuppressionsQuery = ` query getSuppressions( - $repositoryId: ID! + $repositoryId: ID!, + $from: String ) { getSuppressions( input: { - repositoryId: $repositoryId + repositoryId: $repositoryId, + from: $from } ) { isSnapshot data { id fingerprint + description + location status + justification + expiresAt + updatedAt + createdAt isUnderReview isAccepted isRejected isExpired isDeleted + repositoryId } } } @@ -96,6 +165,30 @@ const getRepoIdQuery = ` } `; +const toggleSuppressionMutation = ` + mutation toggleSuppression($fingerprint: String!, $repoId: ID!, $description: String!) { + toggleSuppression( + input: {fingerprint: $fingerprint, repository: $repoId, description: $description, skipReview: true} + ) { + id + fingerprint + description + location + status + justification + expiresAt + updatedAt + createdAt + isUnderReview + isAccepted + isRejected + isExpired + isDeleted + repositoryId + } + } +`; + export type ApiUserProjectRepo = { id: string; projectId: number; @@ -149,12 +242,19 @@ export type ApiPolicyData = { export type ApiSuppression = { id: string; fingerprint: string; + description: string; + locations: string; status: SuppressionStatus; + justification: string; + expiresAt: string; + updatedAt: string; + createdAt: string; isUnderReview: boolean; isAccepted: boolean; isRejected: boolean; isExpired: boolean; isDeleted: boolean; + repositoryId: string; }; export type ApiSuppressionsData = { @@ -183,6 +283,54 @@ export type ClientConfig = { additionalData?: Record; }; +export type ApiProjectPermissions = { + project: { + view: boolean, + update: boolean, + delete: boolean + }, + members: { + view: boolean, + update: boolean, + delete: boolean + }, + repositories: { + read: boolean, + write: boolean + } +}; + +export type ApiProjectPermissionsData = { + data: { + getProject: { + permissions: ApiProjectPermissions + } + } +}; + +export type ApiProjectDetailsData = { + data: { + getProject: { + id: number; + slug: string; + name: string; + projectRepository: ApiUserProjectRepo; + permissions: ApiProjectPermissions; + policy: { + id: string; + updatedAt: string; + } + } + } +}; + +export type getProjectDetailsInput = { + slug: string; + owner: string; + name: string; + provider:string; +} + export class ApiHandler { private _apiUrl: string; private _clientConfig: ClientConfig; @@ -229,12 +377,20 @@ export class ApiHandler { return this.queryApi(getProjectQuery, tokenInfo, {slug}); } + async getProjectPermissions(slug: string, tokenInfo: TokenInfo): Promise { + return this.queryApi(getProjectPermissionsQuery, tokenInfo, {slug}); + } + + async getProjectDetails(input: getProjectDetailsInput, tokenInfo: TokenInfo): Promise{ + return this.queryApi(getProjectDetailsQuery, tokenInfo, input); + } + async getPolicy(slug: string, tokenInfo: TokenInfo): Promise { return this.queryApi(getPolicyQuery, tokenInfo, {slug}); } - async getSuppressions(repositoryId: string, tokenInfo: TokenInfo): Promise { - return this.queryApi(getSuppressionsQuery, tokenInfo, {repositoryId}); + async getSuppressions(repositoryId: string, tokenInfo: TokenInfo, from?: string): Promise { + return this.queryApi(getSuppressionsQuery, tokenInfo, {repositoryId, from}); } async getRepoId( @@ -246,6 +402,10 @@ export class ApiHandler { return this.queryApi(getRepoIdQuery, tokenInfo, {projectSlug, repoOwner, repoName}); } + async toggleSuppression(fingerprint: string, repoId: string, description: string, tokenInfo: TokenInfo) { + return this.queryApi(toggleSuppressionMutation, tokenInfo, {fingerprint, repoId, description}); + } + generateDeepLink(path: string) { let appUrl = this._originConfig?.origin; diff --git a/packages/synchronizer/src/handlers/storageHandler.ts b/packages/synchronizer/src/handlers/storageHandler.ts index 7ebc9b520..8544edcf1 100644 --- a/packages/synchronizer/src/handlers/storageHandler.ts +++ b/packages/synchronizer/src/handlers/storageHandler.ts @@ -41,7 +41,7 @@ export abstract class StorageHandler { try { const data = readFileSync(file, 'utf8'); - const config = parse(data); + const config = this.parseData(data); return config; } catch (err: any) { throw new Error(`Failed to read configuration from '${file}' with error: ${err.message}`); @@ -55,7 +55,7 @@ export abstract class StorageHandler { try { const data = await readFile(file, 'utf8'); - const config = parse(data); + const config = this.parseData(data); return config; } catch (err: any) { throw new Error(`Failed to read configuration from '${file}' with error: ${err.message}`); @@ -72,6 +72,10 @@ export abstract class StorageHandler { throw new Error(`Failed to write configuration to '${file}' with error: ${err.message} and data: ${data}`); } } + + protected parseData(data: string) { + return parse(data); + } } export function getDefaultStorageConfigPaths(suffix = '') { diff --git a/packages/synchronizer/src/handlers/storageHandlerJsonCache.ts b/packages/synchronizer/src/handlers/storageHandlerJsonCache.ts new file mode 100644 index 000000000..f5c409a0a --- /dev/null +++ b/packages/synchronizer/src/handlers/storageHandlerJsonCache.ts @@ -0,0 +1,17 @@ +import {StorageHandler, getDefaultStorageConfigPaths} from './storageHandler.js'; + +export class StorageHandlerJsonCache extends StorageHandler> { + constructor(storageFolderPath: string = getDefaultStorageConfigPaths().cache) { + super(storageFolderPath); + } + + async setStoreData(data: Record, fileName: string): Promise { + const filePath = this.getStoreDataFilePath(fileName); + await this.writeStoreData(filePath, JSON.stringify(data)); + return filePath; + } + + protected parseData(data: string) { + return JSON.parse(data); + } +} diff --git a/packages/synchronizer/src/index.ts b/packages/synchronizer/src/index.ts index 36a7736a2..d6caca9a1 100644 --- a/packages/synchronizer/src/index.ts +++ b/packages/synchronizer/src/index.ts @@ -4,6 +4,7 @@ export * from './handlers/deviceFlowHandler.js'; export * from './handlers/gitHandler.js'; export * from './handlers/storageHandler.js'; export * from './handlers/storageHandlerAuth.js'; +export * from './handlers/storageHandlerJsonCache.js' export * from './handlers/storageHandlerPolicy.js'; export * from './models/user.js'; @@ -11,6 +12,7 @@ export * from './models/user.js'; export * from './utils/authenticator.js'; export * from './utils/fetcher.js'; export * from './utils/synchronizer.js'; +export * from './utils/projectSynchronizer.js'; export * from './constants.js'; @@ -21,3 +23,4 @@ export * from './createDefaultMonokleSynchronizer.js'; export * from './createMonokleAuthenticator.js'; export * from './createMonokleFetcher.js'; export * from './createMonokleSynchronizer.js'; +export * from './createMonokleProjectSynchronizer.js'; diff --git a/packages/synchronizer/src/utils/projectSynchronizer.ts b/packages/synchronizer/src/utils/projectSynchronizer.ts new file mode 100644 index 000000000..165e73d37 --- /dev/null +++ b/packages/synchronizer/src/utils/projectSynchronizer.ts @@ -0,0 +1,376 @@ +import slugify from 'slugify'; +import {EventEmitter} from 'events'; +import {normalize} from 'path'; +import {StorageHandlerPolicy, StoragePolicyFormat} from '../handlers/storageHandlerPolicy.js'; +import {StorageHandlerJsonCache} from '../handlers/storageHandlerJsonCache.js'; +import {ApiHandler} from '../handlers/apiHandler.js'; +import {GitHandler} from '../handlers/gitHandler.js'; +import type {ApiPolicyData, ApiProjectPermissions, ApiSuppression, ApiSuppressionsData, ApiUserProject, ApiUserProjectRepo} from '../handlers/apiHandler.js'; +import type {TokenInfo} from '../handlers/storageHandlerAuth.js'; +import type {RepoRemoteInputData} from './synchronizer.js'; +import type {ValidationConfig} from '@monokle/types'; + +export type CacheMetadata = { + suppressionsLastFetchDate: string; + policyLastUpdatedAt: string; + projectSlug: string; +}; + +export type ProjectDataCache = { + project: { + id: number; + slug: string; + name: string; + }, + permissions: ApiProjectPermissions; + repository: ApiUserProjectRepo; + suppressions: ApiSuppression[]; + policy: ValidationConfig; +}; + +export class ProjectSynchronizer extends EventEmitter { + private _dataCache: Record = {}; + private _pathToRepoMap: Record = {}; + + constructor( + private _storageHandlerPolicy: StorageHandlerPolicy, + private _storageHandlerJsonCache: StorageHandlerJsonCache, + private _apiHandler: ApiHandler, + private _gitHandler: GitHandler + ) { + super(); + } + + getProjectInfo(rootPath: string, projectSlug?: string) { + const cached = this._dataCache[this.getCacheId(rootPath, projectSlug)]; + return cached?.project ?? undefined; + } + + getProjectPermissions(rootPath: string, projectSlug?: string) { + return this._dataCache[this.getCacheId(rootPath, projectSlug)]?.permissions; + } + + getProjectPolicy(rootPath: string, projectSlug?: string) { + const cacheId = this.getCacheId(rootPath, projectSlug); + const cached = this._dataCache[cacheId]; + const repoData = this._pathToRepoMap[cacheId]; + + if (cached && repoData) { + return { + valid: true, + path: this.getPolicyPath(repoData), + policy: cached.policy, + }; + } + + return { + valid: false, + path: '', + policy: '', + }; + } + + getRepositoryData(rootPath: string, projectSlug?: string) { + return this._dataCache[this.getCacheId(rootPath, projectSlug)]?.repository ?? undefined; + } + + getRepositorySuppressions(rootPath: string, projectSlug?: string) { + return this._dataCache[this.getCacheId(rootPath, projectSlug)]?.suppressions ?? []; + } + + async toggleSuppression(tokenInfo: TokenInfo, fingerprint: string, description: string, rootPath: string, projectSlug?: string) { + if (!tokenInfo?.accessToken?.length) { + throw new Error('Cannot use suppressions without access token.'); + } + + const repoData = await this.getRootGitData(rootPath); + + let {id} = this.getRepositoryData(rootPath, projectSlug); + let {slug} = this.getProjectInfo(rootPath, projectSlug); + if (!id || !slug) { + const ownerProject = await this.getMatchingProject(repoData, tokenInfo); + if (!ownerProject) { + // This error would mostly be caused by incorrect integration. Integrator should + // make sure that suppressions are only used for repos with owner project. + throw new Error('Trying to suppress annotation for repo without owner project!'); + } + + const repoIdData = await this._apiHandler.getRepoId(ownerProject.slug, repoData.owner, repoData.name, tokenInfo); + + id = repoIdData?.data.getProject.repository.id ?? ''; + slug = ownerProject.slug; + } + + if (!id || !slug) { + throw new Error('Cannot suppress due to missing repository id or project slug!'); + } + + const suppressionResult = await this._apiHandler.toggleSuppression(fingerprint, id, description, tokenInfo); + if (suppressionResult?.data?.getSuppressions?.data?.length) { + const existingSuppressions = await this.readSuppressions(repoData); + const allSuppressions = this.mergeSuppressions(existingSuppressions, suppressionResult.data.getSuppressions.data); + await this.storeSuppressions(allSuppressions, repoData); + + const cacheId = this.getCacheId(rootPath, projectSlug); + if (this._dataCache[cacheId]) { + this._dataCache[cacheId].suppressions = allSuppressions; + } + } + } + + async synchronize(tokenInfo: TokenInfo, rootPath: string, projectSlug?: string): Promise { + if (!tokenInfo?.accessToken?.length) { + throw new Error('Cannot fetch data without access token.'); + } + + const repoData = await this.getRootGitData(rootPath); + const ownerProjectSlug = projectSlug ?? (await this.getMatchingProject(repoData, tokenInfo))?.slug; + + const cacheId = this.getCacheId(rootPath, projectSlug); + this._pathToRepoMap[cacheId] = { + ...repoData, + ownerProjectSlug: projectSlug, + }; + + if (!ownerProjectSlug) { + const projectUrl = this._apiHandler.generateDeepLink(`/dashboard/projects`); + throw new Error( + `The '${rootPath}' repository does not belong to any project in Monokle Cloud. Configure it on ${projectUrl}.` + ); + } + + if (ownerProjectSlug && repoData?.owner && repoData?.name && repoData?.provider) { + const projectDetails = await this.refetchProjectDetails(repoData, ownerProjectSlug, tokenInfo); + if (!projectDetails?.data?.getProject?.policy) { + const policyUrl = this._apiHandler.generateDeepLink(`/dashboard/projects/${ownerProjectSlug}/policy`); + throw new Error( + `The '${rootPath}' repository project does not have policy defined. Configure it on ${policyUrl}.` + ); + } + + let resyncDueToError = false; + let existingSuppressions: ApiSuppression[] = []; + let existingPolicy = {}; + try { + existingSuppressions = await this.readSuppressions(repoData); + existingPolicy = await this.readPolicy(repoData) ?? {} + } catch (err) { + resyncDueToError = true; + } + + if (resyncDueToError) { + await this.dropCacheMetadata(repoData); + } + + const projectValidationData = await this.refetchProjectValidationData( + repoData, + projectDetails.data.getProject.projectRepository.id, + projectDetails.data.getProject.policy.updatedAt, + ownerProjectSlug, + tokenInfo + ); + + const dataCache: ProjectDataCache = { + project: { + id: projectDetails.data.getProject.id, + slug: projectDetails.data.getProject.slug, + name: projectDetails.data.getProject.name, + }, + permissions: projectDetails.data.getProject.permissions, + repository: projectDetails.data.getProject.projectRepository, + suppressions: existingSuppressions, + policy: existingPolicy + }; + + if (projectValidationData.suppressions.length) { + const allSuppressions = this.mergeSuppressions(existingSuppressions, projectValidationData.suppressions); + await this.storeSuppressions(allSuppressions, repoData); + dataCache.suppressions = allSuppressions; + } + + if (projectValidationData.policy?.json) { + const policyContent: StoragePolicyFormat = projectValidationData.policy.json; + const policyUrl = this._apiHandler.generateDeepLink(`/dashboard/projects/${ownerProjectSlug}/policy`); + const comment = [ + ` This is remote policy downloaded from ${this._apiHandler.apiUrl}.`, + ` You can adjust it on ${policyUrl}.`, + ].join('\n'); + + const policyPath = await this.storePolicy(policyContent, this._pathToRepoMap[cacheId], comment); + if (!policyPath) { + throw new Error(`Error storing policy in local filesystem.`); + } + + dataCache.policy = projectValidationData.policy.json; + } + + this._dataCache[cacheId] = dataCache; + + this.emit('synchronized'); + } + } + + async forceSynchronize(tokenInfo: TokenInfo, rootPath: string, projectSlug?: string): Promise { + const repoData = await this.getRootGitData(rootPath); + await this.dropCacheMetadata(repoData); + return this.synchronize(tokenInfo, rootPath, projectSlug); + } + + private async refetchProjectDetails(repoData: RepoRemoteInputData, ownerProjectSlug: string, tokenInfo: TokenInfo) { + return this._apiHandler.getProjectDetails({ + slug: ownerProjectSlug, + owner: repoData.owner, + name: repoData.name, + provider: repoData.provider, + }, tokenInfo); + } + + private async refetchProjectValidationData(repoData: RepoRemoteInputData, repoId: string, policyUpdatedAt: string, ownerProjectSlug: string, tokenInfo: TokenInfo) { + const cacheMetadata = await this.readCacheMetadata(repoData); + const isCachedProjectMatching = ownerProjectSlug === cacheMetadata?.projectSlug; + + const newSuppressionsLastFetchDate = (new Date()).toISOString(); + const fetchSuppressionsFrom = isCachedProjectMatching ? cacheMetadata?.suppressionsLastFetchDate : undefined; + + const dataRefetchQueries: Promise[] = [ + this._apiHandler.getSuppressions(repoId, tokenInfo, fetchSuppressionsFrom) + ]; + if (!isCachedProjectMatching || !cacheMetadata?.policyLastUpdatedAt || cacheMetadata.policyLastUpdatedAt !== policyUpdatedAt) { + dataRefetchQueries.push(this._apiHandler.getPolicy(ownerProjectSlug, tokenInfo)); + } + + const newData = await Promise.all(dataRefetchQueries); + + const newCacheMetadata: CacheMetadata = { + suppressionsLastFetchDate: newSuppressionsLastFetchDate, + policyLastUpdatedAt: policyUpdatedAt, + projectSlug: ownerProjectSlug + }; + this.storeCacheMetadata(newCacheMetadata, repoData); + + return { + suppressions: (newData[0] as ApiSuppressionsData | undefined)?.data?.getSuppressions?.data ?? [], + policy: (newData[1] as ApiPolicyData | undefined)?.data?.getProject?.policy, + } + } + + private async getMatchingProject( + repoData: RepoRemoteInputData, + tokenInfo: TokenInfo + ): Promise { + const userData = await this._apiHandler.getUser(tokenInfo); + if (!userData?.data?.me) { + throw new Error('Cannot fetch user data, make sure you are authenticated and have internet access.'); + } + + if (!repoData?.provider || !repoData?.owner || !repoData?.name) { + throw new Error(`Provided invalid git repository data: '${JSON.stringify(repoData)}'.`); + } + + const repoMatchingProjectBySlug = userData.data.me.projects.find(project => { + return project.project.slug === repoData.ownerProjectSlug; + }); + + const repoFirstProject = userData.data.me.projects.find(project => { + return project.project.repositories.find( + repo => + repo.owner.toLowerCase() === repoData.owner.toLowerCase() && + repo.name.toLowerCase() === repoData.name.toLowerCase() && + repo.provider.toLowerCase() === repoData.provider.toLowerCase() + ); + }); + + return (repoMatchingProjectBySlug ?? repoFirstProject)?.project ?? null; + } + + private async getRootGitData(rootPath: string) { + const repoData = await this._gitHandler.getRepoRemoteData(rootPath); + if (!repoData) { + throw new Error(`The '${rootPath}' is not a git repository or does not have any remotes.`); + } + + return repoData; + } + + private getCacheId(rootPath: string, projectSlug?: string) { + return `${normalize(rootPath)}${projectSlug ? `+${projectSlug}` : ''}`; + } + + private async storePolicy( + policyContent: StoragePolicyFormat, + repoData: RepoRemoteInputData, + comment: string + ) { + return this._storageHandlerPolicy.setStoreData(policyContent, this.getPolicyFileName(repoData), comment); + } + + private async readPolicy(inputData: RepoRemoteInputData) { + return this._storageHandlerPolicy.getStoreData(this.getPolicyFileName(inputData)); + } + + private getPolicyPath(inputData: RepoRemoteInputData) { + return this._storageHandlerPolicy.getStoreDataFilePath(this.getPolicyFileName(inputData)); + } + + private async storeSuppressions(suppressions: ApiSuppression[], repoData: RepoRemoteInputData) { + return this._storageHandlerJsonCache.setStoreData(suppressions, this.getSuppressionsFileName(repoData)); + } + + private async readSuppressions(repoData: RepoRemoteInputData) { + return (await this._storageHandlerJsonCache.getStoreData(this.getSuppressionsFileName(repoData)) ?? []) as ApiSuppression[]; + } + + private mergeSuppressions(existing: ApiSuppression[], updated: ApiSuppression[]) { + const suppressionsMap = existing.reduce((prev: Record, curr: ApiSuppression) => { + prev[curr.id] = curr; + return prev; + }, {}); + + updated.forEach(suppression => { + suppressionsMap[suppression.id] = suppression; + }); + + return Object.values(suppressionsMap) + .filter(suppression => !(suppression.isDeleted || suppression.isRejected)); + } + + private async storeCacheMetadata(cacheMetadata: CacheMetadata, repoData: RepoRemoteInputData) { + return this._storageHandlerJsonCache.setStoreData(cacheMetadata, this.getMetadataFileName(repoData)); + } + + private async readCacheMetadata(repoData: RepoRemoteInputData) { + try { + return (await this._storageHandlerJsonCache.getStoreData(this.getMetadataFileName(repoData)) ?? {}) as CacheMetadata; + } catch (err) { + return {} as CacheMetadata; + } + } + + private async dropCacheMetadata(repoData: RepoRemoteInputData) { + return this._storageHandlerJsonCache.emptyStoreData(this.getMetadataFileName(repoData)); + } + + private getPolicyFileName(repoData: RepoRemoteInputData) { + return this.getFileName(repoData, 'policy'); + } + + private getSuppressionsFileName(repoData: RepoRemoteInputData) { + return this.getFileName(repoData, 'suppressions'); + } + + private getMetadataFileName(repoData: RepoRemoteInputData) { + return this.getFileName(repoData, 'metadata'); + } + + private getFileName(repoData: RepoRemoteInputData, suffix: string, ext = 'yaml') { + const provider = slugify(repoData.provider, { + replacement: '_', + lower: true, + strict: true, + locale: 'en', + trim: true, + }); + + return `${provider}-${repoData.owner}-${repoData.name}.${suffix}.${ext}`; + } +} diff --git a/packages/synchronizer/src/utils/synchronizer.ts b/packages/synchronizer/src/utils/synchronizer.ts index 83adac238..ff9431310 100644 --- a/packages/synchronizer/src/utils/synchronizer.ts +++ b/packages/synchronizer/src/utils/synchronizer.ts @@ -33,6 +33,9 @@ export type ProjectInfo = { slug: string; }; +/** + * @deprecated ProjectSynchronizer should be used instead if possible. + */ export class Synchronizer extends EventEmitter { private _pullPromise: Promise | undefined; private _projectDataCache: Record = {};