diff --git a/package-lock.json b/package-lock.json index dce2cbcd8..80cb80cd1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28041,7 +28041,7 @@ "devDependencies": { "@ant-design/icons": "4.7.0", "@babel/core": "7.17.8", - "@monokle/validation": "0.25.2", + "@monokle/validation": "0.25.3", "@rjsf/antd": "5.0.0-beta.11", "@storybook/addon-actions": "6.5.16", "@storybook/addon-essentials": "6.5.16", @@ -28318,7 +28318,7 @@ }, "packages/synchronizer": { "name": "@monokle/synchronizer", - "version": "0.3.0", + "version": "0.5.0", "license": "MIT", "dependencies": { "@monokle/types": "*", @@ -28954,13 +28954,14 @@ }, "packages/types": { "name": "@monokle/types", - "version": "0.1.0" + "version": "0.2.0" }, "packages/validation": { "name": "@monokle/validation", - "version": "0.25.2", + "version": "0.25.3", "license": "MIT", "dependencies": { + "@monokle/parser": "*", "@monokle/types": "*", "@open-policy-agent/opa-wasm": "1.8.0", "@rollup/plugin-virtual": "3.0.1", @@ -31847,7 +31848,7 @@ "requires": { "@ant-design/icons": "4.7.0", "@babel/core": "7.17.8", - "@monokle/validation": "0.25.2", + "@monokle/validation": "0.25.3", "@rjsf/antd": "5.0.0-beta.11", "@storybook/addon-actions": "6.5.16", "@storybook/addon-essentials": "6.5.16", @@ -31896,7 +31897,7 @@ "requires": { "@types/lodash": "4.14.196", "lodash": "4.17.21", - "path-browserify": "*", + "path-browserify": "^1.0.1", "rimraf": "3.0.2", "typescript": "4.8.3", "yaml": "2.2.2" @@ -32371,6 +32372,7 @@ "@monokle/validation": { "version": "file:packages/validation", "requires": { + "@monokle/parser": "*", "@monokle/types": "*", "@open-policy-agent/opa-wasm": "1.8.0", "@rollup/plugin-virtual": "3.0.1", diff --git a/packages/validation/package.json b/packages/validation/package.json index 24cbe69da..412f85789 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -60,6 +60,7 @@ "vitest": "0.29.2" }, "dependencies": { + "@monokle/parser": "*", "@monokle/types": "*", "@open-policy-agent/opa-wasm": "1.8.0", "@rollup/plugin-virtual": "3.0.1", diff --git a/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts b/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts index bff483ce7..ec370432f 100644 --- a/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts +++ b/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts @@ -4,7 +4,8 @@ import {processRefs} from '../references/process.js'; // Usage note: This library relies on fetch being on global scope! import 'isomorphic-fetch'; -import {expectResult, extractK8sResources, readDirectory} from './testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; +import {readDirectory, expectResult} from './testUtils.js'; import {ResourceParser} from '../common/resourceParser.js'; import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js'; diff --git a/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts b/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts index c1edbba2c..010011fd8 100644 --- a/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts +++ b/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts @@ -4,10 +4,11 @@ import {processRefs} from '../references/process.js'; // Usage note: This library relies on fetch being on global scope! import 'isomorphic-fetch'; -import {expectResult, extractK8sResources, readDirectory} from './testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; +import {readDirectory, expectResult} from './testUtils.js'; import {ResourceParser} from '../common/resourceParser.js'; import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js'; -import { Config, RuleMap } from '../config/parse.js'; +import {Config, RuleMap} from '../config/parse.js'; it('should detect missing recommended labels (MTD-recommended-labels)', async () => { const {response} = await processResourcesInFolder('src/__tests__/resources/metadata'); @@ -35,7 +36,7 @@ it('should detect missing recommended labels (MTD-recommended-labels)', async () it('should not override recommended labels but allow to config level (MTD-recommended-labels)', async () => { const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', { - 'metadata/recommended-labels': ['warn', ['app', 'tier', 'role']] + 'metadata/recommended-labels': ['warn', ['app', 'tier', 'role']], }); const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0); @@ -62,7 +63,7 @@ it('should not override recommended labels but allow to config level (MTD-recomm it('should detect missing custom labels (MTD-custom-labels)', async () => { const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', { 'metadata/recommended-labels': false, - 'metadata/custom-labels': ['warn', ['app', 'tier', 'role']] + 'metadata/custom-labels': ['warn', ['app', 'tier', 'role']], }); const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0); @@ -78,7 +79,7 @@ it('should detect missing custom labels (MTD-custom-labels)', async () => { it('should detect missing annotations labels (MTD-custom-annotations)', async () => { const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', { 'metadata/recommended-labels': false, - 'metadata/custom-annotations': ['warn', ['revision', 'hash', 'annotation-1', 'annotation-2']] + 'metadata/custom-annotations': ['warn', ['revision', 'hash', 'annotation-1', 'annotation-2']], }); const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0); @@ -100,7 +101,7 @@ it('should not trigger when predefined custom rules have no names defined (MTD-c const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', { 'metadata/recommended-labels': false, 'metadata/custom-labels': 'err', - 'metadata/custom-annotations': 'err' + 'metadata/custom-annotations': 'err', }); const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0); @@ -187,7 +188,7 @@ it('should have custom-* configurable rules', async () => { await configureValidator(validator, { 'metadata/recommended-labels': false, 'metadata/custom-labels': 'err', - 'metadata/custom-annotations': 'err' + 'metadata/custom-annotations': 'err', }); Object.entries(validator.rules)[0][1].forEach(rule => { @@ -241,7 +242,7 @@ async function processResourcesInFolder(path: string, rules?: RuleMap) { async function configureValidator(validator: MonokleValidator, rules?: RuleMap) { const config: Config = { plugins: { - 'metadata': true, + metadata: true, }, settings: { debug: true, diff --git a/packages/validation/src/__tests__/MonokleValidator.pss.test.ts b/packages/validation/src/__tests__/MonokleValidator.pss.test.ts index dc1d4457d..051287bd8 100644 --- a/packages/validation/src/__tests__/MonokleValidator.pss.test.ts +++ b/packages/validation/src/__tests__/MonokleValidator.pss.test.ts @@ -7,7 +7,8 @@ import 'isomorphic-fetch'; import {ResourceParser} from '../common/resourceParser.js'; import {Config, RuleMap} from '../config/parse.js'; import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js'; -import {extractK8sResources, readDirectory} from './testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; +import {readDirectory} from './testUtils.js'; it('should detect invalid volume types', async () => { const {response} = await processResourcesInFolder('src/__tests__/resources/pss-1', { diff --git a/packages/validation/src/__tests__/MonokleValidator.test.ts b/packages/validation/src/__tests__/MonokleValidator.test.ts index 37d8181f8..cbca5825a 100644 --- a/packages/validation/src/__tests__/MonokleValidator.test.ts +++ b/packages/validation/src/__tests__/MonokleValidator.test.ts @@ -6,7 +6,8 @@ import {processRefs} from '../references/process.js'; // Usage note: This library relies on fetch being on global scope! import 'isomorphic-fetch'; import {RESOURCES} from './badResources.js'; -import {extractK8sResources, readDirectory} from './testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; +import {readDirectory} from './testUtils.js'; import {ResourceRefType} from '../common/types.js'; import {ResourceParser} from '../common/resourceParser.js'; import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js'; @@ -79,7 +80,7 @@ it('should support patches and additionalValuesFiles', async () => { it('should support Kustomize Components', async () => { const {resources, response} = await processResourcesInFolder('src/__tests__/resources/kustomize-components'); - expect( resources.length ).toBe( 3 ); + expect(resources.length).toBe(3); }); it('should support ownerRefs', async () => { @@ -140,7 +141,7 @@ it('should allow rules to be configurable', async () => { type: RuleConfigMetadataType.Number, name: 'Required replicas', defaultValue: 1, - } + }, }, validate({resources, params}, {report}) { resources.filter(isDeployment).forEach(deployment => { diff --git a/packages/validation/src/__tests__/sarif/fingerprint.test.ts b/packages/validation/src/__tests__/sarif/fingerprint.test.ts index b417bfd3d..1b2a72d04 100644 --- a/packages/validation/src/__tests__/sarif/fingerprint.test.ts +++ b/packages/validation/src/__tests__/sarif/fingerprint.test.ts @@ -2,7 +2,8 @@ import 'isomorphic-fetch'; import {expect, it} from 'vitest'; import {createDefaultMonokleValidator} from '../../index.js'; -import {PRACTICES_ALL_DISABLED, extractK8sResources, readDirectory} from '../testUtils.js'; +import {PRACTICES_ALL_DISABLED, readDirectory} from '../testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; it('should have fingerprints & baseline', async () => { const validator = createDefaultMonokleValidator(); diff --git a/packages/validation/src/__tests__/sarif/suppression.test.ts b/packages/validation/src/__tests__/sarif/suppression.test.ts index 2a7111e5f..acad37ad3 100644 --- a/packages/validation/src/__tests__/sarif/suppression.test.ts +++ b/packages/validation/src/__tests__/sarif/suppression.test.ts @@ -2,7 +2,8 @@ import 'isomorphic-fetch'; import {expect, it} from 'vitest'; import {createDefaultMonokleValidator} from '../../index.js'; -import {PRACTICES_ALL_DISABLED, extractK8sResources, readDirectory} from '../testUtils.js'; +import {extractK8sResources} from '@monokle/parser'; +import {PRACTICES_ALL_DISABLED, readDirectory} from '../testUtils.js'; import {FakeSuppressor} from '../../sarif/suppressions/plugins/FakeSuppressor.js'; import YAML from 'yaml'; import {set} from 'lodash'; diff --git a/packages/validation/src/__tests__/testUtils.ts b/packages/validation/src/__tests__/testUtils.ts index 287e3444b..1a65c5d70 100644 --- a/packages/validation/src/__tests__/testUtils.ts +++ b/packages/validation/src/__tests__/testUtils.ts @@ -1,188 +1,19 @@ -import {v5} from 'uuid'; -import {parse} from 'path'; + import glob from 'tiny-glob'; import {readFile as readFileFromFs} from 'fs/promises'; import chunkArray from 'lodash/chunk.js'; -import {LineCounter, parseAllDocuments, parseDocument} from 'yaml'; -import {Resource, ValidationResult} from '../index.js'; +import {ValidationResult} from '../index.js'; import {expect} from 'vitest'; +import type {BaseFile} from '@monokle/parser'; -export const KUSTOMIZATION_KIND = 'Kustomization'; -export const KUSTOMIZATION_API_GROUP = 'kustomize.config.k8s.io'; - -/** - * This is all copied from cli/src/utils - need to be moved to their own shared package - * so they can be reused across both cli and validator - see https://github.com/kubeshop/monokle-core/issues/63 - */ - -export type File = { - id: string; - path: string; - content: string; -}; - -export function extractK8sResources(files: File[]): Resource[] { - const resources: Resource[] = []; - - for (const file of files) { - const lineCounter = new LineCounter(); - const documents = parseAllYamlDocuments(file.content, lineCounter); - - for (const document of documents) { - const content = document.toJS(); - - if (document.errors.length) { - continue; - } - - const rawFileOffset = lineCounter.linePos(document.range[0]).line; - const fileOffset = rawFileOffset === 1 ? 0 : rawFileOffset; - - const resourceBase = { - apiVersion: content.apiVersion, - kind: content.kind, - content, - fileId: file.id, - filePath: file.path, - fileOffset, - text: document.toString({directives: false}), - }; - - if (isKubernetesLike(content)) { - const name = createResourceName(file.path, content, content.kind); - const id = createResourceId(file.id, content.kind, name, content.metadata?.namespace); - const namespace = extractNamespace(content); - const resource = { - ...resourceBase, - id, - name, - namespace, - }; - - resources.push(resource); - } else if (content && isUntypedKustomizationFile(file.path) && documents.length === 1) { - const name = createResourceName(file.path, content, KUSTOMIZATION_KIND); - const id = createResourceId(file.id, name, KUSTOMIZATION_KIND); - const resource = { - ...resourceBase, - kind: KUSTOMIZATION_KIND, - apiVersion: KUSTOMIZATION_API_GROUP, - id, - name, - }; - - resources.push(resource); - } else if (content && typeof content.kind === 'string') { - // Load K8s schemas only with no apiVersion present for testing purposes. - const name = createResourceName(file.path, content, content.kind); - const id = createResourceId(file.id, content.kind, name, content.metadata?.namespace); - const namespace = extractNamespace(content); - const resource = { - ...resourceBase, - id, - name, - namespace, - }; - - resources.push(resource); - } - } - } - - return resources; -} - -type KubernetesLike = { - apiVersion: string; - kind: string; - metadata?: { - name: string; - namespace?: string; - }; - spec?: { - names?: { - kind?: string; - }; - }; -}; - -function isKubernetesLike(content: any): content is KubernetesLike { - return content && typeof content.apiVersion === 'string' && typeof content.kind === 'string'; -} - -// some (older) kustomization yamls don't contain kind/group properties to identify them as such -// they are identified only by their name -function isUntypedKustomizationFile(filePath = ''): boolean { - return /kustomization*.yaml/.test(filePath.toLowerCase().trim()); -} - -export function isYamlFile(file: File): boolean { - return file.path.endsWith('.yml') || file.path.endsWith('.yaml'); -} -export function parseYamlDocument(text: string, lineCounter?: LineCounter) { - return parseDocument(text, {lineCounter, uniqueKeys: false, strict: false}); -} - -/** - * Wrapper that ensures consistent options - */ - -export function parseAllYamlDocuments(text: string, lineCounter?: LineCounter) { - return parseAllDocuments(text, { - lineCounter, - uniqueKeys: false, - strict: false, - }); -} - -function extractNamespace(content: any) { - // namespace could be an object if it's a helm template value... - return content.metadata?.namespace && typeof content.metadata.namespace === 'string' - ? content.metadata.namespace - : undefined; -} - -const RESOURCE_UUID_NAMESPACE = '6fa71997-8aa8-4b89-b987-cec4fd3de770'; - -export const createResourceId = (fileId: string, name: string, kind: string, namespace?: string | null): string => { - return v5(`${fileId}${kind}${name}${namespace || ''}`, RESOURCE_UUID_NAMESPACE); -}; - -export function createResourceName(filePath: string, content: any, kind: string): string { - const parsedPath = parse(filePath); - - // dirname for kustomizations - // if ( - // kind === KUSTOMIZATION_KIND && - // (!content?.apiVersion || - // content.apiVersion.startsWith(KUSTOMIZATION_API_GROUP)) - // ) { - // return parsedPath.dir.replace(/^\/*/, ''); - // } - - try { - // metadata name - return typeof content.metadata.name === 'string' - ? content.metadata.name.trim() - : JSON.stringify(content.metadata.name).trim(); - } catch (error) { - // filename - return parsedPath.name; - } -} - -export function getResourcesForPath(filePath: string, resources: Resource[] | undefined) { - return resources ? resources.filter(resource => resource.filePath === filePath) : []; -} - -export async function readDirectory(directoryPath: string): Promise { +export async function readDirectory(directoryPath: string): Promise { const filePaths = await glob(`${directoryPath}/**/*.{yaml,yml}`); - const files: File[] = []; + const files: BaseFile[] = []; for (const chunk of chunkArray(filePaths, 5)) { const promise = await Promise.allSettled( chunk.map(path => { - return readFileFromFs(path, 'utf8').then((content): File => ({id: path, path, content})); + return readFileFromFs(path, 'utf8').then((content): BaseFile => ({id: path, path, content})); }) );