refactor: migrate 'validation' package to 'parser'

kubeshop · Aug 24, 2023 · 750135e · 750135e
1 parent 93062dc
commit 750135e
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 196 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/validation/package.json b/packages/validation/package.json
@@ -60,6 +60,7 @@
     "vitest": "0.29.2"
   },
   "dependencies": {
+    "@monokle/parser": "*",
     "@monokle/types": "*",
     "@open-policy-agent/opa-wasm": "1.8.0",
     "@rollup/plugin-virtual": "3.0.1",

diff --git a/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts b/packages/validation/src/__tests__/MonokleValidator.kubernetes-schema.test.ts
@@ -4,7 +4,8 @@ import {processRefs} from '../references/process.js';
 
 // Usage note: This library relies on fetch being on global scope!
 import 'isomorphic-fetch';
-import {expectResult, extractK8sResources, readDirectory} from './testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
+import {readDirectory, expectResult} from './testUtils.js';
 import {ResourceParser} from '../common/resourceParser.js';
 import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js';
 

diff --git a/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts b/packages/validation/src/__tests__/MonokleValidator.metadata.test.ts
@@ -4,10 +4,11 @@ import {processRefs} from '../references/process.js';
 
 // Usage note: This library relies on fetch being on global scope!
 import 'isomorphic-fetch';
-import {expectResult, extractK8sResources, readDirectory} from './testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
+import {readDirectory, expectResult} from './testUtils.js';
 import {ResourceParser} from '../common/resourceParser.js';
 import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js';
-import { Config, RuleMap } from '../config/parse.js';
+import {Config, RuleMap} from '../config/parse.js';
 
 it('should detect missing recommended labels (MTD-recommended-labels)', async () => {
   const {response} = await processResourcesInFolder('src/__tests__/resources/metadata');
@@ -35,7 +36,7 @@ it('should detect missing recommended labels (MTD-recommended-labels)', async ()
 
 it('should not override recommended labels but allow to config level (MTD-recommended-labels)', async () => {
   const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', {
-    'metadata/recommended-labels': ['warn', ['app', 'tier', 'role']]
+    'metadata/recommended-labels': ['warn', ['app', 'tier', 'role']],
   });
 
   const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0);
@@ -62,7 +63,7 @@ it('should not override recommended labels but allow to config level (MTD-recomm
 it('should detect missing custom labels (MTD-custom-labels)', async () => {
   const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', {
     'metadata/recommended-labels': false,
-    'metadata/custom-labels': ['warn', ['app', 'tier', 'role']]
+    'metadata/custom-labels': ['warn', ['app', 'tier', 'role']],
   });
 
   const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0);
@@ -78,7 +79,7 @@ it('should detect missing custom labels (MTD-custom-labels)', async () => {
 it('should detect missing annotations labels (MTD-custom-annotations)', async () => {
   const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', {
     'metadata/recommended-labels': false,
-    'metadata/custom-annotations': ['warn', ['revision', 'hash', 'annotation-1', 'annotation-2']]
+    'metadata/custom-annotations': ['warn', ['revision', 'hash', 'annotation-1', 'annotation-2']],
   });
 
   const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0);
@@ -100,7 +101,7 @@ it('should not trigger when predefined custom rules have no names defined (MTD-c
   const {response} = await processResourcesInFolder('src/__tests__/resources/metadata', {
     'metadata/recommended-labels': false,
     'metadata/custom-labels': 'err',
-    'metadata/custom-annotations': 'err'
+    'metadata/custom-annotations': 'err',
   });
 
   const hasErrors = response.runs.reduce((sum, r) => sum + r.results.length, 0);
@@ -187,7 +188,7 @@ it('should have custom-* configurable rules', async () => {
   await configureValidator(validator, {
     'metadata/recommended-labels': false,
     'metadata/custom-labels': 'err',
-    'metadata/custom-annotations': 'err'
+    'metadata/custom-annotations': 'err',
   });
 
   Object.entries(validator.rules)[0][1].forEach(rule => {
@@ -241,7 +242,7 @@ async function processResourcesInFolder(path: string, rules?: RuleMap) {
 async function configureValidator(validator: MonokleValidator, rules?: RuleMap) {
   const config: Config = {
     plugins: {
-      'metadata': true,
+      metadata: true,
     },
     settings: {
       debug: true,

diff --git a/packages/validation/src/__tests__/MonokleValidator.pss.test.ts b/packages/validation/src/__tests__/MonokleValidator.pss.test.ts
@@ -7,7 +7,8 @@ import 'isomorphic-fetch';
 import {ResourceParser} from '../common/resourceParser.js';
 import {Config, RuleMap} from '../config/parse.js';
 import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js';
-import {extractK8sResources, readDirectory} from './testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
+import {readDirectory} from './testUtils.js';
 
 it('should detect invalid volume types', async () => {
   const {response} = await processResourcesInFolder('src/__tests__/resources/pss-1', {

diff --git a/packages/validation/src/__tests__/MonokleValidator.test.ts b/packages/validation/src/__tests__/MonokleValidator.test.ts
@@ -6,7 +6,8 @@ import {processRefs} from '../references/process.js';
 // Usage note: This library relies on fetch being on global scope!
 import 'isomorphic-fetch';
 import {RESOURCES} from './badResources.js';
-import {extractK8sResources, readDirectory} from './testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
+import {readDirectory} from './testUtils.js';
 import {ResourceRefType} from '../common/types.js';
 import {ResourceParser} from '../common/resourceParser.js';
 import {createDefaultMonokleValidator} from '../createDefaultMonokleValidator.node.js';
@@ -79,7 +80,7 @@ it('should support patches and additionalValuesFiles', async () => {
 
 it('should support Kustomize Components', async () => {
   const {resources, response} = await processResourcesInFolder('src/__tests__/resources/kustomize-components');
-  expect( resources.length ).toBe( 3 );
+  expect(resources.length).toBe(3);
 });
 
 it('should support ownerRefs', async () => {
@@ -140,7 +141,7 @@ it('should allow rules to be configurable', async () => {
                 type: RuleConfigMetadataType.Number,
                 name: 'Required replicas',
                 defaultValue: 1,
-              }
+              },
             },
             validate({resources, params}, {report}) {
               resources.filter(isDeployment).forEach(deployment => {

diff --git a/packages/validation/src/__tests__/sarif/fingerprint.test.ts b/packages/validation/src/__tests__/sarif/fingerprint.test.ts
@@ -2,7 +2,8 @@ import 'isomorphic-fetch';
 import {expect, it} from 'vitest';
 import {createDefaultMonokleValidator} from '../../index.js';
 
-import {PRACTICES_ALL_DISABLED, extractK8sResources, readDirectory} from '../testUtils.js';
+import {PRACTICES_ALL_DISABLED, readDirectory} from '../testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
 
 it('should have fingerprints & baseline', async () => {
   const validator = createDefaultMonokleValidator();

diff --git a/packages/validation/src/__tests__/sarif/suppression.test.ts b/packages/validation/src/__tests__/sarif/suppression.test.ts
@@ -2,7 +2,8 @@ import 'isomorphic-fetch';
 import {expect, it} from 'vitest';
 import {createDefaultMonokleValidator} from '../../index.js';
 
-import {PRACTICES_ALL_DISABLED, extractK8sResources, readDirectory} from '../testUtils.js';
+import {extractK8sResources} from '@monokle/parser';
+import {PRACTICES_ALL_DISABLED, readDirectory} from '../testUtils.js';
 import {FakeSuppressor} from '../../sarif/suppressions/plugins/FakeSuppressor.js';
 import YAML from 'yaml';
 import {set} from 'lodash';

diff --git a/packages/validation/src/__tests__/testUtils.ts b/packages/validation/src/__tests__/testUtils.ts
@@ -1,188 +1,19 @@
-import {v5} from 'uuid';
-import {parse} from 'path';
+
 import glob from 'tiny-glob';
 import {readFile as readFileFromFs} from 'fs/promises';
 import chunkArray from 'lodash/chunk.js';
-import {LineCounter, parseAllDocuments, parseDocument} from 'yaml';
-import {Resource, ValidationResult} from '../index.js';
+import {ValidationResult} from '../index.js';
 import {expect} from 'vitest';
+import type {BaseFile} from '@monokle/parser';
 
-export const KUSTOMIZATION_KIND = 'Kustomization';
-export const KUSTOMIZATION_API_GROUP = 'kustomize.config.k8s.io';
-
-/**
- * This is all copied from cli/src/utils - need to be moved to their own shared package
- * so they can be reused across both cli and validator - see https://github.com/kubeshop/monokle-core/issues/63
- */
-
-export type File = {
-  id: string;
-  path: string;
-  content: string;
-};
-
-export function extractK8sResources(files: File[]): Resource[] {
-  const resources: Resource[] = [];
-
-  for (const file of files) {
-    const lineCounter = new LineCounter();
-    const documents = parseAllYamlDocuments(file.content, lineCounter);
-
-    for (const document of documents) {
-      const content = document.toJS();
-
-      if (document.errors.length) {
-        continue;
-      }
-
-      const rawFileOffset = lineCounter.linePos(document.range[0]).line;
-      const fileOffset = rawFileOffset === 1 ? 0 : rawFileOffset;
-
-      const resourceBase = {
-        apiVersion: content.apiVersion,
-        kind: content.kind,
-        content,
-        fileId: file.id,
-        filePath: file.path,
-        fileOffset,
-        text: document.toString({directives: false}),
-      };
-
-      if (isKubernetesLike(content)) {
-        const name = createResourceName(file.path, content, content.kind);
-        const id = createResourceId(file.id, content.kind, name, content.metadata?.namespace);
-        const namespace = extractNamespace(content);
-        const resource = {
-          ...resourceBase,
-          id,
-          name,
-          namespace,
-        };
-
-        resources.push(resource);
-      } else if (content && isUntypedKustomizationFile(file.path) && documents.length === 1) {
-        const name = createResourceName(file.path, content, KUSTOMIZATION_KIND);
-        const id = createResourceId(file.id, name, KUSTOMIZATION_KIND);
-        const resource = {
-          ...resourceBase,
-          kind: KUSTOMIZATION_KIND,
-          apiVersion: KUSTOMIZATION_API_GROUP,
-          id,
-          name,
-        };
-
-        resources.push(resource);
-      } else if (content && typeof content.kind === 'string') {
-        // Load K8s schemas only with no apiVersion present for testing purposes.
-        const name = createResourceName(file.path, content, content.kind);
-        const id = createResourceId(file.id, content.kind, name, content.metadata?.namespace);
-        const namespace = extractNamespace(content);
-        const resource = {
-          ...resourceBase,
-          id,
-          name,
-          namespace,
-        };
-
-        resources.push(resource);
-      }
-    }
-  }
-
-  return resources;
-}
-
-type KubernetesLike = {
-  apiVersion: string;
-  kind: string;
-  metadata?: {
-    name: string;
-    namespace?: string;
-  };
-  spec?: {
-    names?: {
-      kind?: string;
-    };
-  };
-};
-
-function isKubernetesLike(content: any): content is KubernetesLike {
-  return content && typeof content.apiVersion === 'string' && typeof content.kind === 'string';
-}
-
-// some (older) kustomization yamls don't contain kind/group properties to identify them as such
-// they are identified only by their name
-function isUntypedKustomizationFile(filePath = ''): boolean {
-  return /kustomization*.yaml/.test(filePath.toLowerCase().trim());
-}
-
-export function isYamlFile(file: File): boolean {
-  return file.path.endsWith('.yml') || file.path.endsWith('.yaml');
-}
-export function parseYamlDocument(text: string, lineCounter?: LineCounter) {
-  return parseDocument(text, {lineCounter, uniqueKeys: false, strict: false});
-}
-
-/**
- * Wrapper that ensures consistent options
- */
-
-export function parseAllYamlDocuments(text: string, lineCounter?: LineCounter) {
-  return parseAllDocuments(text, {
-    lineCounter,
-    uniqueKeys: false,
-    strict: false,
-  });
-}
-
-function extractNamespace(content: any) {
-  // namespace could be an object if it's a helm template value...
-  return content.metadata?.namespace && typeof content.metadata.namespace === 'string'
-    ? content.metadata.namespace
-    : undefined;
-}
-
-const RESOURCE_UUID_NAMESPACE = '6fa71997-8aa8-4b89-b987-cec4fd3de770';
-
-export const createResourceId = (fileId: string, name: string, kind: string, namespace?: string | null): string => {
-  return v5(`${fileId}${kind}${name}${namespace || ''}`, RESOURCE_UUID_NAMESPACE);
-};
-
-export function createResourceName(filePath: string, content: any, kind: string): string {
-  const parsedPath = parse(filePath);
-
-  // dirname for kustomizations
-  // if (
-  //   kind === KUSTOMIZATION_KIND &&
-  //   (!content?.apiVersion ||
-  //     content.apiVersion.startsWith(KUSTOMIZATION_API_GROUP))
-  // ) {
-  //   return parsedPath.dir.replace(/^\/*/, '');
-  // }
-
-  try {
-    //  metadata name
-    return typeof content.metadata.name === 'string'
-      ? content.metadata.name.trim()
-      : JSON.stringify(content.metadata.name).trim();
-  } catch (error) {
-    // filename
-    return parsedPath.name;
-  }
-}
-
-export function getResourcesForPath(filePath: string, resources: Resource[] | undefined) {
-  return resources ? resources.filter(resource => resource.filePath === filePath) : [];
-}
-
-export async function readDirectory(directoryPath: string): Promise<File[]> {
+export async function readDirectory(directoryPath: string): Promise<BaseFile[]> {
   const filePaths = await glob(`${directoryPath}/**/*.{yaml,yml}`);
-  const files: File[] = [];
+  const files: BaseFile[] = [];
 
   for (const chunk of chunkArray(filePaths, 5)) {
     const promise = await Promise.allSettled(
       chunk.map(path => {
-        return readFileFromFs(path, 'utf8').then((content): File => ({id: path, path, content}));
+        return readFileFromFs(path, 'utf8').then((content): BaseFile => ({id: path, path, content}));
       })
     );