cveUtils 1.2.0 and updated update.yml

.github/workflows/dist/src/commands/GenericCommand.d.ts

@@ -7,15 +7,23 @@ export declare abstract class GenericCommand {
     _name: string;
     /** the Command object from the commander library */
     _program: Command;
+    /** ----- cveUtils version string ----- ----- */
+    /** The version string is taken from the version string in package.json to promote a consistent
+     *  location for setting cveUtils metadata.  It is purposely set in "code" instead of in `.env`
+     *  because it should be "baked in" to the code instead of potentially changeable at runtime.
+     *  This way, if there is a problem in CVEProject/cvelistV5, the output in github actions will
+     *  reflect the actual version of this app, and it will
+     *  simplify figuring out what the exact code looked like based on the tag.
+     *
+     */
+    static __utilVersionString: string;
+    static getUtilityVersion(): string;
+    private static setUtilityVersion;
     /** constructor
      * @param name the command name
      * @param program the Command object (from main.ts)
      */
     constructor(name: string, program: Command);
-    /** ----- version string ----- ----- */
-    static __versionString: string;
-    static getUtilityVersion(): string;
-    static setUtilityVersion(versionString: string): string;
     _startTimestamp: number;
     /** resets the command timer */
     timerReset(): number;

.github/workflows/dist/src/commands/MainCommands.d.ts

@@ -1,6 +1,6 @@
 /** object that encapsulates all tested and available cli commands */
 export declare class MainCommands {
     protected _program: any;
-    constructor(version: string);
+    constructor();
     run(): Promise<void>;
 }

.github/workflows/dist/src/commands/UpdateCommand.d.ts

@@ -4,6 +4,8 @@ import { GenericCommand } from './GenericCommand.js';
 export declare class UpdateCommand extends GenericCommand {
     /** default number of minutes to look back when a start date is not specified */
     static defaultMins: number;
+    /** Max file size is used to prevent git commit errors. Currently restricted to 100MB. **/
+    static readonly MAX_FILE_SIZE: number;
     constructor(program: Command);
     /** determines the time options (start, stop, minutesAgo) behavior */
     static determineQueryTimeOptions(options: any, now: string): any;

.github/workflows/dist/src/common/IsoDateString.d.ts

@@ -6,9 +6,16 @@
  *  Note that in the future, if necessary, we can extend what this class covers, but for now
  *    this strict and opinionated set is very useful for processing ISO Date+Time+TZ strings
  */
+/** a regular expression to represent an ISO Date+Time+TZ string
+ *  taken from https://stackoverflow.com/a/3143231/1274852
+ *  works for cases used in CVE representations
+ */
 export declare const IsoDateStringRegEx: RegExp;
 export declare const GregorianLeapDateRegEx: RegExp;
 export declare class IsoDateString {
+    static _minute: number;
+    static _hour: number;
+    static _day: number;
     _isoDateString: string;
     _date: Date;
     /** returns a IsoDateString object iff isoDateStr is a properly formatted ISO Date+Time+TZ string,

.github/workflows/dist/src/core/CveId.d.ts

@@ -12,6 +12,11 @@ export declare type CveIdComponents = [
     string | undefined
 ];
 export declare class CveId {
+    /** kFirstYear: The first year CVE IDs started to be assigned.*/
+    static readonly kFirstYear: number;
+    /** kTestYear: An arbitrary year, that does not overlap with a valid CVE ID year, used for development and testing. */
+    static readonly kTestYear: number;
+    private static _years;
     /** internal representation of the CVE ID */
     id: string;
     /**
@@ -34,10 +39,9 @@ export declare class CveId {
      */
     getFullCvePath(): string;
     /**
-     * returns the official CVEProject/cvelistV5 URL to this CVE ID
+     * returns the raw github URL to this CVE ID
      */
     getRawGithubUrl(): string;
-    private static _years;
     /**
      * checks if a string is a valid CveID
      *  @param id a string to test for CveID validity
@@ -57,10 +61,10 @@ export declare class CveId {
      *  @returns true iff str is a valid CveID
      */
     static isValidCveId(id: string): boolean;
-    /** returns an array of CVE years represented as numbers [1999...2025]
-     *  the algorithm takes the current year from the current (local) time,
-     *    then adds 2 more years to end to accommodate future CVEs,
-     *    and adds 1970 in front
+    /** returns an array of CVE years represented as numbers (e.g. [1970,1999..2025])
+     *  the algorithm builds the valid years from 1999 to the environment variable CVES_MAX_ALLOWABLE_CVE_YEAR
+     *  (or if the environment variable is not present, current year + 2)
+     *  and adds 1970 in front for test CVEs
      */
     static getAllYears(): ReadonlyArray<number>;
     /** given a cveId, returns the git hub repository partial directory it should go into
@@ -73,4 +77,12 @@ export declare class CveId {
      *  @returns string representing the partial path the cve belongs in (e.g., /1999/1xxx/CVE-1999-0001)
      */
     static toCvePath(cveId: string | CveId): string;
+    /** comparator for use with sort() to sort CVE IDs "numerically" by year and then by the ID so that
+     *  CVE-1999-2001 comes before CVE-1999-10001 and CVE-2000-110022
+     *  This is needed because different systems (e.g., fs when sorting local directories, and opensearch sorted responses)
+     *  sort strings differently.  This standardizes on a single sorting algorithm
+     *  @param a string representing a CVE ID
+     *  @param b string representing another CVE ID
+     */
+    static comparator(a: string, b: string): number;
 }

.github/workflows/dist/src/core/CveRecord.d.ts

@@ -1,5 +1,6 @@
 import { CveId } from './CveId.js';
 import { CveRecordV5, CveMetadata, Containers } from '../generated/quicktools/CveRecordV5.js';
+import { CveSignature } from './CveSignature.js';
 export { CveId, CveIdError } from './CveId.js';
 export interface WriteFileOptions {
     prettyprint?: boolean;
@@ -45,6 +46,8 @@ export declare class CveRecord implements CveRecordV5 {
      *  @returns a JSON string
      */
     toJsonString(prettyPrint?: boolean): string;
+    /** generates a CveSignature */
+    static toCveSignature(filepath: any, sha256?: boolean): CveSignature;
     /** writes a CVE Record to a file in CVE JSON 5.0 format
      *  @param relFilepath relative path to the file
      *  @param prettyprint boolean to set whether to pretty print the output

.github/workflows/dist/src/core/Delta.d.ts

@@ -3,10 +3,6 @@
  *  Changes can be a new added file, updated file, or deleted file (though currently, we do not work with deleted
  *  files since no CVEs should ever be deleted once it is published).
  *
- *  Note that this class REQUIRES git and a git history.  It does not look at files, only git commits in git history.
- *  So during testing, simply copying /cves from another directory WILL NOT WORK because git history
- *  does not have those commits.
- *
  *  When making zip files, this class copies CVE JSON files from /cves to a directory, and zip that, so the /cves directory
  *  needs to be in the current directory
  */
@@ -45,6 +41,11 @@ export declare class DeltaOutpuItem {
     };
 }
 export declare class Delta {
+    /** max message length for github commit messages
+     *  needed to truncate commit message when too many CVEs
+     *  were changed
+     */
+    static kMaxGithubCommitMessageLength: number;
     fetchTime?: string;
     numberOfChanges: number;
     new: CveCorePlus[];
@@ -55,12 +56,7 @@ export declare class Delta {
      *                   deltas to the privous ones (default is none)
      */
     constructor(prevDelta?: Partial<Delta>);
-    /**
-     * Factory that generates a new Delta from git log based on a time window
-     * @param start git log start time window
-     * @param stop git log stop time window (defaults to now)
-     */
-    static newDeltaFromGitHistory(start: string, stop?: string, repository?: string): Promise<Delta>;
+    static fromDeltaFile(relFilepath: string): Delta | undefined;
     /**
      * updates data in new and updated lists using CVE ID
      */
@@ -72,11 +68,6 @@ export declare class Delta {
      *  @todo should be in a separate CveId or CveRecord class
      */
     static getCveIdMetaData(path: string): [string | undefined, string | undefined];
-    /** calculates the delta filtering using the specified directory
-     *  @param prevDelta the previous delta
-     *  @param dir directory to filter (note that this cannot have `./` or `../` since this is only doing a simple string match)
-     */
-    static calculateDelta(prevDelta: Partial<Delta>, dir: string): Promise<Delta>;
     /**
      * pure function:  given origQueue, this will either add cve if it is not already in origQueue
      * or replace the original in origQueue with cve
@@ -96,17 +87,10 @@ export declare class Delta {
      *  @param queue the DeltaQueue enum specifying which queue to add to
      */
     add(cve: CveCorePlus, queue: DeltaQueue): void;
-    /** summarize the information in this Delta object in human-readable form */
-    toText(): string;
-    /** writes the delta to a JSON file
-     *  @param relFilepath relative path from current directory
-    */
-    writeFile(relFilepath?: string): void;
     /**
-     * Copies delta CVEs to a specified directory, and optionally zip the resulting directory
-     * @param relDir optional relative path from current directory to write the delta CVEs, default is `deltas` directory
-     * @param zipFile optional relative path from the current directory to write the zip file, default is NOT to write to zip
+     * returns all CVEs from new and updated queues
      */
-    writeCves(relDir?: string | undefined, zipFile?: string | undefined): void;
-    writeTextFile(relFilepath?: string): void;
+    getAllUniqueNewAndUpdatedCves(): Array<CveCorePlus>;
+    /** summarize the information in this Delta object in human-readable form */
+    toText(): string;
 }

.github/workflows/dist/src/core/Delta.test.d.ts

@@ -1 +1,2 @@
-export {};
+export declare const setup_TestGitRepository: () => Promise<void>;
+export declare const cleanup_TestGitRepository: () => Promise<void>;

.github/workflows/dist/src/core/DeltaLog.d.ts

@@ -2,12 +2,16 @@
  *  DeltaLog - log of current and recent historical deltas
  *  Intent is to log all deltas from the current delta to recent historical deltas,
  *  so key information is stored, and other systems using deltas as polling integration points
- *  can poll at almost arbitrary frequency
+ *  can poll at any frequency less than the period
+ *  defined in `.env`'s `CVES_DEFAULT_DELTA_LOG_HISTORY_IN_DAYS` environment variable
+ *  (30 days is current default)
  *
- *  The deltas in the DeltaLog is intended to provide most of the useful information
- *  about a CVE, so that
- *    1. the data can be used as a filter
- *    2. minimize REST calls to CVE REST Services
+ *  The deltas in the DeltaLog is intended to provide just sufficient information
+ *  to decide if a downstream app should be updated:
+ *    1. CVEs that were added
+ *    2. CVEs there were updated
+ *    3. URLs to GitHub and CVE REST Services to retrieve full CVE details
+ *    4. timestamp when the CVEs were committed to the repository
  */
 import { Delta } from './Delta.js';
 import { IsoDateString } from '../common/IsoDateString.js';
@@ -23,6 +27,10 @@ export declare class DeltaLog extends Array<Delta> {
      *
     */
     static fromLogFile(relFilepath?: string, pruneOlderThan?: IsoDateString): DeltaLog;
+    /** prunes and returns a NEW delta log with specified start and stop fetchTimes
+     *
+     */
+    static pruneByFetchTime(origLog: DeltaLog, startDate: IsoDateString | string, stopDate?: IsoDateString | string): DeltaLog;
     /**
      * prepends a delta to log
      * @param delta the Delta object to prepend
@@ -34,10 +42,23 @@ export declare class DeltaLog extends Array<Delta> {
      *            - "latestLast" - chronological order
     */
     sortByFetchTme(direction?: "latestFirst" | "latestLast"): DeltaLog;
+    /**
+     * Creates a single Delta object that contains all of the CVEs in each queue as if
+     * all the operations within the time window had happened as a single event
+     * Note that if a CVE was published and then subsequently updated, that CVE
+     *  will show up in both the new and updated queues.  If you want all CVEs
+     *  from both new and updated queues, run getAllUniqueNewAndUpdatedCves() on the returned Delta object
+     *
+     * @param startWindow IsoDateString for start of time window
+     * @param stopWindow  optional IsoDateString for stop of time window
+     * @returns a single Delta object with all of the consolidated data from all the Deltas in the time window
+     */
+    consolidateDeltas(startWindow: IsoDateString, stopWindow?: IsoDateString): Delta;
     /** writes deltas to a file
      *  @param relFilepath optional relative or full filepath
      *  @returns true iff the file was written (which only happens when
      *    there the [0] delta has changes)
       */
     writeFile(relFilepath?: string): boolean;
+    static fitDeltaLogToFileSize(relFilePath: string, fileSizeLimitBytes: number): boolean;
 }

.github/workflows/dist/src/core/fsUtils.d.ts

@@ -24,14 +24,18 @@ export declare class FsUtils {
      */
     static isSameContent(path1: string, path2: string, ignoreJsonProps?: string[]): boolean;
     /**
-     * Synchronously generate a zip file from an array of files (no directories)
-     * @param filepaths array of filenames to be zipped
-     * @param resultFilepath filepath for resulting zip file
-     * @param zipVirtualDir dir name in zip, defaults to `files`
-     *                      (for example, if you want to add all the files
-     *                       into a zip folder called abc,
-     *                        you would pass 'abc' here)
-     * @param dir path to directory where files are located
+     * Condense json data file.
+     * NOTE: Will overwrite the given file!
+     * condenseLevels:
+     *  0: pretty indent with 2 space
+     *  1: pretty indent with 1 space
+     *  2: strip leading whitespace from pretty file
+     *  3: minified / no whitespace
+     *
+     * @param condenseLevel level to condense to.
+     * @param filePath file with json data to be changed.
+     * @returns the new file size in bytes.
+     * @throws Error if invalid condenseLevel or invalid filePath argument.
      */
-    static generateZipfile(filepaths: string | string[], resultFilepath: string, zipVirtualDir?: string, dir?: string): void;
+    static condenseJsonDataFile(condenseLevel: number, filePath: string): number;
 }

.github/workflows/dist/src/core/git.d.ts

@@ -1,4 +1,15 @@
-/** a wrapper/fascade class to make it easier to use git libraries from within cve utils */
+/** a wrapper/fascade class to make it easier to use git libraries from within cve utils
+ *  Note that because the git utility (and thus this class and the SimpleGit library this class
+ *  depends on) is meant to be used by one process at a time in each "clone" (i.e., each directory
+ *  that contains a `.git` subdirectory), there are operations that is not easily used or tested
+ *  in an asynchronous environment (e.g., cveUtils and jest tests).
+ *
+ *  Specifically, the methods `status()`, `add()`, and "rm()" can have non-deterministric behavior
+ *  when used asynchronously in multiple places.
+ *
+ *  To successfully test these methods, follow the style/pattern of testing described in cveUtil's
+ *  GitLab Issue 7.
+*/
 import { CommitResult, Response, SimpleGit, StatusResult } from 'simple-git';
 import { Delta } from './Delta.js';
 export { StatusResult, Response };
@@ -10,20 +21,42 @@ export declare class Git {
      */
     constructor(init?: Partial<Git>);
     /** returns git status in a promise
+     *  Note that this operation may not be deterministic if, for example, the `rm` method is called
+     *  asynchronously elsewhere in the app.  See the note for this class above for more details.
+     *
      *  Note that while StatusResult shows files with paths relative to pwd, working
      *  with those files (for example, add or rm) requires a full path
     */
     status(): Promise<StatusResult>;
     static genericCallback(err: any): void;
-    /** git add files
-     *  Note that fullPathFiles must be either full path specs or partial paths from this.localDir
-     *  Note that fullPathFiles should NOT be a directory
+    /** calculates the delta filtering using the specified directory
+     *  @param prevDelta the previous delta
+     *  @param dir directory to filter (note that this cannot have `./` or `../` since this is only doing a simple string match)
+     */
+    static calculateDelta(prevDelta: Partial<Delta>, dir: string): Promise<Delta>;
+    /**
+     * Factory that generates a new Delta from git log based on a time window
+     * @param start git log start time window
+     * @param stop git log stop time window (defaults to now)
+     */
+    static newDeltaFromGitHistory(start: string, stop?: string, repository?: string): Promise<Delta>;
+    /** git add files to git stage
+     *  Note that this operation may not be deterministic if, for example, the `rm` method is called
+     *  asynchronously elsewhere in the app.  See the note for this class above for more details.
      *
-    */
+     *  @param fullPathFiles a single file or array of files to be added to stage
+     *    Note that fullPathFiles must be either full path specs or partial paths from this.localDir
+     *    Note that fullPathFiles should NOT be a directory
+     *
+     */
     add(fullPathFiles: string | string[]): Promise<Response<string>>;
-    /** git rm files
-     *  Note that fullPathFiles must be either full path specs or partial paths from this.localDir
-     *  Note that fullPathFiles should NOT be a directory
+    /** git rm files from git stage
+     *  Note that this operation may not be deterministic if, for example, the `rm` method is called
+     *  asynchronously elsewhere in the app.  See the note for this class above for more details.
+     *
+     *  @param fullPathFiles a single file or array of files to be added to stage
+     *    Note that fullPathFiles must be either full path specs or partial paths from this.localDir
+     *    Note that fullPathFiles should NOT be a directory
     */
     rm(fullPathFiles: string | string[]): Promise<Response<void>>;
     /**

.github/workflows/dist/src/core/git.test.d.ts

@@ -1 +1 @@
-export {};
+export declare const GitTestsUsingGit: () => void;