From 525d428075b783147f95d82ff2f1e50e1a0f36ec Mon Sep 17 00:00:00 2001 From: oshinongit Date: Wed, 1 Nov 2023 16:56:58 +0100 Subject: [PATCH] encore profile generation --- examples/{ => encore}/encoreExample.ts | 34 +++++---- examples/encore/encoreProfile.yml | 17 +++++ examples/encore/pipeline.yml | 12 ++++ src/analysis/brute-force.ts | 1 - src/encoreYamlGenerator.ts | 82 ++++------------------ src/load-pipeline.ts | 17 +++++ src/models/encoreProfileTypes.ts | 14 ---- src/pipelines/encore/encore-pipeline.ts | 93 +++++++------------------ 8 files changed, 108 insertions(+), 162 deletions(-) rename examples/{ => encore}/encoreExample.ts (52%) create mode 100644 examples/encore/encoreProfile.yml create mode 100644 examples/encore/pipeline.yml delete mode 100644 src/models/encoreProfileTypes.ts diff --git a/examples/encoreExample.ts b/examples/encore/encoreExample.ts similarity index 52% rename from examples/encoreExample.ts rename to examples/encore/encoreExample.ts index 23aa4a1..9f54246 100644 --- a/examples/encoreExample.ts +++ b/examples/encore/encoreExample.ts @@ -1,6 +1,7 @@ -import { EncorePipelineConfiguration } from '../src/pipelines/encore/encore-pipeline-configuration'; -import { EncorePipeline } from '../src/pipelines/encore/encore-pipeline'; -import { BitrateResolutionPair } from '../src/models/bitrate-resolution-pair'; +import { EncorePipelineConfiguration } from '../../src/pipelines/encore/encore-pipeline-configuration'; +import { EncorePipeline } from '../../src/pipelines/encore/encore-pipeline'; +import createJob from '../../src/create-job'; +import fs from 'fs'; async function transcodeInputsAndAnalyze() { @@ -18,18 +19,25 @@ async function transcodeInputsAndAnalyze() { encoreInstancePostCreationDelay_ms: 10000 }; - const bitrateResolutionPairs: BitrateResolutionPair[] = [ - {resolution: { width: 1280, height: 720}, bitrate: 600000}, - {resolution: { width: 640, height: 360}, bitrate: 600000}, - {resolution: { width: 768, height: 432, range: { + const inlineProfile = fs.readFileSync('encoreProfile.yml', 'utf8'); + + const resolutions = [{ + width: 1280, + height: 720, + range: { min: 500000, max: 600000 - }}, bitrate: 500000} - ] + } + }]; + + const bitrates = [ + 500000, + 600000, + 800000 + ] const pipeline: EncorePipeline = new EncorePipeline(configuration); - await pipeline.transcode(configuration.inputs[0], { width: 1280, height: 720}, 600000, "output", undefined, bitrateResolutionPairs); - + await pipeline.transcode(configuration.inputs[0], { width: 1280, height: 720}, 600000, "output", undefined, inlineProfile, resolutions, bitrates); } - - transcodeInputsAndAnalyze(); \ No newline at end of file + +transcodeInputsAndAnalyze(); \ No newline at end of file diff --git a/examples/encore/encoreProfile.yml b/examples/encore/encoreProfile.yml new file mode 100644 index 0000000..24f1508 --- /dev/null +++ b/examples/encore/encoreProfile.yml @@ -0,0 +1,17 @@ +name: X264Encode +description: Program profile +scaling: bicubic +encodes: + - type: X264Encode + suffix: _x264_ + twoPass: true + height: + width: + params: + b:v: + maxrate: + minrate: + r: 25 + fps_mode: cfr + pix_fmt: yuv420p + force_key_frames: expr:not(mod(n,96)) \ No newline at end of file diff --git a/examples/encore/pipeline.yml b/examples/encore/pipeline.yml new file mode 100644 index 0000000..e00fd1b --- /dev/null +++ b/examples/encore/pipeline.yml @@ -0,0 +1,12 @@ +encore: + apiAddress: https://api-encore.stage.osaas.io, + token: null, + instanceId: dummy, + profilesUrl: profilesUrl, + outputFolder: /usercontent/demo, + baseName: _demo, + inputs: ['https://testcontent.eyevinn.technology/mp4/stswe-tvplus-promo.mp4'], + duration: 120, + priority: 0, + encorePollingInterval_ms: 30000, + encoreInstancePostCreationDelay_ms: 10000, \ No newline at end of file diff --git a/src/analysis/brute-force.ts b/src/analysis/brute-force.ts index 0365014..5c57215 100644 --- a/src/analysis/brute-force.ts +++ b/src/analysis/brute-force.ts @@ -88,7 +88,6 @@ export default async function analyzeBruteForce(directory: string, reference: st if (options.pipelineVariables) { // Create all combinations of bitrate, resolution, and variables Object.entries(options.pipelineVariables).forEach(([variableName, values]) => { - //console.log(`variableName: ${variableName}`); pairs = pairs.flatMap(pair => values.map( value => { const variables = pair.ffmpegOptionVariables ? {...pair.ffmpegOptionVariables} : {}; diff --git a/src/encoreYamlGenerator.ts b/src/encoreYamlGenerator.ts index 3e3578a..3528703 100644 --- a/src/encoreYamlGenerator.ts +++ b/src/encoreYamlGenerator.ts @@ -1,77 +1,25 @@ import * as fs from 'fs'; -import { EncoreTranscodeType, EncoreProgramProfile } from './models/encoreProfileTypes'; +import YAML from 'yaml'; +import { Resolution } from '../src/models/resolution'; export class EncoreYAMLGenerator { - generateYAML(profile: EncoreProgramProfile): string { - const yamlContent = ` -name: ${profile.name} -description: ${profile.description} -scaling: ${profile.scaling} -encodes: -${profile.encodes - .map((encode) => ` - - type: ${encode.type} - suffix: ${encode.suffix} - twoPass: ${encode.twoPass} - height: ${encode.height} - params:${Object.entries(encode.params) - .map(([key, value]) => ` - ${key}: ${value}`) - .join('')} -`) - .join('')} -`; - return yamlContent; - } - - createEncodeObject( - type: string, - suffix: string, - twoPass: boolean, - height: number, - params: { [key: string]: string } - ): EncoreTranscodeType { - return { - type, - suffix, - twoPass, - height, - params, - }; - } + modifyEncoreProfileAttributes(profileEncodesObject: any, resolution: Resolution, bitRate: number): string { - saveToFile(profile: EncoreProgramProfile, filePath: string) { - const generatedYAML = this.generateYAML(profile); - fs.writeFileSync(filePath, generatedYAML); - } -} + const data = profileEncodesObject; + data.height = resolution.height; + data.width = resolution.width; + data.params['b:v'] = bitRate; + data.params['maxrate'] = resolution.range?.max; + data.params['minrate'] = resolution.range?.min; -// Example usage: -function example() { - const yamlGenerator = new EncoreYAMLGenerator(); + const updatedYAML = YAML.stringify(data); - const encodeObject = yamlGenerator.createEncodeObject('X264Encode', '_x264_3100', true, 1080, { - 'b:v': '3100k', - maxrate: '4700k', - bufsize: '6200k', - r: '25', - fps_mode: 'cfr', - pix_fmt: 'yuv420p', - force_key_frames: 'expr:not(mod(n,96))', - preset: 'medium', - }); - - const encodeObjects: EncoreTranscodeType[] = []; - encodeObjects.push(encodeObject); - - const programProfile: EncoreProgramProfile = { - name: 'encoreProgram', - description: 'Program profile', - scaling: 'bicubic', - encodes: encodeObjects, - }; + return updatedYAML; +} - yamlGenerator.saveToFile(programProfile, 'encoreProfile.yml'); + saveToFile(filePath: string, yaml: string) { + fs.writeFileSync(filePath, yaml); + } } \ No newline at end of file diff --git a/src/load-pipeline.ts b/src/load-pipeline.ts index 45b96b5..5ad7bff 100644 --- a/src/load-pipeline.ts +++ b/src/load-pipeline.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import YAML from 'yaml'; import LocalPipeline from './pipelines/local/local-pipeline'; import AWSPipeline from './pipelines/aws/aws-pipeline'; +import { EncorePipeline } from './pipelines/encore/encore-pipeline'; import { Pipeline } from './pipelines/pipeline'; import logger from './logger'; import { LocalPipelineConfiguration } from './pipelines/local/local-pipeline-configuration'; @@ -30,6 +31,20 @@ export type PipelineProfile = { pythonPath: string; ffmpegEncoder: 'libx264' | 'h264_videotoolbox'; }; + + encore?: { + apiAddress: string; + token: string; + instanceId: string; + profilesUrl: string; + outputFolder: string; + baseName: string; + inputs: Array; + duration: number; + priority: number; + encorePollingInterval_ms: number; + encoreInstancePostCreationDelay_ms: number + } }; /** @@ -53,6 +68,8 @@ async function loadPipeline(pipelineFilename: string, encodingProfile?: string): return new AWSPipeline({ ...pipelineProfile.aws, mediaConvertSettings: encodingProfileData }); } else if (pipelineProfile.local !== undefined) { return new LocalPipeline({ ...pipelineProfile.local, ffmpegOptions: encodingProfileData }); + } else if (pipelineProfile.encore !== undefined) { + return new EncorePipeline(pipelineProfile.encore); } else { throw new Error(`Invalid pipeline: ${JSON.stringify(pipelineProfile)}`); } diff --git a/src/models/encoreProfileTypes.ts b/src/models/encoreProfileTypes.ts deleted file mode 100644 index bf2f798..0000000 --- a/src/models/encoreProfileTypes.ts +++ /dev/null @@ -1,14 +0,0 @@ -export type EncoreTranscodeType = { - type: string; - suffix: string; - twoPass: boolean; - height: number; - params: { [key: string]: string }; - } - -export type EncoreProgramProfile = { - name: string; - description: string; - scaling: string; - encodes: EncoreTranscodeType[]; -} \ No newline at end of file diff --git a/src/pipelines/encore/encore-pipeline.ts b/src/pipelines/encore/encore-pipeline.ts index 78866ac..7e22f57 100644 --- a/src/pipelines/encore/encore-pipeline.ts +++ b/src/pipelines/encore/encore-pipeline.ts @@ -1,16 +1,15 @@ import AWSPipeline from '../aws/aws-pipeline'; import { AWSPipelineConfiguration } from '../aws/aws-pipeline-configuration'; -import { BitrateResolutionPair } from '../../models/bitrate-resolution-pair'; import { EncoreInstance } from '../../models/encoreInstance'; import { EncoreJobs, EncoreJob } from '../../models/encoreJobs'; import { EncorePipelineConfiguration } from './encore-pipeline-configuration'; import { EncoreYAMLGenerator } from '../../encoreYamlGenerator'; -import { EncoreTranscodeType, EncoreProgramProfile } from '../../models/encoreProfileTypes'; import { Pipeline } from '../pipeline'; import { Resolution } from '../../models/resolution'; import { QualityAnalysisModel } from '../../models/quality-analysis-model'; import logger from '../../logger'; import fs from 'fs'; +import YAML from 'yaml'; //const ISOBoxer = require('codem-isoboxer'); const { Readable } = require('stream'); //Typescript import library contains different functions. //If someone knows how to achieve the same functionality in Typescript syntax let me know. @@ -20,72 +19,32 @@ export function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -function createYAML(bitrateResolutions: BitrateResolutionPair[]): string { +function createProfile(inlineProfile: string, resolutions: Resolution[], bitRates: number[]): string { const yamlGenerator = new EncoreYAMLGenerator(); - let transcodeObjects: EncoreTranscodeType[] = []; - - bitrateResolutions.forEach(bitRateResolution => { - - const resolution = bitRateResolution.resolution; - - if (resolution.range && resolution.range.min && resolution.range.max) { - const bitRateRange = (resolution.range.max - resolution.range.min); - if (bitRateRange < 1000) { - throw new Error(`Small bitrate range: ${bitRateRange}, use discrete values instead or increase range`); - } - let bitRateStep: number = resolution.range.min; - let bitrateSteps = [bitRateStep]; - while (bitRateStep < resolution.range.max) { - bitRateStep += 1000; - bitrateSteps.push(bitRateStep); - } - if (bitrateSteps[bitrateSteps.length - 1] > resolution.range.max) { - bitrateSteps[bitrateSteps.length - 1] = resolution.range.max; - } - - bitrateSteps.forEach(bitRate => { - - const transcodeObject = yamlGenerator.createEncodeObject('X264Encode', `_${bitRate / 100}`, true, resolution.height, { - 'b:v': `${bitRate / 100}k`, - maxrate: `${(bitRate / 100) * 1.5}k`, - bufsize: `${(bitRate / 100) * 1.5 * 1.5}k`, - r: '25', - fps_mode: 'cfr', - pix_fmt: 'yuv420p', - force_key_frames: 'expr:not(mod(n,96))', - preset: 'medium', - }); - transcodeObjects.push(transcodeObject); - - }) + const inlineProfileObject = YAML.parse(inlineProfile); + let profiles: string[] = []; + console.log(inlineProfileObject) + console.log(inlineProfileObject.encodes[0]) + + resolutions.forEach(resolution => { + bitRates.forEach(bitRate => { + const encoding = yamlGenerator.modifyEncoreProfileAttributes(inlineProfileObject.encodes[0], resolution, bitRate); + profiles.push(encoding) + }) + }); - } - else if (!resolution.range) { - const encodeObject = yamlGenerator.createEncodeObject('X264Encode', `_${bitRateResolution.bitrate / 100}`, true, bitRateResolution.resolution.height, { - 'b:v': `${bitRateResolution.bitrate / 100}k`, - maxrate: `${(bitRateResolution.bitrate / 100) * 1.5}k`, - bufsize: `${(bitRateResolution.bitrate / 100) * 1.5 * 1.5}k`, - r: '25', - fps_mode: 'cfr', - pix_fmt: 'yuv420p', - force_key_frames: 'expr:not(mod(n,96))', - preset: 'medium', - }); - transcodeObjects.push(encodeObject); - } + let i = 0; + profiles.forEach(encodingProfile => { + inlineProfileObject.encodes[i] = encodingProfile; + i++; }) - const programProfile: EncoreProgramProfile = { - name: 'encoreProgram', - description: 'Program profile', - scaling: 'bicubic', - encodes: transcodeObjects, - }; + console.log(inlineProfileObject); + + yamlGenerator.saveToFile("test_output_profile.yml", YAML.stringify(inlineProfileObject)); - yamlGenerator.saveToFile(programProfile, "profile.yml"); //TODO: Remove, for dev only - const transcodingProfile = yamlGenerator.generateYAML(programProfile); - return transcodingProfile + return "transcodingProfile" } export class EncorePipeline implements Pipeline { @@ -111,10 +70,10 @@ export class EncorePipeline implements Pipeline { this.awsPipe = new AWSPipeline(this.awsConf); } - async transcode(input: string, targetResolution: Resolution, targetBitrate: number, output: string, variables?: Record, bitrateResolutions?: BitrateResolutionPair[]): Promise { + async transcode(input: string, targetResolution: Resolution, targetBitrate: number, output: string, variables?: Record, inlineProfile?: string, resolutions?: Resolution[], bitRates?: number[]): Promise { - if (!bitrateResolutions) { - throw new Error('No resolutionsBitrateRange in encore transcode'); + if (!(inlineProfile && resolutions && bitRates)) { + throw new Error('No inline profile in encore transcode'); } const instance: EncoreInstance | undefined = await this.createEncoreInstance(this.configuration.apiAddress, this.configuration.token, this.configuration.instanceId, this.configuration.profilesUrl); @@ -122,8 +81,8 @@ export class EncorePipeline implements Pipeline { if (!instance) { throw new Error('undefined instance'); } - const transcodingProfile = createYAML(bitrateResolutions); - // await this.runTranscodePollUntilFinished(instance, transcodingProfile); + const transcodingProfile = createProfile(inlineProfile, resolutions, bitRates); + await this.runTranscodePollUntilFinished(instance, transcodingProfile); await this.deleteEncoreInstance(instance, this.configuration.apiAddress); return output }