From e29876e7f6d265136d9e8871ace6b1235e95a008 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Fri, 31 May 2024 13:57:02 -0400 Subject: [PATCH 1/8] debugging --- website/src/repl/idbutils.mjs | 47 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index 959dc7d1e..8a95493bc 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -12,7 +12,7 @@ export const userSamplesDBConfig = { }; // deletes all of the databases, useful for debugging -const clearIDB = () => { +function clearIDB() { window.indexedDB .databases() .then((r) => { @@ -21,12 +21,30 @@ const clearIDB = () => { .then(() => { alert('All data cleared.'); }); -}; +} // queries the DB, and registers the sounds so they can be played -export const registerSamplesFromDB = (config = userSamplesDBConfig, onComplete = () => {}) => { +export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = () => {}) { openDB(config, (objectStore) => { + // const keys = objectStore.getAllKeys(); + + // keys.onsuccess = (e) => { + // console.log(e); + // const result = e.target.result[0]; + // console.log(result); + + // const q = objectStore.get(result); + // q.onerror = (e) => console.error(e.target.error); + // q.onsuccess = (e) => console.log(e); + // }; + let query = objectStore.getAll(); + query.onerror = (e) => { + logger('User Samples failed to load ', 'error'); + onComplete(); + console.error(e?.target?.error); + }; + query.onsuccess = (event) => { const soundFiles = event.target.result; if (!soundFiles?.length) { @@ -64,7 +82,7 @@ export const registerSamplesFromDB = (config = userSamplesDBConfig, onComplete = onComplete(); }; }); -}; +} // creates a blob from a buffer that can be read async function bufferToDataUrl(buf) { return new Promise((resolve) => { @@ -77,7 +95,7 @@ async function bufferToDataUrl(buf) { }); } //open db and initialize it if necessary -const openDB = (config, onOpened) => { +function openDB(config, onOpened) { const { dbName, version, table, columns } = config; if (typeof window === 'undefined') { return; @@ -102,16 +120,15 @@ const openDB = (config, onOpened) => { dbOpen.onsuccess = () => { const db = dbOpen.result; - - const // lock store for writing - writeTransaction = db.transaction([table], 'readwrite'), - // get object store - objectStore = writeTransaction.objectStore(table); + // lock store for writing + const writeTransaction = db.transaction([table], 'readwrite'); + // get object store + const objectStore = writeTransaction.objectStore(table); onOpened(objectStore, db); }; -}; +} -const processFilesForIDB = async (files) => { +async function processFilesForIDB(files) { return await Promise.all( Array.from(files) .map(async (s) => { @@ -139,9 +156,9 @@ const processFilesForIDB = async (files) => { logger('Something went wrong while processing uploaded files', 'error'); console.error(error); }); -}; +} -export const uploadSamplesToDB = async (config, files) => { +export async function uploadSamplesToDB(config, files) { await processFilesForIDB(files).then((files) => { const onOpened = (objectStore, _db) => { files.forEach((file) => { @@ -153,4 +170,4 @@ export const uploadSamplesToDB = async (config, files) => { }; openDB(config, onOpened); }); -}; +} From 1fca6f25b8cb7d84bf643d13634f43dddb2e9ef4 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 11:35:24 -0400 Subject: [PATCH 2/8] faster upload --- website/src/repl/files.mjs | 3 +- website/src/repl/idbutils.mjs | 78 +++++++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/website/src/repl/files.mjs b/website/src/repl/files.mjs index 63ac5f853..3e3db94d8 100644 --- a/website/src/repl/files.mjs +++ b/website/src/repl/files.mjs @@ -75,7 +75,8 @@ export const walkFileTree = (node, fn) => { } }; -export const isAudioFile = (filename) => ['wav', 'mp3'].includes(filename.split('.').slice(-1)[0]); +export const isAudioFile = (filename) => + ['wav', 'mp3', 'flac', 'ogg', 'm4a'].includes(filename.split('.').slice(-1)[0]); function uint8ArrayToDataURL(uint8Array) { const blob = new Blob([uint8Array], { type: 'audio/*' }); diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index 8a95493bc..54f982163 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -50,32 +50,39 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = if (!soundFiles?.length) { return; } + console.log(soundFiles); const sounds = new Map(); [...soundFiles] .sort((a, b) => a.title.localeCompare(b.title, undefined, { numeric: true, sensitivity: 'base' })) - .forEach((soundFile) => { + .forEach((soundFile, i) => { const title = soundFile.title; if (!isAudioFile(title)) { return; } const splitRelativePath = soundFile.id.split('/'); - const parentDirectory = + let parentDirectory = //fallback to file name before period and seperator if no parent directory splitRelativePath[splitRelativePath.length - 2] ?? soundFile.id.split(/\W+/)[0] ?? 'user'; - const soundPath = soundFile.blob; + + const soundPath = bufferToDataUrl(soundFile.buf); + + // const soundPath = soundFile.blob; const soundPaths = sounds.get(parentDirectory) ?? new Set(); soundPaths.add(soundPath); sounds.set(parentDirectory, soundPaths); }); sounds.forEach((soundPaths, key) => { - const value = Array.from(soundPaths); - registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { - type: 'sample', - samples: value, - baseUrl: undefined, - prebake: false, - tag: undefined, + const paths = Array.from(soundPaths); + Promise.all(paths).then((value) => { + // console.log({ value }); + registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { + type: 'sample', + samples: value, + baseUrl: undefined, + prebake: false, + tag: undefined, + }); }); }); logger('imported sounds registered!', 'success'); @@ -94,6 +101,21 @@ async function bufferToDataUrl(buf) { reader.readAsDataURL(blob); }); } + +function bufferToBlob(buf) { + return new Blob([buf], { type: 'application/octet-binary' }); +} + +async function blobToDataUrl(blob) { + return new Promise((resolve) => { + var reader = new FileReader(); + reader.onload = function (event) { + resolve(event.target.result); + }; + reader.readAsDataURL(blob); + }); +} + //open db and initialize it if necessary function openDB(config, onOpened) { const { dbName, version, table, columns } = config; @@ -126,12 +148,13 @@ function openDB(config, onOpened) { const objectStore = writeTransaction.objectStore(table); onOpened(objectStore, db); }; + return dbOpen; } async function processFilesForIDB(files) { - return await Promise.all( + return Promise.all( Array.from(files) - .map(async (s) => { + .map((s) => { const title = s.name; if (!isAudioFile(title)) { @@ -140,16 +163,24 @@ async function processFilesForIDB(files) { //create obscured url to file system that can be fetched const sUrl = URL.createObjectURL(s); //fetch the sound and turn it into a buffer array - const buf = await fetch(sUrl).then((res) => res.arrayBuffer()); + return fetch(sUrl).then((res) => { + return res.arrayBuffer().then((buf) => { + const path = s.webkitRelativePath; + let id = path?.length ? path : title; + if (id == null || title == null || buf == null) { + return; + } + return { + title, + buf, + // blob: base64, + id, + }; + }); + }); + //create a url base64 containing all of the buffer data - const base64 = await bufferToDataUrl(buf); - const path = s.webkitRelativePath; - const id = path?.length ? path : title; - return { - title, - blob: base64, - id, - }; + // const base64 = await bufferToDataUrl(buf); }) .filter(Boolean), ).catch((error) => { @@ -159,14 +190,19 @@ async function processFilesForIDB(files) { } export async function uploadSamplesToDB(config, files) { + logger('procesing user samples...'); await processFilesForIDB(files).then((files) => { + console.log(files); + logger('user samples processed... opening db'); const onOpened = (objectStore, _db) => { + logger('index db opened... writing files to db'); files.forEach((file) => { if (file == null) { return; } objectStore.put(file); }); + logger('user samples written successfully'); }; openDB(config, onOpened); }); From 1fd9ba5dbfad00c0986332e958be7b27f872c532 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 12:11:28 -0400 Subject: [PATCH 3/8] convert to blob --- website/src/repl/idbutils.mjs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index 54f982163..a6b191798 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -64,7 +64,7 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = //fallback to file name before period and seperator if no parent directory splitRelativePath[splitRelativePath.length - 2] ?? soundFile.id.split(/\W+/)[0] ?? 'user'; - const soundPath = bufferToDataUrl(soundFile.buf); + const soundPath = blobToDataUrl(soundFile.blob); // const soundPath = soundFile.blob; const soundPaths = sounds.get(parentDirectory) ?? new Set(); @@ -164,16 +164,16 @@ async function processFilesForIDB(files) { const sUrl = URL.createObjectURL(s); //fetch the sound and turn it into a buffer array return fetch(sUrl).then((res) => { - return res.arrayBuffer().then((buf) => { + return res.blob().then((blob) => { const path = s.webkitRelativePath; let id = path?.length ? path : title; - if (id == null || title == null || buf == null) { + if (id == null || title == null || blob == null) { return; } return { title, - buf, - // blob: base64, + // buf, + blob, id, }; }); From bf7fe2bf755c45b220c95df97ba10e6af3e6b608 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 12:39:45 -0400 Subject: [PATCH 4/8] fixed indexdb --- packages/superdough/superdough.mjs | 2 +- website/src/repl/idbutils.mjs | 90 +++++++++++------------------- 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/packages/superdough/superdough.mjs b/packages/superdough/superdough.mjs index 018caa9a3..e663fc6df 100644 --- a/packages/superdough/superdough.mjs +++ b/packages/superdough/superdough.mjs @@ -99,7 +99,7 @@ export const connectToDestination = (input, channels = [0, 1]) => { //This upmix can be removed if correct channel counts are set throughout the app, // and then strudel could theoretically support surround sound audio files const stereoMix = new StereoPannerNode(ctx); - input.connect(stereoMix); + input?.connect(stereoMix); const splitter = new ChannelSplitterNode(ctx, { numberOfOutputs: stereoMix.channelCount, diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index a6b191798..1c46c5d7a 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -26,19 +26,7 @@ function clearIDB() { // queries the DB, and registers the sounds so they can be played export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = () => {}) { openDB(config, (objectStore) => { - // const keys = objectStore.getAllKeys(); - - // keys.onsuccess = (e) => { - // console.log(e); - // const result = e.target.result[0]; - // console.log(result); - - // const q = objectStore.get(result); - // q.onerror = (e) => console.error(e.target.error); - // q.onsuccess = (e) => console.log(e); - // }; - - let query = objectStore.getAll(); + const query = objectStore.getAll(); query.onerror = (e) => { logger('User Samples failed to load ', 'error'); onComplete(); @@ -50,32 +38,36 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = if (!soundFiles?.length) { return; } - console.log(soundFiles); const sounds = new Map(); - [...soundFiles] - .sort((a, b) => a.title.localeCompare(b.title, undefined, { numeric: true, sensitivity: 'base' })) - .forEach((soundFile, i) => { - const title = soundFile.title; - if (!isAudioFile(title)) { - return; - } - const splitRelativePath = soundFile.id.split('/'); - let parentDirectory = - //fallback to file name before period and seperator if no parent directory - splitRelativePath[splitRelativePath.length - 2] ?? soundFile.id.split(/\W+/)[0] ?? 'user'; - - const soundPath = blobToDataUrl(soundFile.blob); - - // const soundPath = soundFile.blob; - const soundPaths = sounds.get(parentDirectory) ?? new Set(); - soundPaths.add(soundPath); - sounds.set(parentDirectory, soundPaths); - }); - sounds.forEach((soundPaths, key) => { - const paths = Array.from(soundPaths); - Promise.all(paths).then((value) => { - // console.log({ value }); + Promise.all( + [...soundFiles] + .sort((a, b) => a.title.localeCompare(b.title, undefined, { numeric: true, sensitivity: 'base' })) + .map((soundFile, i) => { + const title = soundFile.title; + if (!isAudioFile(title)) { + return; + } + const splitRelativePath = soundFile.id.split('/'); + let parentDirectory = + //fallback to file name before period and seperator if no parent directory + splitRelativePath[splitRelativePath.length - 2] ?? soundFile.id.split(/\W+/)[0] ?? 'user'; + const blob = soundFile.blob; + + // Files used to be uploaded as base64 strings, After Jan 1 2025 this check can be safely deleted + if (typeof blob === 'string') { + return blob; + } + + return blobToDataUrl(blob).then((soundPath) => { + const soundPaths = sounds.get(parentDirectory) ?? new Set(); + soundPaths.add(soundPath); + sounds.set(parentDirectory, soundPaths); + }); + }), + ).then(() => { + sounds.forEach((soundPaths, key) => { + const value = Array.from(soundPaths); registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { type: 'sample', samples: value, @@ -84,27 +76,13 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = tag: undefined, }); }); + + logger('imported sounds registered!', 'success'); + onComplete(); }); - logger('imported sounds registered!', 'success'); - onComplete(); }; }); } -// creates a blob from a buffer that can be read -async function bufferToDataUrl(buf) { - return new Promise((resolve) => { - var blob = new Blob([buf], { type: 'application/octet-binary' }); - var reader = new FileReader(); - reader.onload = function (event) { - resolve(event.target.result); - }; - reader.readAsDataURL(blob); - }); -} - -function bufferToBlob(buf) { - return new Blob([buf], { type: 'application/octet-binary' }); -} async function blobToDataUrl(blob) { return new Promise((resolve) => { @@ -172,15 +150,11 @@ async function processFilesForIDB(files) { } return { title, - // buf, blob, id, }; }); }); - - //create a url base64 containing all of the buffer data - // const base64 = await bufferToDataUrl(buf); }) .filter(Boolean), ).catch((error) => { From 321a88892100fa4ea38e50f04f047cd5babdd54f Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 12:42:07 -0400 Subject: [PATCH 5/8] add catch --- website/src/repl/idbutils.mjs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index 1c46c5d7a..c49362da9 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -65,21 +65,26 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = sounds.set(parentDirectory, soundPaths); }); }), - ).then(() => { - sounds.forEach((soundPaths, key) => { - const value = Array.from(soundPaths); - registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { - type: 'sample', - samples: value, - baseUrl: undefined, - prebake: false, - tag: undefined, + ) + .then(() => { + sounds.forEach((soundPaths, key) => { + const value = Array.from(soundPaths); + registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { + type: 'sample', + samples: value, + baseUrl: undefined, + prebake: false, + tag: undefined, + }); }); - }); - logger('imported sounds registered!', 'success'); - onComplete(); - }); + logger('imported sounds registered!', 'success'); + onComplete(); + }) + .catch((error) => { + logger('Something went wrong while registering saved samples from the index db', 'error'); + console.error(error); + }); }; }); } From 6b2d080afcfbcd402bf1f97e11d541e034fb1edc Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 19:17:00 -0400 Subject: [PATCH 6/8] remove unessecary change --- packages/superdough/superdough.mjs | 2 +- website/src/repl/idbutils.mjs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/superdough/superdough.mjs b/packages/superdough/superdough.mjs index e663fc6df..018caa9a3 100644 --- a/packages/superdough/superdough.mjs +++ b/packages/superdough/superdough.mjs @@ -99,7 +99,7 @@ export const connectToDestination = (input, channels = [0, 1]) => { //This upmix can be removed if correct channel counts are set throughout the app, // and then strudel could theoretically support surround sound audio files const stereoMix = new StereoPannerNode(ctx); - input?.connect(stereoMix); + input.connect(stereoMix); const splitter = new ChannelSplitterNode(ctx, { numberOfOutputs: stereoMix.channelCount, diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index c49362da9..f5cbf52bd 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -56,19 +56,24 @@ export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = // Files used to be uploaded as base64 strings, After Jan 1 2025 this check can be safely deleted if (typeof blob === 'string') { - return blob; + const soundPaths = sounds.get(parentDirectory) ?? new Set(); + soundPaths.add(blob); + sounds.set(parentDirectory, soundPaths); + return; } return blobToDataUrl(blob).then((soundPath) => { const soundPaths = sounds.get(parentDirectory) ?? new Set(); soundPaths.add(soundPath); sounds.set(parentDirectory, soundPaths); + return; }); }), ) .then(() => { sounds.forEach((soundPaths, key) => { const value = Array.from(soundPaths); + registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), { type: 'sample', samples: value, From ffa21ddf0df30e9452e5ee894ef1fff9c101e7e1 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sat, 1 Jun 2024 19:19:47 -0400 Subject: [PATCH 7/8] remove console statement --- website/src/repl/idbutils.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/website/src/repl/idbutils.mjs b/website/src/repl/idbutils.mjs index f5cbf52bd..70ccd8f8f 100644 --- a/website/src/repl/idbutils.mjs +++ b/website/src/repl/idbutils.mjs @@ -176,7 +176,6 @@ async function processFilesForIDB(files) { export async function uploadSamplesToDB(config, files) { logger('procesing user samples...'); await processFilesForIDB(files).then((files) => { - console.log(files); logger('user samples processed... opening db'); const onOpened = (objectStore, _db) => { logger('index db opened... writing files to db'); From 8c73ad088a6020603c955e669fe362a3aa77e621 Mon Sep 17 00:00:00 2001 From: "Jade (Rose) Rowland" Date: Sun, 2 Jun 2024 12:32:54 -0400 Subject: [PATCH 8/8] support aac --- website/src/repl/files.mjs | 2 +- website/src/repl/panel/ImportSoundsButton.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/repl/files.mjs b/website/src/repl/files.mjs index 3e3db94d8..d2b95f6ad 100644 --- a/website/src/repl/files.mjs +++ b/website/src/repl/files.mjs @@ -76,7 +76,7 @@ export const walkFileTree = (node, fn) => { }; export const isAudioFile = (filename) => - ['wav', 'mp3', 'flac', 'ogg', 'm4a'].includes(filename.split('.').slice(-1)[0]); + ['wav', 'mp3', 'flac', 'ogg', 'm4a', 'aac'].includes(filename.split('.').slice(-1)[0]); function uint8ArrayToDataURL(uint8Array) { const blob = new Blob([uint8Array], { type: 'audio/*' }); diff --git a/website/src/repl/panel/ImportSoundsButton.jsx b/website/src/repl/panel/ImportSoundsButton.jsx index 7a97b9e09..da45705f4 100644 --- a/website/src/repl/panel/ImportSoundsButton.jsx +++ b/website/src/repl/panel/ImportSoundsButton.jsx @@ -33,7 +33,7 @@ export default function ImportSoundsButton({ onComplete }) { directory="" webkitdirectory="" multiple - accept="audio/*, .wav, .mp3, .m4a, .flac" + accept="audio/*, .wav, .mp3, .m4a, .flac, .aac, .ogg" onChange={() => { onChange(); }}