From 92fd07acfac5127dc48c7fc0d4496224e5cbda4f Mon Sep 17 00:00:00 2001 From: Colin Fruit <17092461+colinfruit@users.noreply.github.com> Date: Tue, 3 Dec 2019 14:17:50 -0500 Subject: [PATCH] feat: import files to MFS (#810) * upload files to MFS * format directory path properly on upload * move open web UI call out of file upload block * reword upload to import in UI messages * move default import directory to preferences screen * Ensure imported files have the same CID when imported from Web UI or ipfs-companion This has been done using adding them to IPFS first (ipfs.add) first and then using the ipfs files copy api (ipfs.files.cp) to move the files into the desired directory. * open resource at gateway if using embedded node * add date symbols to import directory * add import options to import screen * fix: only display openViaWebUI option if opening via web UI is possible * fix: remove wrapping directory from files API import * upload => import in MFS import variables * trim double slashes from import directory * add file import section on options page * move preloadAtPublicGateway option to file import section * preload files when opening imported files in web UI * test quick-upload formatImportDirectory * add openViaWebUI option to file import form * Remove low-level pin when using ipfs.add * Import to MFS from context menu This commit required abstracting the ipfs import functionality into a separate file to avoid a circular dependency between quick-upload and ipfs-companion, and continue to enable testing functionality. --- add-on/_locales/en/messages.json | 54 ++++++---- add-on/src/lib/context-menus.js | 23 ++--- add-on/src/lib/ipfs-companion.js | 79 ++++---------- add-on/src/lib/ipfs-import.js | 103 +++++++++++++++++++ add-on/src/lib/options.js | 4 +- add-on/src/options/forms/experiments-form.js | 11 -- add-on/src/options/forms/file-import-form.js | 59 +++++++++++ add-on/src/options/page.js | 8 +- add-on/src/popup/quick-upload.js | 100 +++++++++--------- test/functional/lib/ipfs-import.test.js | 74 +++++++++++++ 10 files changed, 354 insertions(+), 161 deletions(-) create mode 100644 add-on/src/lib/ipfs-import.js create mode 100644 add-on/src/options/forms/file-import-form.js create mode 100644 test/functional/lib/ipfs-import.test.js diff --git a/add-on/_locales/en/messages.json b/add-on/_locales/en/messages.json index 841ed34f6..047bd360e 100644 --- a/add-on/_locales/en/messages.json +++ b/add-on/_locales/en/messages.json @@ -127,17 +127,13 @@ "message": "This Page", "description": "An item in right-click context menu (contextMenu_parentPage)" }, - "contextMenu_AddToIpfsKeepFilename": { - "message": "Add to IPFS (Keep Filename)", - "description": "An item in right-click context menu (contextMenu_AddToIpfsKeepFilename)" + "contextMenu_importToIpfs": { + "message": "Import to IPFS", + "description": "An item in right-click context menu (contextMenu_importToIpfs)" }, - "contextMenu_AddToIpfsRawCid": { - "message": "Add to IPFS", - "description": "An item in right-click context menu (contextMenu_AddToIpfsRawCid)" - }, - "contextMenu_AddToIpfsSelection": { - "message": "Add Selected Text to IPFS", - "description": "An item in right-click context menu (contextMenu_AddToIpfsSelection)" + "contextMenu_importToIpfsSelection": { + "message": "Import Selected Text to IPFS", + "description": "An item in right-click context menu (contextMenu_importToIpfsSelection)" }, "notify_addonIssueTitle": { "message": "IPFS Add-on Issue", @@ -215,6 +211,10 @@ "message": "IPFS Node", "description": "A section header on the Preferences screen (option_header_nodeType)" }, + "option_header_fileImport": { + "message": "File Import", + "description": "A section header on the Preferences screen (option_header_fileImport)" + }, "option_ipfsNodeType_title": { "message": "IPFS Node Type", "description": "An option title on the Preferences screen (option_ipfsNodeType_title)" @@ -411,12 +411,20 @@ "message": "Manage permissions", "description": "Link text for managing permissions" }, + "option_openViaWebUI_title": { + "message": "Open imported files in Web UI", + "description": "An option title on the Preferences screen (option_openViaWebUI_title)" + }, + "option_openViaWebUI_description": { + "message": "Display files in Web UI rather than opening file or parent directory at gateway.", + "description": "An option description on the Preferences screen (option_openViaWebUI_description)" + }, "option_preloadAtPublicGateway_title": { - "message": "Preload Uploads", + "message": "Preload Imports", "description": "An option title on the Preferences screen (option_preloadAtPublicGateway_title)" }, "option_preloadAtPublicGateway_description": { - "message": "Enables automatic preload of uploaded assets via asynchronous HTTP HEAD request to a Public Gateway", + "message": "Enables automatic preload of imported assets via asynchronous HTTP HEAD request to a Public Gateway", "description": "An option description on the Preferences screen (option_preloadAtPublicGateway_description)" }, "option_logNamespaces_title": { @@ -427,6 +435,14 @@ "message": "Customize which namespaces are logged to Browser Console. Changing this value will trigger extension restart.", "description": "An option description for the log level (option_logNamespaces_description)" }, + "option_importDir_title": { + "message": "File Import Directory", + "description": "An option title on the Preferences screen (option_importDir_title)" + }, + "option_importDir_description": { + "message": "Customize the directory used for imported files.", + "description": "An option description on the Preferences screen (option_importDir_description)" + }, "option_resetAllOptions_title": { "message": "Reset Everything", "description": "An option title and button label on the Preferences screen (option_resetAllOptions_title)" @@ -472,16 +488,16 @@ "description": "Status label on the share files page (quickUpload_state_buffering)" }, "quickUpload_options_show": { - "message": "upload options", + "message": "import options", "description": "Button on the share files page (quickUpload_options_show)" }, - "quickUpload_options_wrapWithDirectory": { - "message": "Wrap single files in a directory to preserve their filenames.", - "description": "Checkbox label on the share files page (quickUpload_options_wrapWithDirectory)" + "quickUpload_options_importDir": { + "message": "Path to store imported files", + "description": "Textbox label on the share files page (quickUpload_options_importDir)" }, - "quickUpload_options_pinUpload": { - "message": "Pin files so they are retained when performing garbage collection on your IPFS repo.", - "description": "Checkbox label on the share files page (quickUpload_options_pinUpload)" + "quickUpload_options_openViaWebUI": { + "message": "Open in Web UI", + "description": "Checkbox label on the share files page (quickUpload_options_openViaWebUI)" }, "page_proxyAcl_title": { "message": "Manage Permissions", diff --git a/add-on/src/lib/context-menus.js b/add-on/src/lib/context-menus.js index 4806b1aac..bf78c4bd4 100644 --- a/add-on/src/lib/context-menus.js +++ b/add-on/src/lib/context-menus.js @@ -51,10 +51,9 @@ const menuParentLink = 'contextMenu_parentLink' const menuParentPage = 'contextMenu_parentPage' // const menuParentText = 'contextMenu_parentText' // Generic Add to IPFS -const contextMenuAddToIpfsRawCid = 'contextMenu_AddToIpfsRawCid' -const contextMenuAddToIpfsKeepFilename = 'contextMenu_AddToIpfsKeepFilename' +const contextMenuImportToIpfs = 'contextMenu_importToIpfs' // Add X to IPFS -const contextMenuAddToIpfsSelection = 'contextMenu_AddToIpfsSelection' +const contextMenuImportToIpfsSelection = 'contextMenu_importToIpfsSelection' // Copy X const contextMenuCopyCanonicalAddress = 'panelCopy_currentIpfsAddress' const contextMenuCopyRawCid = 'panelCopy_copyRawCid' @@ -78,15 +77,7 @@ function createContextMenus (getState, runtime, ipfsPathValidator, { onAddFromCo contexts: [contextType] }) } - const createSeparator = (parentId, id, contextType) => { - return browser.contextMenus.create({ - id: `${parentId}_${id}`, - parentId, - type: 'separator', - contexts: ['all'] - }) - } - const createAddToIpfsMenuItem = (parentId, id, contextType, ipfsAddOptions) => { + const createImportToIpfsMenuItem = (parentId, id, contextType, ipfsAddOptions) => { const itemId = `${parentId}_${id}` apiMenuItems.add(itemId) return browser.contextMenus.create({ @@ -125,9 +116,7 @@ function createContextMenus (getState, runtime, ipfsPathValidator, { onAddFromCo } const buildSubmenu = (parentId, contextType) => { createSubmenu(parentId, contextType) - createAddToIpfsMenuItem(parentId, contextMenuAddToIpfsKeepFilename, contextType, { wrapWithDirectory: true }) - createAddToIpfsMenuItem(parentId, contextMenuAddToIpfsRawCid, contextType, { wrapWithDirectory: false }) - createSeparator(parentId, 'separator-1', contextType) + createImportToIpfsMenuItem(parentId, contextMenuImportToIpfs, contextType, { wrapWithDirectory: true, pin: false }) createCopierMenuItem(parentId, contextMenuCopyAddressAtPublicGw, contextType, onCopyAddressAtPublicGw) createCopierMenuItem(parentId, contextMenuCopyCanonicalAddress, contextType, onCopyCanonicalAddress) createCopierMenuItem(parentId, contextMenuCopyRawCid, contextType, onCopyRawCid) @@ -135,9 +124,9 @@ function createContextMenus (getState, runtime, ipfsPathValidator, { onAddFromCo /* createSubmenu(menuParentText, 'selection') - createAddToIpfsMenuItem(menuParentText, contextMenuAddToIpfsSelection, 'selection') + createImportToIpfsMenuItem(menuParentText, contextMenuImportToIpfsSelection, 'selection') */ - createAddToIpfsMenuItem(null, contextMenuAddToIpfsSelection, 'selection') + createImportToIpfsMenuItem(null, contextMenuImportToIpfsSelection, 'selection', { pin: false }) buildSubmenu(menuParentImage, 'image') buildSubmenu(menuParentVideo, 'video') buildSubmenu(menuParentAudio, 'audio') diff --git a/add-on/src/lib/ipfs-companion.js b/add-on/src/lib/ipfs-companion.js index c459e4a04..759a53222 100644 --- a/add-on/src/lib/ipfs-companion.js +++ b/add-on/src/lib/ipfs-companion.js @@ -12,9 +12,10 @@ const { optionDefaults, storeMissingOptions, migrateOptions } = require('./optio const { initState, offlinePeerCount } = require('./state') const { createIpfsPathValidator } = require('./ipfs-path') const createDnslinkResolver = require('./dnslink') -const { createRequestModifier, redirectOptOutHint } = require('./ipfs-request') +const { createRequestModifier } = require('./ipfs-request') const { initIpfsClient, destroyIpfsClient } = require('./ipfs-client') const { createIpfsUrlProtocolHandler } = require('./ipfs-protocol') +const createIpfsImportHandler = require('./ipfs-import') const createNotifier = require('./notifier') const createCopier = require('./copier') const { createRuntimeChecks } = require('./runtime-checks') @@ -38,6 +39,7 @@ module.exports = async function init () { var apiStatusUpdateInterval var ipfsProxy var ipfsProxyContentScript + var ipfsImportHandler const idleInSecs = 5 * 60 const browserActionPortName = 'browser-action-port' @@ -65,6 +67,7 @@ module.exports = async function init () { dnslinkResolver = createDnslinkResolver(getState) ipfsPathValidator = createIpfsPathValidator(getState, getIpfs, dnslinkResolver) + ipfsImportHandler = createIpfsImportHandler(getState, getIpfs, ipfsPathValidator, runtime) copier = createCopier(notify, ipfsPathValidator) contextMenus = createContextMenus(getState, runtime, ipfsPathValidator, { onAddFromContext, @@ -230,6 +233,8 @@ module.exports = async function init () { gwURLString: dropSlash(state.gwURLString), pubGwURLString: dropSlash(state.pubGwURLString), webuiRootUrl: state.webuiRootUrl, + importDir: state.importDir, + openViaWebUI: state.openViaWebUI, apiURLString: dropSlash(state.apiURLString), redirect: state.redirect, noRedirectHostnames: state.noRedirectHostnames, @@ -257,40 +262,16 @@ module.exports = async function init () { } } - // GUI - // =================================================================== - - function preloadAtPublicGateway (path) { - if (!state.preloadAtPublicGateway) return - // asynchronous HTTP HEAD request preloads triggers content without downloading it - return new Promise((resolve, reject) => { - const http = new XMLHttpRequest() - // Make sure preload request is excluded from global redirect - const preloadUrl = ipfsPathValidator.resolveToPublicUrl(`${path}#${redirectOptOutHint}`, state.pubGwURLString) - http.open('HEAD', preloadUrl) - http.onreadystatechange = function () { - if (this.readyState === this.DONE) { - console.info(`[ipfs-companion] preloadAtPublicGateway(${path}):`, this.statusText) - if (this.status === 200) { - resolve(this.statusText) - } else { - reject(new Error(this.statusText)) - } - } - } - http.send() - }) - } - // Context Menu Uploader // ------------------------------------------------------------------- async function onAddFromContext (context, contextType, options) { + const importDir = ipfsImportHandler.formatImportDirectory(state.importDir) let result try { const dataSrc = await findValueForContext(context, contextType) if (contextType === 'selection') { - result = await ipfs.add(Buffer.from(dataSrc), options) + result = await ipfsImportHandler.importFiles(Buffer.from(dataSrc), options, importDir) } else { // Enchanced addFromURL // -------------------- @@ -319,7 +300,7 @@ module.exports = async function init () { path: decodeURIComponent(filename), content: buffer } - result = await ipfs.add(data, options) + result = await ipfsImportHandler.importFiles(data, options, importDir) } } catch (error) { console.error('Error in upload to IPFS context menu', error) @@ -334,37 +315,12 @@ module.exports = async function init () { } return } - - return uploadResultHandler({ result, openRootInNewTab: true }) - } - - // TODO: feature detect and push to client type specific modules. - function getIpfsPathAndNativeAddress (hash) { - const path = `/ipfs/${hash}` - if (runtime.hasNativeProtocolHandler) { - return { path, url: `ipfs://${hash}` } + ipfsImportHandler.preloadFilesAtPublicGateway(result) + if (state.ipfsNodeType === 'embedded' || !state.openViaWebUI) { + return ipfsImportHandler.openFilesAtGateway({ result, openRootInNewTab: true }) } else { - // open at public GW (will be redirected to local elsewhere, if enabled) - const url = new URL(path, state.pubGwURLString).toString() - return { path, url: url } - } - } - - async function uploadResultHandler ({ result, openRootInNewTab = false }) { - for (const file of result) { - if (file && file.hash) { - const { path, url } = getIpfsPathAndNativeAddress(file.hash) - preloadAtPublicGateway(path) - console.info('[ipfs-companion] successfully stored', file) - // open the wrapping directory (or the CID if wrapping was disabled) - if (openRootInNewTab && (result.length === 1 || file.path === '' || file.path === file.hash)) { - await browser.tabs.create({ - url: url - }) - } - } + return ipfsImportHandler.openFilesAtWebUI(importDir) } - return result } // Page-specific Actions @@ -709,12 +665,16 @@ module.exports = async function init () { shouldReloadExtension = true state[key] = localStorage.debug = change.newValue break + case 'importDir': + state[key] = change.newValue + break case 'linkify': case 'catchUnhandledProtocols': case 'displayNotifications': case 'automaticMode': case 'detectIpfsPathHeader': case 'preloadAtPublicGateway': + case 'openViaWebUI': case 'noRedirectHostnames': state[key] = change.newValue break @@ -783,8 +743,8 @@ module.exports = async function init () { return notify }, - get uploadResultHandler () { - return uploadResultHandler + get ipfsImportHandler () { + return ipfsImportHandler }, destroy () { @@ -796,6 +756,7 @@ module.exports = async function init () { dnslinkResolver = null modifyRequest = null ipfsPathValidator = null + ipfsImportHandler = null notify = null copier = null contextMenus = null diff --git a/add-on/src/lib/ipfs-import.js b/add-on/src/lib/ipfs-import.js new file mode 100644 index 000000000..727d6b098 --- /dev/null +++ b/add-on/src/lib/ipfs-import.js @@ -0,0 +1,103 @@ +'use strict' +/* eslint-env browser, webextensions */ + +const browser = require('webextension-polyfill') + +const { redirectOptOutHint } = require('./ipfs-request') + +function createIpfsImportHandler (getState, getIpfs, ipfsPathValidator, runtime) { + const ipfsImportHandler = { + formatImportDirectory (path) { + path = path.replace(/\/$|$/, '/') + path = path.replace(/(\/)\/+/g, '$1') + + // needed to handle date symbols in the import directory + const now = new Date() + const dateSymbols = [/%Y/g, /%M/g, /%D/g, /%h/g, /%m/g, /%s/g] + const symbolReplacements = [now.getFullYear(), now.getMonth() + 1, now.getDate(), now.getHours(), now.getMinutes(), now.getSeconds()].map(n => String(n).padStart(2, '0')) + dateSymbols.forEach((symbol, i) => { path = path.replace(symbol, symbolReplacements[i]) }) + return path + }, + // TODO: feature detect and push to client type specific modules. + getIpfsPathAndNativeAddress (hash) { + const state = getState() + const path = `/ipfs/${hash}` + if (runtime.hasNativeProtocolHandler) { + return { path, url: `ipfs://${hash}` } + } else { + // open at public GW (will be redirected to local elsewhere, if enabled) + const url = new URL(path, state.pubGwURLString).toString() + return { path, url: url } + } + }, + async openFilesAtGateway ({ result, openRootInNewTab = false }) { + for (const file of result) { + if (file && file.hash) { + const { url } = this.getIpfsPathAndNativeAddress(file.hash) + if (openRootInNewTab && (result.length === 1 || file.path === '' || file.path === file.hash)) { + await browser.tabs.create({ + url: url + }) + } + } + } + return result + }, + async openFilesAtWebUI (dir) { + const state = getState() + await browser.tabs.create({ + url: `${state.webuiRootUrl}#/files${dir}` + }) + }, + async importFiles (data, options, importDir) { + const ipfs = getIpfs() + // files are first `add`ed to IPFS + // and then copied to an MFS directory + // to ensure that CIDs for any created file + // remain the same for ipfs-companion and Web UI + const result = await ipfs.add(data, options) + // cp will fail if directory does not exist + await ipfs.files.mkdir(`${importDir}`, { parents: true }) + // remove directory from files API import files + let files = result.filter(file => (file.path !== '')) + files = files.map(file => (ipfs.files.cp(`/ipfs/${file.hash}`, `${importDir}${file.path}`))) + await Promise.all(files) + + return result + }, + preloadAtPublicGateway (path) { + const state = getState() + if (!state.preloadAtPublicGateway) return + // asynchronous HTTP HEAD request preloads triggers content without downloading it + return new Promise((resolve, reject) => { + const http = new XMLHttpRequest() + // Make sure preload request is excluded from global redirect + const preloadUrl = ipfsPathValidator.resolveToPublicUrl(`${path}#${redirectOptOutHint}`, state.pubGwURLString) + http.open('HEAD', preloadUrl) + http.onreadystatechange = function () { + if (this.readyState === this.DONE) { + console.info(`[ipfs-companion] preloadAtPublicGateway(${path}):`, this.statusText) + if (this.status === 200) { + resolve(this.statusText) + } else { + reject(new Error(this.statusText)) + } + } + } + http.send() + }) + }, + preloadFilesAtPublicGateway (files) { + files.forEach(file => { + if (file && file.hash) { + const { path } = this.getIpfsPathAndNativeAddress(file.hash) + this.preloadAtPublicGateway(path) + console.info('[ipfs-companion] successfully stored', file) + } + }) + } + } + return ipfsImportHandler +} + +module.exports = createIpfsImportHandler diff --git a/add-on/src/lib/options.js b/add-on/src/lib/options.js index 9ed986a1f..dfc31076a 100644 --- a/add-on/src/lib/options.js +++ b/add-on/src/lib/options.js @@ -26,7 +26,9 @@ exports.optionDefaults = Object.freeze({ ipfsApiUrl: buildIpfsApiUrl(), ipfsApiPollMs: 3000, ipfsProxy: true, // window.ipfs - logNamespaces: 'jsipfs*,ipfs*,libp2p:mdns*,libp2p-delegated*,-*:ipns*,-ipfs:preload*,-ipfs-http-client:request*,-ipfs:http-api*' + logNamespaces: 'jsipfs*,ipfs*,libp2p:mdns*,libp2p-delegated*,-*:ipns*,-ipfs:preload*,-ipfs-http-client:request*,-ipfs:http-api*', + importDir: '/ipfs-companion-imports/%Y-%M-%D_%h%m%s/', + openViaWebUI: true }) function buildCustomGatewayUrl () { diff --git a/add-on/src/options/forms/experiments-form.js b/add-on/src/options/forms/experiments-form.js index 6c1d95376..c5e2a5e95 100644 --- a/add-on/src/options/forms/experiments-form.js +++ b/add-on/src/options/forms/experiments-form.js @@ -7,7 +7,6 @@ const switchToggle = require('../../pages/components/switch-toggle') function experimentsForm ({ displayNotifications, - preloadAtPublicGateway, catchUnhandledProtocols, linkify, dnslinkPolicy, @@ -19,7 +18,6 @@ function experimentsForm ({ onOptionsReset }) { const onDisplayNotificationsChange = onOptionChange('displayNotifications') - const onPreloadAtPublicGatewayChange = onOptionChange('preloadAtPublicGateway') const onCatchUnhandledProtocolsChange = onOptionChange('catchUnhandledProtocols') const onLinkifyChange = onOptionChange('linkify') const onDnslinkPolicyChange = onOptionChange('dnslinkPolicy') @@ -41,15 +39,6 @@ function experimentsForm ({
${switchToggle({ id: 'displayNotifications', checked: displayNotifications, onchange: onDisplayNotificationsChange })}
-
- -
${switchToggle({ id: 'preloadAtPublicGateway', checked: preloadAtPublicGateway, onchange: onPreloadAtPublicGatewayChange })}
-