diff --git a/package-lock.json b/package-lock.json index b3a95892..98255d5d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3832,6 +3832,12 @@ "integrity": "sha512-IxYxNhwE5VwOm52L1yoFWYLP7q9Pd+NJjzOC5tlepfvEGaY3o9hslhUrx9BgseqdfZtKSDtd/4NfCSMjNzQalA==", "license": "Apache-2.0" }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.17", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz", + "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==", + "license": "Apache-2.0" + }, "node_modules/@mexp/ai": { "resolved": "packages/ai", "link": true @@ -29057,6 +29063,7 @@ "version": "0.0.1", "license": "GPL-2.0-or-later", "dependencies": { + "@mediapipe/tasks-vision": "^0.10.17", "@mexp/ai": "file:../ai", "@mexp/interface": "file:../interface", "@mexp/media-recording": "file:../media-recording", diff --git a/packages/editor/package.json b/packages/editor/package.json index 58892bcb..2fcb1867 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -33,6 +33,7 @@ } }, "dependencies": { + "@mediapipe/tasks-vision": "^0.10.17", "@mexp/ai": "file:../ai", "@mexp/interface": "file:../interface", "@mexp/media-recording": "file:../media-recording", diff --git a/packages/editor/src/block-media-panel/generate-caption.tsx b/packages/editor/src/block-media-panel/generate-caption.tsx deleted file mode 100644 index 4e670d56..00000000 --- a/packages/editor/src/block-media-panel/generate-caption.tsx +++ /dev/null @@ -1,125 +0,0 @@ -/** - * External dependencies - */ -import { createWorkerFactory } from '@shopify/web-worker'; - -/** - * WordPress dependencies - */ -import { __ } from '@wordpress/i18n'; -import { BlockControls } from '@wordpress/block-editor'; -import { ToolbarDropdownMenu } from '@wordpress/components'; -import { useState } from '@wordpress/element'; -import { store as noticesStore } from '@wordpress/notices'; -import { useDispatch, useSelect } from '@wordpress/data'; -import { store as preferencesStore } from '@wordpress/preferences'; - -/** - * Internal dependencies - */ -import { PREFERENCES_NAME } from '../constants'; -import { ReactComponent as PhotoSpark } from '../icons/photo-spark.svg'; - -const createAiWorker = createWorkerFactory( - () => import( /* webpackChunkName: 'ai' */ '@mexp/ai' ) -); - -const aiWorker = createAiWorker(); - -interface GenerateCaptionsProps { - url?: string; - onUpdateCaption: ( caption: string ) => void; - onUpdateAltText: ( alt: string ) => void; -} - -export function GenerateCaptions( { - url, - onUpdateCaption, - onUpdateAltText, -}: GenerateCaptionsProps ) { - const [ captionInProgress, setCaptionInProgress ] = useState( false ); - const [ altInProgress, setAltInProgress ] = useState( false ); - - const { createErrorNotice } = useDispatch( noticesStore ); - - const useAi = useSelect( ( select ) => { - return select( preferencesStore ).get( - PREFERENCES_NAME, - 'useAi' - ) as boolean; - }, [] ); - - if ( ! url || ! useAi ) { - return null; - } - - const controls = [ - { - title: __( 'Write caption', 'media-experiments' ), - onClick: async () => { - setCaptionInProgress( true ); - - try { - const result = await aiWorker.generateCaption( - url, - '' - ); - onUpdateCaption( result ); - } catch { - void createErrorNotice( - __( - 'There was an error generating the caption', - 'media-experiments' - ), - { - type: 'snackbar', - } - ); - } finally { - setCaptionInProgress( false ); - } - }, - role: 'menuitemradio', - icon: undefined, - isDisabled: captionInProgress, - }, - { - title: __( 'Write alternative text', 'media-experiments' ), - onClick: async () => { - setAltInProgress( true ); - - try { - const result = await aiWorker.generateCaption( - url, - '' - ); - onUpdateAltText( result ); - } catch { - void createErrorNotice( - __( - 'There was an error generating the alternative text', - 'media-experiments' - ), - { - type: 'snackbar', - } - ); - } finally { - setAltInProgress( false ); - } - }, - role: 'menuitemradio', - icon: undefined, - isDisabled: altInProgress, - }, - ]; - return ( - - } - controls={ controls } - /> - - ); -} diff --git a/packages/editor/src/block-media-panel/image-block-controls.tsx b/packages/editor/src/block-media-panel/image-block-controls.tsx new file mode 100644 index 00000000..20b06e33 --- /dev/null +++ b/packages/editor/src/block-media-panel/image-block-controls.tsx @@ -0,0 +1,268 @@ +/** + * External dependencies + */ +import { createWorkerFactory } from '@shopify/web-worker'; + +/** + * WordPress dependencies + */ +import { __ } from '@wordpress/i18n'; +import { BlockControls } from '@wordpress/block-editor'; +import { ToolbarDropdownMenu } from '@wordpress/components'; +import { useState } from '@wordpress/element'; +import { store as noticesStore } from '@wordpress/notices'; +import { useDispatch, useSelect } from '@wordpress/data'; +import { store as preferencesStore } from '@wordpress/preferences'; + +/** + * Internal dependencies + */ +import { PREFERENCES_NAME } from '../constants'; +import { ReactComponent as PhotoSpark } from '../icons/photo-spark.svg'; +import { store as uploadStore } from '@mexp/upload-media'; +import { image } from '@wordpress/icons'; +import { createBlobURL } from '@wordpress/blob'; + +const createAiWorker = createWorkerFactory( + () => import( /* webpackChunkName: 'ai' */ '@mexp/ai' ) +); + +const aiWorker = createAiWorker(); + +interface GenerateCaptionsProps { + id?: number; + url?: string; + onUpdateCaption: ( caption: string ) => void; + onUpdateAltText: ( alt: string ) => void; +} + +export function ImageBlockControls( { + id, + url, + onUpdateCaption, + onUpdateAltText, +}: GenerateCaptionsProps ) { + const [ captionInProgress, setCaptionInProgress ] = useState( false ); + const [ altInProgress, setAltInProgress ] = useState( false ); + + const { createErrorNotice } = useDispatch( noticesStore ); + + const isUploadingById = useSelect( + ( select ) => + id ? select( uploadStore ).isUploadingById( id ) : false, + [ id ] + ); + + const useAi = useSelect( ( select ) => { + return select( preferencesStore ).get( + PREFERENCES_NAME, + 'useAi' + ) as boolean; + }, [] ); + + if ( ! url || ! useAi ) { + return null; + } + + const controls = [ + { + title: __( 'Write caption', 'media-experiments' ), + onClick: async () => { + setCaptionInProgress( true ); + + try { + const result = await aiWorker.generateCaption( + url, + '' + ); + onUpdateCaption( result ); + } catch { + void createErrorNotice( + __( + 'There was an error generating the caption', + 'media-experiments' + ), + { + type: 'snackbar', + } + ); + } finally { + setCaptionInProgress( false ); + } + }, + role: 'menuitemradio', + icon: undefined, + isDisabled: captionInProgress, + }, + { + title: __( 'Write alternative text', 'media-experiments' ), + onClick: async () => { + setAltInProgress( true ); + + try { + const result = await aiWorker.generateCaption( + url, + '' + ); + onUpdateAltText( result ); + } catch { + void createErrorNotice( + __( + 'There was an error generating the alternative text', + 'media-experiments' + ), + { + type: 'snackbar', + } + ); + } finally { + setAltInProgress( false ); + } + }, + role: 'menuitemradio', + icon: undefined, + isDisabled: altInProgress, + }, + ]; + + if ( id ) { + controls.push( { + title: __( 'Remove background', 'media-experiments' ), + onClick: async () => { + const editorCanvas = + ( + ( document.querySelector( + 'iframe[name="editor-canvas"]' + ) as HTMLIFrameElement ) || null + )?.contentDocument || document; + + const imgElement = editorCanvas.querySelector( + `img[src="${ url }"]` + ) as HTMLImageElement; + + const { FilesetResolver, ImageSegmenter } = await import( + /* webpackChunkName: "chunk-tasks-vision" */ '@mediapipe/tasks-vision' + ); + + const wasmFileset = await FilesetResolver.forVisionTasks( + 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17/wasm' + ); + + const imageSegmenter = await ImageSegmenter.createFromOptions( + wasmFileset, + { + baseOptions: { + modelAssetPath: + 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite', + delegate: 'GPU', + }, + runningMode: 'IMAGE', + outputCategoryMask: true, + } + ); + + const imageCanvas = new OffscreenCanvas( + imgElement.naturalWidth, + imgElement.naturalHeight + ); + const newCanvas = new OffscreenCanvas( + imgElement.naturalWidth, + imgElement.naturalHeight + ); + + const imageCanvasCtx = imageCanvas.getContext( '2d', { + alpha: true, + } ); + const newCanvasCtx = newCanvas.getContext( '2d', { + alpha: true, + } ); + + if ( ! imageCanvasCtx || ! newCanvasCtx ) { + return; + } + + newCanvasCtx.clearRect( + 0, + 0, + newCanvas.width, + newCanvas.height + ); + newCanvasCtx.drawImage( imgElement, 0, 0 ); + + // Get pixel data from canvas containing original video frame + const imageData = imageCanvasCtx.getImageData( + 0, + 0, + imgElement.naturalWidth, + imgElement.naturalHeight + ).data; + + // Get pixel data from canvas for background image + const newCanvasImageData = newCanvasCtx.getImageData( + 0, + 0, + imgElement.naturalWidth, + imgElement.naturalHeight + ).data; + + const result = imageSegmenter.segment( imgElement ); + + // Get mask from result - contains values 0-1 for foreground vs background + const mask = result.categoryMask?.getAsFloat32Array() || []; + let j = 0; + + // Loop through each pixel in mask + for ( let i = 0; i < mask.length; ++i ) { + // Convert float mask value to 0-255 integer + const maskVal = Math.round( mask[ i ] * 255.0 ); + + // Increment index by 4 for RGBA + j += 4; + + // If mask pixel is background... + if ( maskVal === 255 ) { + // Copy pixel colors from imageData to backgroundData + newCanvasImageData[ j ] = imageData[ j ]; + newCanvasImageData[ j + 1 ] = imageData[ j + 1 ]; + newCanvasImageData[ j + 2 ] = imageData[ j + 2 ]; + newCanvasImageData[ j + 3 ] = imageData[ j + 3 ]; + } + } + + // Create new ImageData from modified background pixel data + const uint8Array = new Uint8ClampedArray( + newCanvasImageData.buffer + ); + const dataNew = new ImageData( + uint8Array, + imgElement.naturalWidth, + imgElement.naturalHeight + ); + + // Draw new background to canvas + newCanvasCtx.putImageData( dataNew, 0, 0 ); + + const blob = await newCanvas.convertToBlob( { + type: 'image/webp', + } ); + + const blobURL = createBlobURL( blob ); + + imgElement.src = blobURL; + }, + role: 'menuitemradio', + icon: undefined, + isDisabled: isUploadingById, + } ); + } + + return ( + + } + controls={ controls } + /> + + ); +} diff --git a/packages/editor/src/block-media-panel/image-controls.tsx b/packages/editor/src/block-media-panel/image-controls.tsx index 8de92802..30c82aa1 100644 --- a/packages/editor/src/block-media-panel/image-controls.tsx +++ b/packages/editor/src/block-media-panel/image-controls.tsx @@ -19,7 +19,7 @@ import { DebugInfo } from './debug-info'; import type { ImageBlock } from '../types'; import { AnimatedGifConverter } from './animated-gif-converter'; import { UploadRequestControls } from './upload-requests/controls'; -import { GenerateCaptions } from './generate-caption'; +import { ImageBlockControls } from './image-block-controls'; import { BulkOptimization } from '../components/bulk-optimization'; import { useBlockAttachments } from '../utils/hooks'; @@ -108,7 +108,8 @@ export function ImageControls( props: ImageControlsProps ) { /> ) : null } -