diff --git a/package-lock.json b/package-lock.json index a6413c15..976632f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15549,7 +15549,6 @@ "version": "2.19.3", "resolved": "https://registry.npmjs.org/react-color/-/react-color-2.19.3.tgz", "integrity": "sha512-LEeGE/ZzNLIsFWa1TMe8y5VYqr7bibneWmvJwm1pCn/eNmrabWDh659JSPn9BuaMpEfU83WTOJfnCcjDZwNQTA==", - "license": "MIT", "dependencies": { "@icons/material": "^0.2.4", "lodash": "^4.17.15", diff --git a/packages/cdk/lambda/utils/models.ts b/packages/cdk/lambda/utils/models.ts index 133d6956..85dafcab 100644 --- a/packages/cdk/lambda/utils/models.ts +++ b/packages/cdk/lambda/utils/models.ts @@ -5,6 +5,7 @@ import { PromptTemplate, StableDiffusionParams, TitanImageParams, + TitanImageV2Params, UnrecordedMessage, ConverseInferenceParams, UsecaseConverseInferenceParams, @@ -371,7 +372,7 @@ const createBodyImageStableDiffusion = (params: GenerateImageParams) => { init_image: params.initImage, mask_image: params.maskImage, mask_source: - params.maskMode === 'INPAINTING' + params.taskType === 'INPAINTING' ? 'MASK_IMAGE_BLACK' : 'MASK_IMAGE_WHITE', }; @@ -390,7 +391,7 @@ const createBodyImageTitanImage = (params: GenerateImageParams) => { seed: params.seed % 214783648, // max for titan image }; let body: Partial = {}; - if (params.initImage && params.maskMode === undefined) { + if (params.initImage && params.taskType === 'IMAGE_VARIATION') { body = { taskType: 'IMAGE_VARIATION', imageVariationParams: { @@ -404,7 +405,7 @@ const createBodyImageTitanImage = (params: GenerateImageParams) => { }, imageGenerationConfig: imageGenerationConfig, }; - } else if (params.initImage && params.maskMode === 'INPAINTING') { + } else if (params.initImage && params.taskType === 'INPAINTING') { body = { taskType: 'INPAINTING', inPaintingParams: { @@ -419,7 +420,7 @@ const createBodyImageTitanImage = (params: GenerateImageParams) => { }, imageGenerationConfig: imageGenerationConfig, }; - } else if (params.initImage && params.maskMode === 'OUTPAINTING') { + } else if (params.initImage && params.taskType === 'OUTPAINTING') { body = { taskType: 'OUTPAINTING', outPaintingParams: { @@ -435,7 +436,7 @@ const createBodyImageTitanImage = (params: GenerateImageParams) => { }, imageGenerationConfig: imageGenerationConfig, }; - } else { + } else if (params.taskType === 'TEXT_IMAGE') { body = { taskType: 'TEXT_IMAGE', textToImageParams: { @@ -447,7 +448,51 @@ const createBodyImageTitanImage = (params: GenerateImageParams) => { }, imageGenerationConfig: imageGenerationConfig, }; + } else { + body = { + imageGenerationConfig: imageGenerationConfig, + }; + } + return JSON.stringify(body); +}; + +const createBodyImageTitanImageV2 = (params: GenerateImageParams) => { + // 既存の関数を呼び出して基本的なボディを取得 + const baseBody = JSON.parse(createBodyImageTitanImage(params)); + + let body: Partial = { + ...baseBody, + }; + + // 新しいタスクタイプの処理 + if (params.taskType === 'COLOR_GUIDED_GENERATION') { + body = { + taskType: 'COLOR_GUIDED_GENERATION', + colorGuidedGenerationParams: { + text: params.textPrompt.find((x) => x.weight > 0)?.text || '', + negativeText: params.textPrompt.find((x) => x.weight < 0)?.text, + referenceImage: params.initImage, + colors: params.colors!, + }, + imageGenerationConfig: body.imageGenerationConfig, + }; + } else if (params.taskType === 'BACKGROUND_REMOVAL') { + body = { + taskType: 'BACKGROUND_REMOVAL', + backgroundRemovalParams: { + image: params.initImage!, + }, + }; + } else if (body.textToImageParams) { + // TEXT_IMAGE タスクタイプの拡張(Image Conditioning) + body.textToImageParams = { + ...body.textToImageParams, + conditionImage: params.initImage, + controlMode: params.controlMode, + controlStrength: params.controlStrength, + }; } + return JSON.stringify(body); }; @@ -753,7 +798,7 @@ export const BEDROCK_IMAGE_GEN_MODELS: { extractOutputImage: extractOutputImageTitanImage, }, 'amazon.titan-image-generator-v2:0': { - createBodyImage: createBodyImageTitanImage, + createBodyImage: createBodyImageTitanImageV2, extractOutputImage: extractOutputImageTitanImage, }, }; diff --git a/packages/types/src/image.d.ts b/packages/types/src/image.d.ts index f8e5d112..f761f984 100644 --- a/packages/types/src/image.d.ts +++ b/packages/types/src/image.d.ts @@ -1,5 +1,20 @@ +export type ControlMode = 'CANNY_EDGE' | 'SEGMENTATION'; +export type BaseGenerationMode = + | 'TEXT_IMAGE' + | 'IMAGE_VARIATION' + | 'INPAINTING' + | 'OUTPAINTING'; +export type TitanImageV2GenerationMode = + | 'IMAGE_CONDITIONING' + | 'COLOR_GUIDED_GENERATION' + | 'BACKGROUND_REMOVAL'; +export type GenerationMode = BaseGenerationMode | TitanImageV2GenerationMode; // 標準化したパラメータ export type GenerateImageParams = { + taskType?: + | BaseGenerationMode + | 'COLOR_GUIDED_GENERATION' + | 'BACKGROUND_REMOVAL'; textPrompt: { text: string; weight: number; @@ -16,7 +31,11 @@ export type GenerateImageParams = { // Inpaint / Outpaint maskImage?: string; maskPrompt?: string; - maskMode?: 'INPAINTING' | 'OUTPAINTING'; + // Color Guided Generation + colors?: string[]; + // Image Conditioning + controlStrength?: number; + controlMode?: ControlMode; }; // Stable Diffusion @@ -47,7 +66,7 @@ export type StableDiffusionParams = { // Titan Image // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-image.html export type TitanImageParams = { - taskType: 'TEXT_IMAGE' | 'INPAINTING' | 'OUTPAINTING' | 'IMAGE_VARIATION'; + taskType: BaseGenerationMode; textToImageParams?: { text: string; negativeText?: string; @@ -83,6 +102,27 @@ export type TitanImageParams = { }; }; +export type TitanImageV2Params = Omit & { + taskType: + | BaseGenerationMode + | 'COLOR_GUIDED_GENERATION' + | 'BACKGROUND_REMOVAL'; + textToImageParams?: TitanImageParams['textToImageParams'] & { + conditionImage?: string; // base64 encoded image + controlMode?: ControlMode; + controlStrength?: number; + }; + colorGuidedGenerationParams?: { + text: string; + negativeText?: string; + referenceImage?: string; // base64 encoded image + colors: string[]; // list of color hex codes + }; + backgroundRemovalParams?: { + image: string; // base64 encoded image + }; +}; + export type BedrockImageGenerationResponse = { result: string; artifacts: { diff --git a/packages/web/src/components/Select.tsx b/packages/web/src/components/Select.tsx index ee61f70c..95ddb3d1 100644 --- a/packages/web/src/components/Select.tsx +++ b/packages/web/src/components/Select.tsx @@ -3,6 +3,7 @@ import { Listbox, Transition } from '@headlessui/react'; import { PiCaretUpDown, PiCheck, PiX } from 'react-icons/pi'; import RowItem, { RowItemProps } from './RowItem'; import ButtonIcon from './ButtonIcon'; +import Help from './Help'; type Props = RowItemProps & { label?: string; @@ -11,34 +12,71 @@ type Props = RowItemProps & { value: string; label: string; }[]; + help?: string; clearable?: boolean; fullWidth?: boolean; + colorchip?: boolean; onChange: (value: string) => void; }; const Select: React.FC = (props) => { - const selectedLabel = useMemo(() => { + const selectedOption = useMemo(() => { return props.value === '' - ? '' - : props.options.filter((o) => o.value === props.value)[0].label; + ? null + : props.options.find((o) => o.value === props.value); }, [props.options, props.value]); const onClear = useCallback(() => { props.onChange(''); }, [props]); + const renderColorChips = (value: string) => { + const colors = value.split(',').map((color) => color.trim()); + return ( +
+ {colors.map((color, index) => ( +
+ ))} +
+ ); + }; + + const renderOptionContent = (option: { value: string; label: string }) => { + if (props.colorchip) { + return ( +
+ {renderColorChips(option.value)} + {option.label} +
+ ); + } + return option.label; + }; + return ( {props.label && ( -
+
{props.label} + {props.help && }
)}
- {selectedLabel} + + {selectedOption ? renderOptionContent(selectedOption) : ''} + @@ -73,7 +111,7 @@ const Select: React.FC = (props) => { className={`block truncate ${ selected ? 'font-medium' : 'font-normal' }`}> - {option.label} + {renderOptionContent(option)} {selected ? ( diff --git a/packages/web/src/pages/GenerateImagePage.tsx b/packages/web/src/pages/GenerateImagePage.tsx index fc6ef363..97489711 100644 --- a/packages/web/src/pages/GenerateImagePage.tsx +++ b/packages/web/src/pages/GenerateImagePage.tsx @@ -22,24 +22,39 @@ import { GenerateImagePageQueryParams } from '../@types/navigate'; import { MODELS } from '../hooks/useModel'; import { getPrompter } from '../prompts'; import queryString from 'query-string'; -import { GenerateImageParams } from 'generative-ai-use-cases-jp'; +import { + GenerateImageParams, + ControlMode, + GenerationMode, +} from 'generative-ai-use-cases-jp'; const MAX_SAMPLE = 7; -type GenerationMode = - | 'TEXT_IMAGE' - | 'IMAGE_VARIATION' - | 'INPAINTING' - | 'OUTPAINTING'; -const modeOptions = [ - 'TEXT_IMAGE', - 'IMAGE_VARIATION', - 'INPAINTING', - 'OUTPAINTING', -].map((s) => ({ - value: s as GenerationMode, - label: s as GenerationMode, -})); +const getModeOptions = (imageGenModelId: string) => { + const baseOptions = [ + 'TEXT_IMAGE', + 'IMAGE_VARIATION', + 'INPAINTING', + 'OUTPAINTING', + ]; + + if (imageGenModelId === 'amazon.titan-image-generator-v2:0') { + return [ + ...baseOptions, + 'IMAGE_CONDITIONING', + 'COLOR_GUIDED_GENERATION', + 'BACKGROUND_REMOVAL', + ].map((s) => ({ + value: s as GenerationMode, + label: s as GenerationMode, + })); + } + + return baseOptions.map((s) => ({ + value: s as GenerationMode, + label: s as GenerationMode, + })); +}; const resolutionPresets = [ '512 x 512', @@ -51,6 +66,61 @@ const resolutionPresets = [ label: s, })); +const colorsOptions = [ + { + value: '#efd9b4,#d6a692,#a39081,#4d6164,#292522', + label: 'Earthy Neutrals', + }, + { + value: '#001449,#012677,#005bc5,#00b4fc,#17f9ff', + label: 'Ocean Blues', + }, + { + value: '#c7003f,#f90050,#f96a00,#faab00,#daf204', + label: 'Fiery Sunset', + }, + { + value: '#ffd100,#ffee32,#ffd100,#00a86b,#004b23', + label: 'Lemon Lime', + }, + { + value: '#006400,#228B22,#32CD32,#90EE90,#98FB98', + label: 'Forest Greens', + }, + { + value: '#4B0082,#8A2BE2,#9370DB,#BA55D3,#DDA0DD', + label: 'Royal Purples', + }, + { + value: '#FF8C00,#FFA500,#FFD700,#FFFF00,#F0E68C', + label: 'Golden Ambers', + }, + { + value: '#FFB6C1,#FFC0CB,#FFE4E1,#E6E6FA,#F0F8FF', + label: 'Soft Pastels', + }, + { + value: '#FF00FF,#00FFFF,#FF0000,#00FF00,#0000FF', + label: 'Vivid Rainbow', + }, + { + value: '#000000,#333333,#666666,#999999,#CCCCCC', + label: 'Classic Monochrome', + }, + { + value: '#FFFFFF,#F2F2F2,#E6E6E6,#D9D9D9,#CCCCCC', + label: 'Light Grayscale', + }, + { + value: '#704214,#8B4513,#A0522D,#CD853F,#DEB887', + label: 'Vintage Sepia', + }, + { + value: '#FF9900,#232F3E,#ffffff,#00464F,#6C7778', + label: 'Smile and Sky', + }, +]; + type StateType = { imageGenModelId: string; setImageGenModelId: (c: string) => void; @@ -70,6 +140,10 @@ type StateType = { setCfgScale: (n: number) => void; imageStrength: number; setImageStrength: (n: number) => void; + controlStrength: number; + setControlStrength: (n: number) => void; + controlMode: ControlMode; + setControlMode: (s: ControlMode) => void; generationMode: GenerationMode; setGenerationMode: (s: GenerationMode) => void; initImage: Canvas; @@ -78,6 +152,8 @@ type StateType = { setMaskImage: (s: Canvas) => void; maskPrompt: string; setMaskPrompt: (s: string) => void; + colors: string; + setColors: (colors: string) => void; imageSample: number; setImageSample: (n: number) => void; image: { @@ -104,7 +180,9 @@ const useGenerateImagePageState = create((set, get) => { step: 50, cfgScale: 7, imageStrength: 0.35, - generationMode: modeOptions[0]['value'], + controlStrength: 0.7, + controlMode: 'CANNY_EDGE' as ControlMode, + generationMode: 'TEXT_IMAGE' as GenerationMode, initImage: { imageBase64: '', foregroundBase64: '', @@ -116,6 +194,7 @@ const useGenerateImagePageState = create((set, get) => { backgroundColor: '', }, maskPrompt: '', + colors: colorsOptions[0].value, imageSample: 3, image: new Array(MAX_SAMPLE).fill({ base64: '', @@ -127,9 +206,18 @@ const useGenerateImagePageState = create((set, get) => { return { ...INIT_STATE, setImageGenModelId: (s: string) => { - set(() => ({ - imageGenModelId: s, - })); + set((state) => { + const newModeOptions = getModeOptions(s); + const newGenerationMode = newModeOptions.some( + (option) => option.value === state.generationMode + ) + ? state.generationMode + : newModeOptions[0].value; + return { + imageGenModelId: s, + generationMode: newGenerationMode, + }; + }); }, setPrompt: (s) => { set(() => ({ @@ -173,6 +261,16 @@ const useGenerateImagePageState = create((set, get) => { imageStrength: n, })); }, + setControlStrength: (n) => { + set(() => ({ + controlStrength: n, + })); + }, + setControlMode: (n) => { + set(() => ({ + controlMode: n, + })); + }, setGenerationMode: (s) => { set(() => ({ generationMode: s, @@ -193,6 +291,11 @@ const useGenerateImagePageState = create((set, get) => { maskPrompt: s, })); }, + setColors: (s) => { + set(() => ({ + colors: s, + })); + }, setImageSample: (n) => { set(() => ({ imageSample: n, @@ -263,6 +366,13 @@ const stylePresetOptions = [ label: s, })); +// Titan Image Generator v2のImage Conditioning適用時のControl Mode +// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-image.html +const controlModeOptions = ['CANNY_EDGE', 'SEGMENTATION'].map((s) => ({ + value: s as ControlMode, + label: s as ControlMode, +})); + const GenerateImagePage: React.FC = () => { const { imageGenModelId, @@ -289,6 +399,8 @@ const GenerateImagePage: React.FC = () => { setMaskImage, maskPrompt, setMaskPrompt, + colors, + setColors, image, setImage, setImageError, @@ -297,6 +409,10 @@ const GenerateImagePage: React.FC = () => { setImageSample, imageStrength, setImageStrength, + controlStrength, + setControlStrength, + controlMode, + setControlMode, chatContent, setChatContent, clear, @@ -317,6 +433,12 @@ const GenerateImagePage: React.FC = () => { const [isOpenMask, setIsOpenMask] = useState(false); const [selectedImageIndex, setSelectedImageIndex] = useState(0); const [detailExpanded, setDetailExpanded] = useState(false); + const [previousImageSample, setPreviousImageSample] = useState(3); + const [previousGenerationMode, setPreviousGenerationMode] = + useState('TEXT_IMAGE'); + const [modeOptions, setModeOptions] = useState( + getModeOptions(imageGenModelId) + ); const { modelIds, imageGenModelIds, imageGenModels } = MODELS; const modelId = getModelId(); const prompter = useMemo(() => { @@ -337,6 +459,24 @@ const GenerateImagePage: React.FC = () => { ); }, [imageGenModelId]); + useEffect(() => { + setModeOptions(getModeOptions(imageGenModelId)); + }, [imageGenModelId]); + + useEffect(() => { + setPreviousGenerationMode(generationMode); + }, [generationMode]); + + useEffect(() => { + if (generationMode === 'BACKGROUND_REMOVAL') { + setPreviousImageSample(imageSample); + setImageSample(1); + } else if (previousGenerationMode === 'BACKGROUND_REMOVAL') { + setImageSample(previousImageSample); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [generationMode]); + useEffect(() => { updateSystemContextByModel(); // eslint-disable-next-line react-hooks/exhaustive-deps @@ -409,6 +549,10 @@ const GenerateImagePage: React.FC = () => { seed: _seed, step, stylePreset: _stylePreset ?? stylePreset, + taskType: + generationMode === 'IMAGE_CONDITIONING' + ? 'TEXT_IMAGE' + : generationMode, }; if (generationMode === 'IMAGE_VARIATION') { @@ -426,10 +570,26 @@ const GenerateImagePage: React.FC = () => { initImage: initImage.imageBase64, maskPrompt: maskImage.imageBase64 ? undefined : maskPrompt, maskImage: maskImage.imageBase64, - maskMode: generationMode, + }; + } else if (generationMode === 'IMAGE_CONDITIONING') { + params = { + ...params, + initImage: initImage.imageBase64, + controlStrength, + controlMode, + }; + } else if (generationMode === 'COLOR_GUIDED_GENERATION') { + params = { + ...params, + initImage: initImage.imageBase64, + colors: colors.split(',').map((color) => color.trim()), + }; + } else if (generationMode === 'BACKGROUND_REMOVAL') { + params = { + ...params, + initImage: initImage.imageBase64, }; } - return generate( params, imageGenModels.find((m) => m.modelId === imageGenModelId) @@ -462,12 +622,15 @@ const GenerateImagePage: React.FC = () => { initImage, maskPrompt, maskImage, + colors, seed, setImage, setImageError, setSeed, step, stylePreset, + controlMode, + controlStrength, ] ); @@ -512,7 +675,6 @@ const GenerateImagePage: React.FC = () => { foregroundBase64: img, backgroundColor: '', }); - setDetailExpanded(true); } }, [ image, @@ -520,7 +682,6 @@ const GenerateImagePage: React.FC = () => { selectedImageIndex, setGenerationMode, setInitImage, - setDetailExpanded, ]); const clearAll = useCallback(() => { @@ -533,9 +694,9 @@ const GenerateImagePage: React.FC = () => {
{ setIsOpenSketch(false); }}> @@ -637,24 +798,27 @@ const GenerateImagePage: React.FC = () => {
+ {generationMode !== 'BACKGROUND_REMOVAL' && ( + <> +