diff --git a/crates/openplotva-app/src/image_jobs.rs b/crates/openplotva-app/src/image_jobs.rs index de761053..d835f412 100644 --- a/crates/openplotva-app/src/image_jobs.rs +++ b/crates/openplotva-app/src/image_jobs.rs @@ -74,6 +74,7 @@ pub type ImageJobEffectFuture<'a, T> = Pin + Send + ' pub const AIFARM_DRAW_API_SERVICE_NAME: &str = "draw-api"; pub const AIFARM_DRAW_API_ENDPOINT_NAME: &str = "generate"; pub const AIFARM_DRAW_API_FLUX3_ENDPOINT_NAME: &str = "flux3_generate"; +pub const AIFARM_DRAW_API_MOSS_ENDPOINT_NAME: &str = "moss_generate"; pub const AIFARM_DRAW_API_BOOGU_TURBO_ENDPOINT_NAME: &str = "boogu_turbo_generate"; pub const AIFARM_DRAW_API_BOOGU_EDIT_ENDPOINT_NAME: &str = "boogu_edit_generate"; pub const AIFARM_DRAW_API_DEFAULT_BASE_URL: &str = "http://127.0.0.1:50051"; @@ -101,6 +102,8 @@ pub const IMAGE_EDIT_WORKFLOW_KEY: &str = "image_edit"; pub const IMAGE_GENERATION_WORKFLOW_KEY: &str = "image_generation"; pub const IMAGE_GENERATION_FLUX_WORKFLOW_KEY: &str = "image_generation_flux"; pub const IMAGE_GENERATION_BOOGU_TURBO_WORKFLOW_KEY: &str = "image_generation_boogu_turbo"; +/// Third VIP text-to-image slot, routed to MOSS-Image. +pub const IMAGE_GENERATION_MOSS_WORKFLOW_KEY: &str = "image_generation_moss"; pub const IMAGE_EDIT_FLUX_WORKFLOW_KEY: &str = "image_edit_flux"; pub const IMAGE_EDIT_BOOGU_TURBO_WORKFLOW_KEY: &str = "image_edit_boogu_turbo"; pub const DRAW_SUPPORT_ME_URL: &str = "https://t.me/PlotvoBot?start=donate"; @@ -1196,7 +1199,11 @@ where match self .first - .generate_image(image_generation_request_for_prompt_slot(&request, 0)) + .generate_image(image_generation_request_for_prompt_slots( + &request, + 0, + self.first.expected_image_count(), + )) .await { Ok(result) if image_generation_result_has_images(&result) => { @@ -1216,9 +1223,10 @@ where let second_slot = self.first.expected_image_count().max(1); match self .second - .generate_image(image_generation_request_for_prompt_slot( + .generate_image(image_generation_request_for_prompt_slots( &request, second_slot, + self.second.expected_image_count(), )) .await { @@ -1258,7 +1266,11 @@ where match self .first - .generate_image(image_generation_request_for_prompt_slot(&request, 0)) + .generate_image(image_generation_request_for_prompt_slots( + &request, + 0, + self.first.expected_image_count(), + )) .await { Ok(result) if image_generation_result_has_images(&result) => { @@ -1279,9 +1291,10 @@ where let second_slot = self.first.expected_image_count().max(1); match self .second - .generate_image(image_generation_request_for_prompt_slot( + .generate_image(image_generation_request_for_prompt_slots( &request, second_slot, + self.second.expected_image_count(), )) .await { @@ -1346,8 +1359,16 @@ where Box::pin(async move { let first_slot = 0; let second_slot = self.first.expected_image_count(); - let first_request = image_generation_request_for_prompt_slot(&request, first_slot); - let second_request = image_generation_request_for_prompt_slot(&request, second_slot); + let first_request = image_generation_request_for_prompt_slots( + &request, + first_slot, + self.first.expected_image_count(), + ); + let second_request = image_generation_request_for_prompt_slots( + &request, + second_slot, + self.second.expected_image_count(), + ); let (first, second) = tokio::join!( self.first.generate_image(first_request), self.second.generate_image(second_request) @@ -1364,8 +1385,16 @@ where Box::pin(async move { let first_slot = 0; let second_slot = self.first.expected_image_count(); - let first_request = image_generation_request_for_prompt_slot(&request, first_slot); - let second_request = image_generation_request_for_prompt_slot(&request, second_slot); + let first_request = image_generation_request_for_prompt_slots( + &request, + first_slot, + self.first.expected_image_count(), + ); + let second_request = image_generation_request_for_prompt_slots( + &request, + second_slot, + self.second.expected_image_count(), + ); let first_progress = progress.clone(); let second_progress = progress; let (first, second) = tokio::join!( @@ -2171,16 +2200,20 @@ fn image_generation_result_has_images(result: &ImageGenerationResult) -> bool { result.first_image_url().is_some() || result.image_bytes.iter().any(|image| !image.is_empty()) } -fn image_generation_request_for_prompt_slot( +/// The request one side of a combinator sees: the prompt variants of its own slots, +/// `start..start + count`, so a nested combinator still hands each slot its prompt. +fn image_generation_request_for_prompt_slots( request: &ImageGenerationRequest, - slot: usize, + start: usize, + count: usize, ) -> ImageGenerationRequest { let mut request = request.clone(); request.prompt_variants = request .prompt_variants - .get(slot) - .and_then(|variant| non_empty(variant.clone())) - .into_iter() + .iter() + .skip(start) + .take(count.max(1)) + .filter_map(|variant| non_empty(variant.clone())) .collect(); request } @@ -6654,6 +6687,49 @@ mod tests { assert_eq!(regular.expected_image_count(), 1); } + #[tokio::test] + async fn nested_parallel_generators_hand_each_slot_its_own_prompt() { + let krea2 = GeneratorStub::success("https://img.test/krea2.png"); + let flux3 = GeneratorStub::success("https://img.test/flux3.png"); + let moss = GeneratorStub::success("https://img.test/moss.png"); + let vip = ParallelImageGenerator::new( + ParallelImageGenerator::new(krea2.clone(), flux3.clone()), + moss.clone(), + ); + assert_eq!(vip.expected_image_count(), 3); + + let result = vip + .generate_image(ImageGenerationRequest { + prompt_variants: vec![ + "krea2 prompt".to_owned(), + "flux3 prompt".to_owned(), + "moss prompt".to_owned(), + ], + ..ImageGenerationRequest::default() + }) + .await + .expect("vip result"); + + assert_eq!( + image_generation_urls(&result), + vec![ + "https://img.test/krea2.png".to_owned(), + "https://img.test/flux3.png".to_owned(), + "https://img.test/moss.png".to_owned(), + ] + ); + for (generator, prompt) in [ + (&krea2, "krea2 prompt"), + (&flux3, "flux3 prompt"), + (&moss, "moss prompt"), + ] { + assert_eq!( + generator.requests()[0].prompt_variants, + vec![prompt.to_owned()] + ); + } + } + fn empty_routed_attempt_walker() -> RoutedAttemptWalker { RoutedAttemptWalker::new( RouterHandle::new(RoutingTable::default()), diff --git a/crates/openplotva-app/src/lib.rs b/crates/openplotva-app/src/lib.rs index 023ecd27..9f1be958 100644 --- a/crates/openplotva-app/src/lib.rs +++ b/crates/openplotva-app/src/lib.rs @@ -13288,10 +13288,18 @@ async fn start_runtime_workers( image_jobs::aifarm_draw_api_config_from_app_config(config), ) .with_workflow_key(image_jobs::IMAGE_GENERATION_BOOGU_TURBO_WORKFLOW_KEY); + let routed_vip_moss_image_generator = image_jobs::RoutedImageGenerator::new( + image_attempt_walker.clone(), + image_jobs::aifarm_draw_api_config_from_app_config(config), + ) + .with_workflow_key(image_jobs::IMAGE_GENERATION_MOSS_WORKFLOW_KEY); let vip_image_generator = image_jobs::OptimizingImageGenerator::new( image_jobs::ParallelImageGenerator::new( - routed_vip_flux_image_generator, - routed_vip_boogu_image_generator, + image_jobs::ParallelImageGenerator::new( + routed_vip_flux_image_generator, + routed_vip_boogu_image_generator, + ), + routed_vip_moss_image_generator, ), media_prompt_optimizer.clone(), ) diff --git a/crates/openplotva-media/src/lib.rs b/crates/openplotva-media/src/lib.rs index 965bb369..7be4f4fc 100644 --- a/crates/openplotva-media/src/lib.rs +++ b/crates/openplotva-media/src/lib.rs @@ -69,6 +69,9 @@ pub enum ImageModel { Flux3, /// Krea 2 Turbo: one natural-prose paragraph, medium and style first. Krea2, + /// MOSS-Image SFT: a detailed caption in the form of its recaptioner, read by a + /// Qwen3.5 encoder. + Moss, } impl ImageModel { @@ -77,6 +80,7 @@ impl ImageModel { match self { Self::Flux3 => "FLUX.3 [klein]", Self::Krea2 => "Krea 2 Turbo", + Self::Moss => "MOSS-Image", } } @@ -86,6 +90,7 @@ impl ImageModel { match value.trim().to_ascii_lowercase().as_str() { "flux3" => Some(Self::Flux3), "krea2" => Some(Self::Krea2), + "moss" => Some(Self::Moss), _ => None, } } @@ -96,6 +101,7 @@ impl ImageModel { match self { Self::Flux3 => FLUX3_PROMPT_MAX_CHARS, Self::Krea2 => KREA2_PROMPT_MAX_CHARS, + Self::Moss => MOSS_PROMPT_MAX_CHARS, } } } @@ -119,6 +125,7 @@ impl Default for ImageTargets { impl ImageTargets { pub const FLUX3: Self = Self::single(ImageModel::Flux3); pub const KREA2: Self = Self::single(ImageModel::Krea2); + pub const MOSS: Self = Self::single(ImageModel::Moss); #[must_use] pub const fn single(model: ImageModel) -> Self { @@ -417,6 +424,9 @@ pub const FLUX3_PROMPT_MAX_CHARS: usize = 4000; /// Krea 2 rules ask for up to 200 words. pub const KREA2_PROMPT_MAX_CHARS: usize = 1600; +/// MOSS-Image rules ask for up to about 350 words, like FLUX.3. +pub const MOSS_PROMPT_MAX_CHARS: usize = 4000; + /// Cut every optimized output to at most `max_chars` characters. The bound is applied /// after decoding rather than as a schema `maxLength`: a bounded string makes the /// structured-output grammar of large-vocabulary engines slow to compile and decode. @@ -448,6 +458,7 @@ fn optimizer_prompt_data(options: OptimizePromptOptions) -> Value { "multi": variant_count > 1, "flux3": models.contains(&ImageModel::Flux3), "krea2": models.contains(&ImageModel::Krea2), + "moss": models.contains(&ImageModel::Moss), }) } @@ -906,6 +917,10 @@ mod tests { ImageModel::from_prompt_target("krea2"), Some(ImageModel::Krea2) ); + assert_eq!( + ImageModel::from_prompt_target("moss"), + Some(ImageModel::Moss) + ); for retired in ["klein", "boogu", "qwen_image", "dall-e"] { assert_eq!(ImageModel::from_prompt_target(retired), None); } @@ -924,6 +939,36 @@ mod tests { ImageTargets::KREA2.prompt_max_chars(), KREA2_PROMPT_MAX_CHARS ); + assert_eq!(ImageTargets::MOSS.prompt_max_chars(), MOSS_PROMPT_MAX_CHARS); + } + + #[test] + fn moss_slots_get_their_own_rules_and_example() { + let moss = render_image_optimizer_prompt(OptimizePromptOptions { + variant_count: 1, + targets: ImageTargets::MOSS, + }) + .expect("render moss"); + assert!(moss.contains("`outputs[0]` is rendered by MOSS-Image")); + assert!(moss.contains("**MOSS-Image**")); + assert!(moss.contains("single MOSS-Image slot")); + assert!(!moss.contains("**FLUX.3 [klein]**")); + assert!(!moss.contains("**Krea 2 Turbo**")); + assert!(!moss.contains("{{")); + + let vip = render_image_optimizer_prompt(OptimizePromptOptions { + variant_count: 3, + targets: ImageTargets::KREA2 + .followed_by(1, ImageTargets::FLUX3, 1) + .followed_by(2, ImageTargets::MOSS, 1), + }) + .expect("render vip"); + assert!(vip.contains("`outputs[0]` is rendered by Krea 2 Turbo")); + assert!(vip.contains("`outputs[1]` is rendered by FLUX.3 [klein]")); + assert!(vip.contains("`outputs[2]` is rendered by MOSS-Image")); + assert!(vip.contains("**Krea 2 Turbo**")); + assert!(vip.contains("**FLUX.3 [klein]**")); + assert!(vip.contains("**MOSS-Image**")); } #[test] diff --git a/migrations/209_moss_image_slot.down.sql b/migrations/209_moss_image_slot.down.sql new file mode 100644 index 00000000..0b7885bc --- /dev/null +++ b/migrations/209_moss_image_slot.down.sql @@ -0,0 +1,16 @@ +DELETE FROM workflow_assignments +WHERE workflow_key = 'image_generation_moss' + AND inference_overrides ? 'created_by_moss_image_slot_v1'; + +DELETE FROM provider_models +WHERE config ? 'created_by_moss_image_slot_v1' + AND NOT EXISTS ( + SELECT 1 FROM workflow_assignments + WHERE workflow_assignments.provider_model_id = provider_models.id + ); + +DELETE FROM workflows +WHERE key = 'image_generation_moss' + AND NOT EXISTS ( + SELECT 1 FROM workflow_assignments WHERE workflow_key = 'image_generation_moss' + ); diff --git a/migrations/209_moss_image_slot.up.sql b/migrations/209_moss_image_slot.up.sql new file mode 100644 index 00000000..6009e883 --- /dev/null +++ b/migrations/209_moss_image_slot.up.sql @@ -0,0 +1,82 @@ +-- VIP draws get a third picture from MOSS-Image (draw-api `moss_generate` on the AI Farm +-- GPU1 worker). `image_generation_moss` is a user-facing tool (lane priority 1) like the +-- other VIP slots. Its model shares the GPU1 media pool with the other draw-api models, +-- and FLUX.3 is its fallback, as it is for the Krea 2 slot. A MOSS draw takes 40 s (30 +-- steps), and the first one after a worker load also builds the fused caches (about 90 +-- s), so the retry wall is 120 s: the fallback can still start after a MOSS attempt that +-- fails late, where the 60 s of the other slots would end the request. The model and the +-- assignments are created only when the aifarm-draw provider, the GPU1 media pool and the +-- FLUX.3 model exist; otherwise the operator assigns models in the admin routing page. +-- Every created row carries a marker, so the down migration removes exactly these rows. + +INSERT INTO workflows (key, kind, full_routing, retry_max_hops, retry_wall_ms, enabled, lane_priority) +VALUES ('image_generation_moss', 'image', TRUE, 3, 120000, TRUE, 1) +ON CONFLICT (key) DO NOTHING; + +INSERT INTO provider_models (provider_id, model_name, display_name, capabilities, enabled, config, pool_id) +SELECT + provider.id, + 'moss-image-sft-heretic', + 'MOSS-Image SFT Heretic', + ARRAY['image'], + TRUE, + jsonb_build_object( + 'service_name', 'draw-api', + 'endpoint_name', 'moss_generate', + 'prompt_target', 'moss', + 'created_by_moss_image_slot_v1', TRUE + ), + capacity_pool.id +FROM llm_providers AS provider +JOIN llm_capacity_pools AS capacity_pool ON capacity_pool.name = 'aifarm-gpu1-media' +WHERE provider.name = 'aifarm-draw' + AND NOT EXISTS ( + SELECT 1 FROM provider_models AS model + WHERE model.provider_id = provider.id + AND model.config ->> 'endpoint_name' = 'moss_generate' + ); + +WITH draw_models AS ( + SELECT DISTINCT ON (model.config ->> 'endpoint_name') + model.config ->> 'endpoint_name' AS endpoint_name, + model.id AS provider_model_id + FROM provider_models AS model + JOIN llm_providers AS provider ON provider.id = model.provider_id + WHERE provider.name = 'aifarm-draw' + AND model.enabled + AND model.config ->> 'endpoint_name' IN ('moss_generate', 'flux3_generate') + ORDER BY model.config ->> 'endpoint_name', model.id +), targets (role, endpoint_name, weight, fallback_order) AS ( + VALUES + ('primary', 'moss_generate', 100, NULL::INTEGER), + ('fallback', 'flux3_generate', NULL::INTEGER, 0) +) +INSERT INTO workflow_assignments ( + workflow_key, + scope, + role, + provider_model_id, + weight, + fallback_order, + enabled, + inference_overrides, + cb_failure_threshold, + cb_cooldown_ms +) +SELECT + 'image_generation_moss', + 'global', + targets.role, + draw_models.provider_model_id, + targets.weight, + targets.fallback_order, + TRUE, + '{"created_by_moss_image_slot_v1": true}'::jsonb, + 5, + 30000 +FROM targets +JOIN draw_models USING (endpoint_name) +WHERE (SELECT count(*) FROM draw_models) = 2 + AND NOT EXISTS ( + SELECT 1 FROM workflow_assignments WHERE workflow_key = 'image_generation_moss' + ); diff --git a/prompts/image/optimizer.prompt b/prompts/image/optimizer.prompt index f410d43e..d19ca81e 100644 --- a/prompts/image/optimizer.prompt +++ b/prompts/image/optimizer.prompt @@ -61,6 +61,15 @@ FLUX.3 learned from captions written by its own prompt upsampler; write in exact 5. Fill out the scene with visible detail only; never invent names, brands, logos, or text. 6. Put every piece of text in double quotes exactly as written, in its own script, and say where it sits and how it is lettered. {{/if}} +{{#if moss}} +**MOSS-Image** +MOSS-Image learned from detailed captions written by a vision-language recaptioner; write in that form. +1. Write one cohesive paragraph of English prose in the generative voice, never "this image shows" or "we can see". Typical length is 100 to 300 words; a simple sign or symbol needs only a few sentences. A short request becomes a complete scene, and a long one is reorganized, never cut. +2. Open with the medium and the composition, such as "A photograph of ..." or "A watercolour illustration of ...", then the key subjects and their actions, the setting, the colours, the lighting, and the style. +3. Place each subject with spatial words (left, right, center, foreground, background, above, behind) and keep one consistent order. +4. For every piece of text in the image, give the exact wording in double quotes and its layout: position, line order, alignment, size relative to other text, letterform, and colour. +5. Write prose only: no tag lists, quality tags, or score tags. +{{/if}} ### CONTEXT 1. The input may start with a "Context:" block followed by a "Request:" block. @@ -159,3 +168,12 @@ Request: кот-астронавт на луне {"input": "кот-астронавт на луне", "outputs": ["A cinematic digital painting of a ginger tabby cat in a white astronaut suit standing on the grey surface of the Moon. The cat faces the viewer in the centre of the frame, its round glass helmet reflecting a small blue Earth that hangs above the horizon in the black sky on the right. Powdery regolith and shallow craters stretch across the foreground, and a trail of small paw prints leads in from the left edge. Hard, low sunlight from the left casts long black shadows and bright rims along the helmet and the suit. Calm, quietly heroic mood in a cold grey palette with warm orange accents."], "nsfw_result": "safe", "aspect_ratio": "16:9"} {{/if}} +{{#if moss}} + +The next example shows the answer for a single MOSS-Image slot. + + +Request: записка на холодильнике: купить молоко и хлеб +{"input": "записка на холодильнике: купить молоко и хлеб", "outputs": ["A photograph of a white refrigerator door in a home kitchen, taken at eye level in soft daylight from a window on the left. In the center of the door, a square yellow sticky note is held at its top edge by a round red magnet. The note carries one line of handwritten Cyrillic letters in blue ballpoint ink, \"купить молоко и хлеб\", written slightly uphill from left to right across the middle of the paper, the letters thin and rounded with a small loop on each \"л\". The lower right corner of the note curls away from the door and casts a thin grey shadow on the glossy white enamel. Below the note, a child's crayon drawing of a red house under a yellow sun is pinned by a blue magnet. On the right, the edge of a wooden counter holds a ceramic bowl of green apples, softly out of focus against a pale tiled wall."], "nsfw_result": "safe", "aspect_ratio": "3:4"} + +{{/if}}