Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 89 additions & 13 deletions crates/openplotva-app/src/image_jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ pub type ImageJobEffectFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + '
pub const AIFARM_DRAW_API_SERVICE_NAME: &str = "draw-api";
pub const AIFARM_DRAW_API_ENDPOINT_NAME: &str = "generate";
pub const AIFARM_DRAW_API_FLUX3_ENDPOINT_NAME: &str = "flux3_generate";
pub const AIFARM_DRAW_API_MOSS_ENDPOINT_NAME: &str = "moss_generate";
pub const AIFARM_DRAW_API_BOOGU_TURBO_ENDPOINT_NAME: &str = "boogu_turbo_generate";
pub const AIFARM_DRAW_API_BOOGU_EDIT_ENDPOINT_NAME: &str = "boogu_edit_generate";
pub const AIFARM_DRAW_API_DEFAULT_BASE_URL: &str = "http://127.0.0.1:50051";
Expand Down Expand Up @@ -101,6 +102,8 @@ pub const IMAGE_EDIT_WORKFLOW_KEY: &str = "image_edit";
pub const IMAGE_GENERATION_WORKFLOW_KEY: &str = "image_generation";
pub const IMAGE_GENERATION_FLUX_WORKFLOW_KEY: &str = "image_generation_flux";
pub const IMAGE_GENERATION_BOOGU_TURBO_WORKFLOW_KEY: &str = "image_generation_boogu_turbo";
/// Third VIP text-to-image slot, routed to MOSS-Image.
pub const IMAGE_GENERATION_MOSS_WORKFLOW_KEY: &str = "image_generation_moss";
pub const IMAGE_EDIT_FLUX_WORKFLOW_KEY: &str = "image_edit_flux";
pub const IMAGE_EDIT_BOOGU_TURBO_WORKFLOW_KEY: &str = "image_edit_boogu_turbo";
pub const DRAW_SUPPORT_ME_URL: &str = "https://t.me/PlotvoBot?start=donate";
Expand Down Expand Up @@ -1196,7 +1199,11 @@ where

match self
.first
.generate_image(image_generation_request_for_prompt_slot(&request, 0))
.generate_image(image_generation_request_for_prompt_slots(
&request,
0,
self.first.expected_image_count(),
))
.await
{
Ok(result) if image_generation_result_has_images(&result) => {
Expand All @@ -1216,9 +1223,10 @@ where
let second_slot = self.first.expected_image_count().max(1);
match self
.second
.generate_image(image_generation_request_for_prompt_slot(
.generate_image(image_generation_request_for_prompt_slots(
&request,
second_slot,
self.second.expected_image_count(),
))
.await
{
Expand Down Expand Up @@ -1258,7 +1266,11 @@ where

match self
.first
.generate_image(image_generation_request_for_prompt_slot(&request, 0))
.generate_image(image_generation_request_for_prompt_slots(
&request,
0,
self.first.expected_image_count(),
))
.await
{
Ok(result) if image_generation_result_has_images(&result) => {
Expand All @@ -1279,9 +1291,10 @@ where
let second_slot = self.first.expected_image_count().max(1);
match self
.second
.generate_image(image_generation_request_for_prompt_slot(
.generate_image(image_generation_request_for_prompt_slots(
&request,
second_slot,
self.second.expected_image_count(),
))
.await
{
Expand Down Expand Up @@ -1346,8 +1359,16 @@ where
Box::pin(async move {
let first_slot = 0;
let second_slot = self.first.expected_image_count();
let first_request = image_generation_request_for_prompt_slot(&request, first_slot);
let second_request = image_generation_request_for_prompt_slot(&request, second_slot);
let first_request = image_generation_request_for_prompt_slots(
&request,
first_slot,
self.first.expected_image_count(),
);
let second_request = image_generation_request_for_prompt_slots(
&request,
second_slot,
self.second.expected_image_count(),
);
let (first, second) = tokio::join!(
self.first.generate_image(first_request),
self.second.generate_image(second_request)
Expand All @@ -1364,8 +1385,16 @@ where
Box::pin(async move {
let first_slot = 0;
let second_slot = self.first.expected_image_count();
let first_request = image_generation_request_for_prompt_slot(&request, first_slot);
let second_request = image_generation_request_for_prompt_slot(&request, second_slot);
let first_request = image_generation_request_for_prompt_slots(
&request,
first_slot,
self.first.expected_image_count(),
);
let second_request = image_generation_request_for_prompt_slots(
&request,
second_slot,
self.second.expected_image_count(),
);
let first_progress = progress.clone();
let second_progress = progress;
let (first, second) = tokio::join!(
Expand Down Expand Up @@ -2171,16 +2200,20 @@ fn image_generation_result_has_images(result: &ImageGenerationResult) -> bool {
result.first_image_url().is_some() || result.image_bytes.iter().any(|image| !image.is_empty())
}

fn image_generation_request_for_prompt_slot(
/// The request one side of a combinator sees: the prompt variants of its own slots,
/// `start..start + count`, so a nested combinator still hands each slot its prompt.
fn image_generation_request_for_prompt_slots(
request: &ImageGenerationRequest,
slot: usize,
start: usize,
count: usize,
) -> ImageGenerationRequest {
let mut request = request.clone();
request.prompt_variants = request
.prompt_variants
.get(slot)
.and_then(|variant| non_empty(variant.clone()))
.into_iter()
.iter()
.skip(start)
.take(count.max(1))
.filter_map(|variant| non_empty(variant.clone()))
.collect();
request
}
Expand Down Expand Up @@ -6654,6 +6687,49 @@ mod tests {
assert_eq!(regular.expected_image_count(), 1);
}

#[tokio::test]
async fn nested_parallel_generators_hand_each_slot_its_own_prompt() {
let krea2 = GeneratorStub::success("https://img.test/krea2.png");
let flux3 = GeneratorStub::success("https://img.test/flux3.png");
let moss = GeneratorStub::success("https://img.test/moss.png");
let vip = ParallelImageGenerator::new(
ParallelImageGenerator::new(krea2.clone(), flux3.clone()),
moss.clone(),
);
assert_eq!(vip.expected_image_count(), 3);

let result = vip
.generate_image(ImageGenerationRequest {
prompt_variants: vec![
"krea2 prompt".to_owned(),
"flux3 prompt".to_owned(),
"moss prompt".to_owned(),
],
..ImageGenerationRequest::default()
})
.await
.expect("vip result");

assert_eq!(
image_generation_urls(&result),
vec![
"https://img.test/krea2.png".to_owned(),
"https://img.test/flux3.png".to_owned(),
"https://img.test/moss.png".to_owned(),
]
);
for (generator, prompt) in [
(&krea2, "krea2 prompt"),
(&flux3, "flux3 prompt"),
(&moss, "moss prompt"),
] {
assert_eq!(
generator.requests()[0].prompt_variants,
vec![prompt.to_owned()]
);
}
}

fn empty_routed_attempt_walker() -> RoutedAttemptWalker {
RoutedAttemptWalker::new(
RouterHandle::new(RoutingTable::default()),
Expand Down
12 changes: 10 additions & 2 deletions crates/openplotva-app/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13288,10 +13288,18 @@ async fn start_runtime_workers(
image_jobs::aifarm_draw_api_config_from_app_config(config),
)
.with_workflow_key(image_jobs::IMAGE_GENERATION_BOOGU_TURBO_WORKFLOW_KEY);
let routed_vip_moss_image_generator = image_jobs::RoutedImageGenerator::new(
image_attempt_walker.clone(),
image_jobs::aifarm_draw_api_config_from_app_config(config),
)
.with_workflow_key(image_jobs::IMAGE_GENERATION_MOSS_WORKFLOW_KEY);
let vip_image_generator = image_jobs::OptimizingImageGenerator::new(
image_jobs::ParallelImageGenerator::new(
routed_vip_flux_image_generator,
routed_vip_boogu_image_generator,
image_jobs::ParallelImageGenerator::new(
routed_vip_flux_image_generator,
routed_vip_boogu_image_generator,
),
routed_vip_moss_image_generator,
),
media_prompt_optimizer.clone(),
)
Expand Down
45 changes: 45 additions & 0 deletions crates/openplotva-media/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ pub enum ImageModel {
Flux3,
/// Krea 2 Turbo: one natural-prose paragraph, medium and style first.
Krea2,
/// MOSS-Image SFT: a detailed caption in the form of its recaptioner, read by a
/// Qwen3.5 encoder.
Moss,
}

impl ImageModel {
Expand All @@ -77,6 +80,7 @@ impl ImageModel {
match self {
Self::Flux3 => "FLUX.3 [klein]",
Self::Krea2 => "Krea 2 Turbo",
Self::Moss => "MOSS-Image",
}
}

Expand All @@ -86,6 +90,7 @@ impl ImageModel {
match value.trim().to_ascii_lowercase().as_str() {
"flux3" => Some(Self::Flux3),
"krea2" => Some(Self::Krea2),
"moss" => Some(Self::Moss),
_ => None,
}
}
Expand All @@ -96,6 +101,7 @@ impl ImageModel {
match self {
Self::Flux3 => FLUX3_PROMPT_MAX_CHARS,
Self::Krea2 => KREA2_PROMPT_MAX_CHARS,
Self::Moss => MOSS_PROMPT_MAX_CHARS,
}
}
}
Expand All @@ -119,6 +125,7 @@ impl Default for ImageTargets {
impl ImageTargets {
pub const FLUX3: Self = Self::single(ImageModel::Flux3);
pub const KREA2: Self = Self::single(ImageModel::Krea2);
pub const MOSS: Self = Self::single(ImageModel::Moss);

#[must_use]
pub const fn single(model: ImageModel) -> Self {
Expand Down Expand Up @@ -417,6 +424,9 @@ pub const FLUX3_PROMPT_MAX_CHARS: usize = 4000;
/// Krea 2 rules ask for up to 200 words.
pub const KREA2_PROMPT_MAX_CHARS: usize = 1600;

/// MOSS-Image rules ask for up to about 350 words, like FLUX.3.
pub const MOSS_PROMPT_MAX_CHARS: usize = 4000;

/// Cut every optimized output to at most `max_chars` characters. The bound is applied
/// after decoding rather than as a schema `maxLength`: a bounded string makes the
/// structured-output grammar of large-vocabulary engines slow to compile and decode.
Expand Down Expand Up @@ -448,6 +458,7 @@ fn optimizer_prompt_data(options: OptimizePromptOptions) -> Value {
"multi": variant_count > 1,
"flux3": models.contains(&ImageModel::Flux3),
"krea2": models.contains(&ImageModel::Krea2),
"moss": models.contains(&ImageModel::Moss),
})
}

Expand Down Expand Up @@ -906,6 +917,10 @@ mod tests {
ImageModel::from_prompt_target("krea2"),
Some(ImageModel::Krea2)
);
assert_eq!(
ImageModel::from_prompt_target("moss"),
Some(ImageModel::Moss)
);
for retired in ["klein", "boogu", "qwen_image", "dall-e"] {
assert_eq!(ImageModel::from_prompt_target(retired), None);
}
Expand All @@ -924,6 +939,36 @@ mod tests {
ImageTargets::KREA2.prompt_max_chars(),
KREA2_PROMPT_MAX_CHARS
);
assert_eq!(ImageTargets::MOSS.prompt_max_chars(), MOSS_PROMPT_MAX_CHARS);
}

#[test]
fn moss_slots_get_their_own_rules_and_example() {
let moss = render_image_optimizer_prompt(OptimizePromptOptions {
variant_count: 1,
targets: ImageTargets::MOSS,
})
.expect("render moss");
assert!(moss.contains("`outputs[0]` is rendered by MOSS-Image"));
assert!(moss.contains("**MOSS-Image**"));
assert!(moss.contains("single MOSS-Image slot"));
assert!(!moss.contains("**FLUX.3 [klein]**"));
assert!(!moss.contains("**Krea 2 Turbo**"));
assert!(!moss.contains("{{"));

let vip = render_image_optimizer_prompt(OptimizePromptOptions {
variant_count: 3,
targets: ImageTargets::KREA2
.followed_by(1, ImageTargets::FLUX3, 1)
.followed_by(2, ImageTargets::MOSS, 1),
})
.expect("render vip");
assert!(vip.contains("`outputs[0]` is rendered by Krea 2 Turbo"));
assert!(vip.contains("`outputs[1]` is rendered by FLUX.3 [klein]"));
assert!(vip.contains("`outputs[2]` is rendered by MOSS-Image"));
assert!(vip.contains("**Krea 2 Turbo**"));
assert!(vip.contains("**FLUX.3 [klein]**"));
assert!(vip.contains("**MOSS-Image**"));
}

#[test]
Expand Down
16 changes: 16 additions & 0 deletions migrations/209_moss_image_slot.down.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
DELETE FROM workflow_assignments
WHERE workflow_key = 'image_generation_moss'
AND inference_overrides ? 'created_by_moss_image_slot_v1';

DELETE FROM provider_models
WHERE config ? 'created_by_moss_image_slot_v1'
AND NOT EXISTS (
SELECT 1 FROM workflow_assignments
WHERE workflow_assignments.provider_model_id = provider_models.id
);

DELETE FROM workflows
WHERE key = 'image_generation_moss'
AND NOT EXISTS (
SELECT 1 FROM workflow_assignments WHERE workflow_key = 'image_generation_moss'
);
Loading
Loading