From 4e2312cdc462ffc33b2eec81ff82f3cdcb792e62 Mon Sep 17 00:00:00 2001 From: Youpeng Date: Wed, 9 Sep 2026 20:44:05 -0400 Subject: [PATCH 1/5] update preview version --- README.md | 2 ++ run.sh | 10 ++++++ scripts/import-docs.mjs | 43 +++++++++++++++++++++++ src/components/PageTitle.astro | 6 +++- src/content/blog/introducing-fluxserve.md | 29 +++++++++++++++ src/pages/blog/[...slug].astro | 6 ++++ src/pages/index.astro | 14 +++----- src/styles/site.css | 14 ++++---- tests/import-docs.test.mjs | 15 +++++++- 9 files changed, 122 insertions(+), 17 deletions(-) create mode 100755 run.sh create mode 100644 src/content/blog/introducing-fluxserve.md diff --git a/README.md b/README.md index 6eb59c3..f8b1e49 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ The importer rewrites links between public docs, copies referenced image/PDF ass The disposable source cache is under `.cache/fluxserve/main`. The importer only resets this cache; it never resets the local FluxServe checkout. +The three LLaDA serving guides are combined during import into `src/content/docs/docs/serving/model-recipes.md`, published at `/docs/serving/model-recipes/`. Each model has its own section; links to the original guides are rewritten to the corresponding sections. Edit the upstream recipe files to update their contents. + ## Writing a blog post Create `src/content/blog/your-post.md`. The filename becomes `/blog/your-post/`. Use lowercase, hyphenated filenames. diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..549c32c --- /dev/null +++ b/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +# Stop a previous static preview, rebuild local content, and serve the new output. +npm run preview -- stop || true +npm exec -- astro build +npm run preview diff --git a/scripts/import-docs.mjs b/scripts/import-docs.mjs index 64daad8..68d4306 100644 --- a/scripts/import-docs.mjs +++ b/scripts/import-docs.mjs @@ -117,6 +117,7 @@ export async function importDocs({ sourceDir, outputDir, publicDir, config, loca }); } // Validate everything before replacing the generated output. + consolidateModelRecipes(rendered); for (const asset of assets) await stat(within(sourceDir, asset)); await rm(outputDir, { recursive: true, force: true }); await rm(publicDir, { recursive: true, force: true }); @@ -133,6 +134,48 @@ export async function importDocs({ sourceDir, outputDir, publicDir, config, loca return { pages: rendered.length, assets: assets.size }; } +export function consolidateModelRecipes(rendered) { + const recipes = [ + ['llada2-mini', 'LLaDA2.0 Mini'], + ['llada2-flash', 'LLaDA2.0 Flash'], + ['llada2.1', 'LLaDA2.1'], + ]; + const selected = recipes.map(([slug]) => rendered.find(page => page.file === `docs/serving/${slug}.md`)); + if (selected.some(page => !page)) return; + const destination = '/docs/serving/model-recipes/'; + const links = new Map(); + const slugger = new GithubSlugger(); + const sections = selected.map((page, i) => { + const [slug, title] = recipes[i]; + const route = `/docs/serving/${slug}/`; + const sectionId = slugger.slug(title); + links.set(route, destination + '#' + sectionId); + links.set(route + '#_top', destination + '#' + sectionId); + const body = page.content.replace(/^---\n[\s\S]*?\n---\n/, ''); + const tree = processor.parse(body); + const oldSlugger = new GithubSlugger(); + visit(tree, 'heading', node => { + const heading = textOf(node); + links.set(route + '#' + oldSlugger.slug(heading), destination + '#' + slugger.slug(heading)); + node.depth = Math.min(6, node.depth + 1); + }); + return '## ' + title + '\n\n' + processor.stringify(tree); + }); + for (const page of selected) rendered.splice(rendered.indexOf(page), 1); + rendered.push({ + file: 'docs/serving/model-recipes.md', + content: '---\ntitle: Model Recipes\ndescription: Serving and benchmarking recipes for LLaDA2.0 Mini, LLaDA2.0 Flash, and LLaDA2.1.\neditUrl: false\n---\n\n' + sections.join('\n'), + }); + for (const page of rendered) { + const frontmatter = page.content.match(/^---\n[\s\S]*?\n---\n/)[0]; + const tree = processor.parse(page.content.slice(frontmatter.length)); + visit(tree, node => { + if (['link', 'definition'].includes(node.type) && links.has(node.url)) node.url = links.get(node.url); + }); + page.content = frontmatter + '\n' + processor.stringify(tree); + } +} + export async function fetchMain(sourceDir, repositoryURL) { await mkdir(sourceDir, { recursive: true }); const git = (...args) => execFileSync('git', args, { cwd: sourceDir, stdio: 'pipe' }).toString().trim(); diff --git a/src/components/PageTitle.astro b/src/components/PageTitle.astro index a535dd7..dafe676 100644 --- a/src/components/PageTitle.astro +++ b/src/components/PageTitle.astro @@ -2,11 +2,15 @@ import DefaultPageTitle from '@astrojs/starlight/components/PageTitle.astro'; import PageActions from './PageActions.astro'; const { hasSidebar, entry } = Astro.locals.starlightRoute; +const isBlogPost = Astro.url.pathname.startsWith('/blog/') && Astro.url.pathname !== '/blog/'; --- {Astro.url.pathname !== '/' && ( - hasSidebar ?
: + hasSidebar + ?
+ :
)} diff --git a/src/content/blog/introducing-fluxserve.md b/src/content/blog/introducing-fluxserve.md new file mode 100644 index 0000000..24b1c30 --- /dev/null +++ b/src/content/blog/introducing-fluxserve.md @@ -0,0 +1,29 @@ +--- +title: "FluxServe: A Flexible and High-Performance Inference Engine for Open Diffusion Language Models" +description: We are execited to announce FluxServe, a new inference engine for open diffusion langauge models. FluxServe is desigend and implemented to deliver low-latency and high-throughput for autoregressive (AR) diffusion models with optimized attention runtime, dynamic block-level scheduling and efficient multi-GPU serving. +date: 2026-09-15 +author: FluxServe Team +draft: false +--- + +Recently, diffusion large language models (dLLMs) have emerged as a highly promising alternative paradigm to their AR-based LLM counterparts. +The rapidly growing popularity of dLLMs stems from their unique architectural advantages of combining long-horizon casual sequencing with high-fidelity iterative refinement, where multiple tokens can be denoised at once in a bidirectional manner. +Exemplified by models such as LLaDA, Diffusion-Gemma, and Mercury, dLLMs offer compelling generative capabilities with better compute utilization that promotes a more efficient token economy for modern AI workloads. + +### FluxServe Overview + +Diffusion language models refine multiple token positions over a sequence of steps. That workload behaves differently from traditional autoregressive decoding, where generation advances one token at a time. + +FluxServe uses block-level scheduling to keep requests moving efficiently while supporting variable-length prefill and decoding. Paged KV caching and CUDA graph support help reduce overhead along the way. + +### Performance Results + + +### Acknowledgements + + +### What comes next + +We plan to use this blog for engineering notes, release updates, benchmarks, and deeper explanations of the ideas behind FluxServe. + +For a closer look at the system today, read the [architecture overview](/docs/architecture/). diff --git a/src/pages/blog/[...slug].astro b/src/pages/blog/[...slug].astro index 0d477c5..d20c7ca 100644 --- a/src/pages/blog/[...slug].astro +++ b/src/pages/blog/[...slug].astro @@ -1,6 +1,7 @@ --- import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; import { getCollection, render, type CollectionEntry } from 'astro:content'; +import BenchmarkChart from '../../components/BenchmarkChart.astro'; import { publishedPosts } from '../../lib/blog.mjs'; export async function getStaticPaths() { return publishedPosts(await getCollection('blog')).map(post => ({ params: {slug: post.id}, props: {post} })); @@ -11,6 +12,11 @@ const { Content, headings } = await render(post); ---

{post.data.author} ·

+ {post.id === 'introducing-fluxserve' &&
}

← All posts

 + + diff --git a/src/pages/index.astro b/src/pages/index.astro index c097de2..6d8c311 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -1,30 +1,26 @@ --- import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; import { Code } from '@astrojs/starlight/components'; -import BenchmarkChart from '../components/BenchmarkChart.astro'; const install = 'docker pull flxoss/fluxserve:v0.1-cu130-fa4'; --- - +
-

OPEN SOURCE · DIFFUSION INFERENCE

-

Serving diffusion,
one block
at a time.

+

FluxServe
Flexible Diffusion
LLM Inference.

A lightweight serving engine for diffusion language models. From your first request to multi-GPU inference.

Get started View on GitHub
-

Docker setup ↗

-

INSIDE FLUXSERVE

Built for the way diffusion generates.

Attention, scheduling, and execution. Designed around the block.

-
01 / ATTENTION

Every block counts.

Block-causal attention for variable-length prefill and decoding, with paged KV caching and CUDA graph support.

Explore block attention
-
02 / SCHEDULING

Keep requests moving.

A C++ control plane and Python execution plane coordinate requests with fine-grained, block-level scheduling.

Understand the scheduler
-
03 / EXECUTION

Room for larger models.

Tensor, data, and expert parallelism bring diffusion inference to documented multi-GPU configurations.

Deploy across GPUs
+
01 / ATTENTION

Every block counts.

Block-causal attention for variable-length prefill and decoding, with native FlashInfer and FA4 support.

+
02 / SCHEDULING

Keep requests moving.

A C++ control plane and Python execution plane coordinate requests with fine-grained, block-level scheduling.

+
03 / EXECUTION

Room for larger models.

Tensor, data, and expert parallelism bring diffusion inference to documented multi-GPU configurations.

diff --git a/src/styles/site.css b/src/styles/site.css index c461ef3..1880907 100644 --- a/src/styles/site.css +++ b/src/styles/site.css @@ -49,7 +49,7 @@ a:focus-visible, button:focus-visible, select:focus-visible { outline: 2px solid .flux-header { display: flex; align-items: center; gap: 2.75rem; height: 100%; max-width: 94rem; margin: auto; } .brand { display: block; flex: 0 0 auto; line-height: 0; border-radius: 4px; } .brand img { width: 155px; height: auto; display: block; } -[data-theme='dark'] .brand { background: #fff; padding: 7px 10px; } +[data-theme='dark'] .brand { background: transparent; padding: 7px 10px; } [data-theme='dark'] .brand img { width: 135px; } .primary-nav { display: flex; align-items: center; gap: 1.75rem; white-space: nowrap; } .primary-nav a { color: var(--sl-color-gray-2); font-weight: 500; text-decoration: none; font-size: .9rem; padding: .5rem 0; } @@ -67,11 +67,12 @@ html:has(.landing) main { padding-top: 0; } .eyebrow { font-family: var(--sl-font-mono); font-size: .75rem; font-weight: 500; letter-spacing: .1em; color: var(--muted); } .status-dot, .small-dot { display: inline-block; width: 7px; height: 7px; border-radius: 50%; background: var(--brand); } .status-dot { margin-right: 8px; } -.hero-section { display: grid; grid-template-columns: 1.06fr 1fr; gap: 3.4rem; align-items: center; padding: 6.25rem 0 5.5rem; } +.hero-section { display: grid; grid-template-columns: 1fr; gap: 3.4rem; align-items: center; padding: 6.25rem 0 5.5rem; } .hero-copy { min-width: 0; } .hero-copy h1 { font-size: clamp(2.9rem, 4.65vw, 4.4rem); letter-spacing: -.057em; line-height: 1.045; font-weight: 730; margin: 1.6rem 0 1.5rem; } -.hero-copy h1 > span { background: linear-gradient(115deg,#f46e58,#da2057); background-clip: text; color: transparent; } -[data-theme='dark'] .hero-copy h1 > span { background-image: linear-gradient(115deg,#ff9379,#fc6c9a); } +.hero-copy h1 > span:not(.hero-tagline) { background: linear-gradient(115deg,#f46e58,#da2057); background-clip: text; color: transparent; } +[data-theme='dark'] .hero-copy h1 > span:not(.hero-tagline) { background-image: linear-gradient(115deg,#ff9379,#fc6c9a); } +.hero-copy h1 > .hero-tagline { background: none; color: #e2e2e2; } .hero-description { max-width: 29rem; font-size: 1.14rem; line-height: 1.75; color: var(--muted); } .hero-actions { display: flex; gap: .85rem; flex-wrap: wrap; margin: 1.9rem 0 1.1rem; } .button { display: inline-flex; align-items: center; justify-content: center; gap: 1.4rem; min-height: 48px; padding: .7rem 1.15rem; border-radius: 7px; font-size: .94rem; font-weight: 600; } @@ -96,7 +97,7 @@ html:has(.landing) main { padding-top: 0; } .scheduler-symbol { display: flex; flex-direction: column; gap: 5px; justify-content: center; } .scheduler-symbol i { height: 6px; border-radius: 2px; background: var(--brand); }.scheduler-symbol i:nth-child(2) { width: 70%; opacity: .6; }.scheduler-symbol i:nth-child(3) { width: 85%; opacity: .8; }.scheduler-symbol i:nth-child(4) { width: 50%; opacity: .4; } .gpu-symbol { display: grid; grid-template-columns: 1fr 1fr; gap: 7px; }.gpu-symbol i { border: 2px solid var(--brand); border-radius: 4px; background: var(--sl-color-accent-low); } -.capability-grid h3 { font-size: 1.25rem; letter-spacing: -.025em; font-weight: 650; margin: 0 0 .65rem; } +.capability-grid h3 { font-size: 1.1rem; letter-spacing: -.025em; font-weight: 650; margin: 0 0 .65rem; } .capability-grid p { font-size: .92rem; color: var(--muted); line-height: 1.75; margin-bottom: 1.5rem; } .capability-grid a { font-size: .85rem; color: var(--brand); margin-top: auto; } .capability-grid a span { margin-left: .3rem; } @@ -106,7 +107,8 @@ html:has(.landing) main { padding-top: 0; } .start-links a:hover { color: var(--brand); }.start-links span { font: .75rem var(--sl-font-mono); color: var(--muted); }.start-links b { margin-left: auto; font-weight: 400; } .section-intro { max-width: 47rem; color: var(--muted); font-size: 1.05rem; line-height: 1.8; }.section-intro p + p { margin-top: .75rem; }.section-intro a { color: var(--brand); } .blog-empty { text-align: center; padding: 5rem 1.5rem; margin-top: 2.5rem; background: var(--surface); border: 1px solid var(--line); border-radius: 10px; }.empty-mark { color: var(--brand); font: 2.5rem var(--sl-font-mono); }.blog-empty h2 { font-size: 1.8rem; letter-spacing: -.035em; margin: 1rem 0 .7rem; }.blog-empty p { color: var(--muted); line-height: 1.8; }.blog-empty > a { display: inline-flex; margin-top: 1.5rem; color: var(--brand); gap: 1rem; }.blog-empty > a.primary { color: #fff; } -.post-list article { padding: 2.5rem 0; border-bottom: 1px solid var(--line); }.post-list time,.post-list article > span,.post-meta { font-size: .875rem; color: var(--muted); }.post-list h2 { font-size: 1.7rem; margin: .5rem 0; }.post-list a { color: var(--sl-color-white); }.post-list p { color: var(--muted); margin-bottom: 1rem; } +.post-list article { padding: 2.5rem 0; border-bottom: 1px solid var(--line); }.post-list time,.post-list article > span,.post-meta { font-size: .875rem; color: var(--muted); }.post-list h2 { font-size: 1rem; margin: .5rem 0; }.post-list a { color: var(--sl-color-white); }.post-list p { color: var(--muted); margin-bottom: 1rem; } +html:has(.post-meta) .sl-markdown-content h3 { font-size: 1.1rem; line-height: 1.4; } @media(min-width: 80rem) { .flux-header { padding-inline: .6rem; } } @media(max-width: 65rem) { .flux-header { gap: 1.5rem; }.primary-nav { gap: 1rem; }.header-tools { gap: .6rem; }.header-tools site-search > button { width: auto; }.hero-section { gap: 2rem; padding-block: 4rem; }.hero-copy h1 { font-size: 3.5rem; }.capability-grid article { padding: 1.4rem; } } @media(max-width: 49.99rem) { diff --git a/tests/import-docs.test.mjs b/tests/import-docs.test.mjs index b45cd76..38572cc 100644 --- a/tests/import-docs.test.mjs +++ b/tests/import-docs.test.mjs @@ -4,7 +4,20 @@ import assert from 'node:assert/strict'; import { mkdtemp, mkdir, writeFile, readFile, rm, stat } from 'node:fs/promises'; import path from 'node:path'; import os from 'node:os'; -import { importDocs, validateSource, fetchMain } from '../scripts/import-docs.mjs'; +import { importDocs, validateSource, fetchMain, consolidateModelRecipes } from '../scripts/import-docs.mjs'; + +test('combines recipes and rewrites links to duplicate heading anchors', () => { + const pages = ['llada2-mini', 'llada2-flash', 'llada2.1'].map(slug => ({ + file: `docs/serving/${slug}.md`, + content: '---\ntitle: Recipe\n---\n\n## Launch\n\n```bash\nserve model\n```\n', + })); + pages.push({file: 'docs/index.md', content: '---\ntitle: Docs\n---\n\n[Flash](/docs/serving/llada2-flash/#launch)\n'}); + consolidateModelRecipes(pages); + assert.equal(pages.length, 2); + assert.match(pages[0].content, /model-recipes\/#launch-1/); + assert.equal(pages[1].file, 'docs/serving/model-recipes.md'); + assert.equal(pages[1].content.match(/serve model/g).length, 3); +}); async function fixture(t, markdown = '# Intro\n\n[Other](other.md#details)\n\n![Plot](../assets/plot.png)\n\n[Code](../runtime.js)\n') { const dir = await mkdtemp(path.join(os.tmpdir(), 'flux-docs-')); From d548e72cfe87c021cfc4bfae995ceb7b60c8e2db Mon Sep 17 00:00:00 2001 From: Youpeng Date: Thu, 10 Sep 2026 20:38:29 -0400 Subject: [PATCH 2/5] fix: update model recipes --- astro.config.mjs | 11 +- scripts/check-site.mjs | 3 + src/components/BenchmarkChart.astro | 2 +- src/components/Header.astro | 32 +++++- src/components/ModelRecipeSelector.astro | 129 ++++++++++++++++++++++ src/components/PageActions.astro | 19 ---- src/components/PageTitle.astro | 7 +- src/content/blog/introducing-fluxserve.md | 2 +- src/pages/blog/index.astro | 3 +- src/pages/index.astro | 15 +-- src/styles/site.css | 45 ++++++-- 11 files changed, 222 insertions(+), 46 deletions(-) create mode 100644 src/components/ModelRecipeSelector.astro diff --git a/astro.config.mjs b/astro.config.mjs index b391880..c50ed29 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -5,6 +5,11 @@ export default defineConfig({ site: 'https://flx-oss.github.io', trailingSlash: 'always', output: 'static', + redirects: { + '/docs/': '/docs/getting_started/', + '/docs/llada2.0-flash/': '/docs/model-recipes/', + '/docs/model-recipes/llada2.0-flash/': '/docs/model-recipes/', + }, cacheDir: './.astro/cache', integrations: [starlight({ title: 'FluxServe', @@ -18,6 +23,10 @@ export default defineConfig({ PageTitle: './src/components/PageTitle.astro', Footer: './src/components/Footer.astro', }, - sidebar: [{ autogenerate: { directory: 'docs' } }], + sidebar: [ + { label: 'Getting Started', slug: 'docs/getting_started' }, + { label: 'Configuration', slug: 'docs/configuration' }, + { label: 'Model Recipes', slug: 'docs/model-recipes' }, + ], })], }); diff --git a/scripts/check-site.mjs b/scripts/check-site.mjs index 8736300..df675b3 100644 --- a/scripts/check-site.mjs +++ b/scripts/check-site.mjs @@ -19,11 +19,14 @@ for (const file of files.filter(file => file.endsWith('.html'))) { const failures = []; for (const [file, $] of html) { const pathname = '/' + path.relative(root, file).replace(/index\.html$/, ''); + const isRedirect = $('meta[http-equiv="refresh"]').length > 0; + if (!isRedirect) { if ($('h1').length !== 1) failures.push(pathname + ': expected one h1'); if (!$('title').text() || !$('meta[name="description"]').attr('content')) failures.push(pathname + ': missing title or description'); if (!$('link[rel="canonical"]').attr('href')?.startsWith('https://flx-oss.github.io/')) failures.push(pathname + ': missing canonical URL'); + } for (const element of $('a[href],img[src],script[src],link[rel="stylesheet"][href]').toArray()) { const url = $(element).attr('href') ?? $(element).attr('src'); if (!url || /^(https?:|mailto:|tel:|data:|\/\/)/i.test(url)) continue; diff --git a/src/components/BenchmarkChart.astro b/src/components/BenchmarkChart.astro index 7bc971d..4a553f6 100644 --- a/src/components/BenchmarkChart.astro +++ b/src/components/BenchmarkChart.astro @@ -37,7 +37,7 @@ const x = (rate: number) => 60 + (rate - 1) / 16 * 420;

LLaDA2.0-{chart.model.toLowerCase()} · {chart.config}

; })} - + diff --git a/src/components/ModelRecipeSelector.astro b/src/components/ModelRecipeSelector.astro new file mode 100644 index 0000000..d78f430 --- /dev/null +++ b/src/components/ModelRecipeSelector.astro @@ -0,0 +1,129 @@ +--- +const initialCommand = [ + 'fluxserve serve \\', + ' --model inclusionAI/LLaDA2.0-mini \\', + ' --tp-size 1 \\', + ' --dp-size 1 \\', + ' --ep-size 1 \\', + ' --attention-backend flashinfer', +].join('\n'); +--- + +
+
+ + 
{initialCommand}
+
+
+
Model
+
Hardware
+
Variant
+
Attention Backend
+
Parallelism Settings
+
+

Hardware selects the target label; FluxServe detects the installed GPU. Adjust the checkpoint in the command as needed.

+

+ + + diff --git a/src/components/PageActions.astro b/src/components/PageActions.astro index 2b12a23..eeaa47c 100644 --- a/src/components/PageActions.astro +++ b/src/components/PageActions.astro @@ -6,34 +6,15 @@ const markdownURL = '/markdown/' + id.split('/').map(encodeURIComponent).join('/ ---
-
View as Markdown ↗
-
+ +
+ + Single-model scaling performance + Placeholder data comparing FluxServe with a baseline. Both lines slope downward as user input tokens per second increase, while FluxServe maintains higher output tokens per second. + + + {yTicks.map(tick => + + {tick.toLocaleString('en-US')} + )} + {xTicks.map(tick => + + {tick} + )} + + + User/Input/Tok/S + Output/Tok/S + + + + + + {series.map(item => + `${x(input)},${y(output)}`).join(' ')} /> + {item.values.map(([input, output], index) => )} + )} + + +
+ +
+ Design preview: values are placeholders intended to demonstrate the visualization and animation. + Benchmark Guide ↗ +
+ + + + diff --git a/src/components/Header.astro b/src/components/Header.astro index 07c55d7..c2cbde9 100644 --- a/src/components/Header.astro +++ b/src/components/Header.astro @@ -5,12 +5,13 @@ const links = [ { href: '/docs/getting_started/', label: 'Getting Started' }, { href: '/docs/configuration/', label: 'Configuration' }, { href: '/docs/model-recipes/', label: 'Model Recipes' }, + { href: '/docs/benchmark-guide/', label: 'Benchmark Guide' }, { href: '/blog/', label: 'Blog' }, ]; ---
- FluxServe + FluxServe
@@ -29,6 +30,10 @@ const links = [ + diff --git a/src/components/SpeedBenchmarkChart.astro b/src/components/SpeedBenchmarkChart.astro new file mode 100644 index 0000000..99eebea --- /dev/null +++ b/src/components/SpeedBenchmarkChart.astro @@ -0,0 +1,98 @@ +--- +interface Props { compact?: boolean; } +const { compact = false } = Astro.props; + +// Illustrative placeholder data for the blog design. Replace with measured results. +const results = [ + { name: 'FluxServe', value: 1100, featured: true }, + { name: 'SGLang', value: 610, featured: false }, + { name: 'vLLM', value: 480, featured: false }, +]; +const maxValue = Math.max(...results.map(result => result.value)); +const barWidth = (value: number) => value / maxValue * 760; +const verticalTicks = [0, 400, 800, 1200]; +const verticalY = (value: number) => 500 - value / 1200 * 360; +const verticalX = (index: number) => 92 + index * 164; +--- +
+
+ {compact && + Speed benchmark + Placeholder output speed comparison. FluxServe reaches 1100 tokens per second, SGLang reaches 610, and vLLM reaches 480. + + + Speed Benchmark + Tokens/sec + + + {verticalTicks.map(tick => + + {tick.toLocaleString('en-US')} + )} + + {results.map((result, index) => { + const top = verticalY(result.value); + return + + {result.value.toLocaleString('en-US')} + {result.name} + ; + })} + } + {!compact && + Speed benchmark + Placeholder output speed comparison. FluxServe reaches 1100 tokens per second, SGLang reaches 610, and vLLM reaches 480. + + + + Speed Benchmark + Tokens/sec + + + {results.map((result, index) => { + const top = 108 + index * 88; + return + + {result.name} + {result.value.toLocaleString('en-US')} + ; + })} + } +
+
Design preview: values are placeholders and do not represent measured performance.
+
+ + diff --git a/src/content/blog/introducing-fluxserve.md b/src/content/blog/introducing-fluxserve.md index 7be432a..5c68006 100644 --- a/src/content/blog/introducing-fluxserve.md +++ b/src/content/blog/introducing-fluxserve.md @@ -18,6 +18,7 @@ FluxServe uses block-level scheduling to keep requests moving efficiently while ### Performance Results +
### Acknowledgements diff --git a/src/lib/model-recipes.ts b/src/lib/model-recipes.ts new file mode 100644 index 0000000..9a6937d --- /dev/null +++ b/src/lib/model-recipes.ts @@ -0,0 +1,216 @@ +/** Edit model metadata and available choices here. */ +export interface ModelRecipe { + label: string; + checkpoint: string; + variants: Record; + variantLinks: Record; + parallelism: string[]; +} + +interface VariantLink { + label: string; + href: string; +} + +const placeholderLink = (model: string, variant: string): VariantLink => ({ + label: `placeholder/${model}-${variant.toLowerCase()}`, + href: `https://huggingface.co/placeholder/${model}-${variant.toLowerCase()}`, +}); + +export const models: Record = { + 'LLaDA2.0-mini': { + label: 'LLaDA2.0 Mini', + checkpoint: 'inclusionAI/LLaDA2.0-mini', + variants: { BF16: 'BF16 33GB'}, + variantLinks: { BF16: + { + label: 'inclusionAI/LLaDA2.0-mini', + href: 'https://huggingface.co/inclusionAI/LLaDA2.0-mini', + }, + }, + parallelism: ['1', '4'], + }, + 'LLaDA2.0-flash': { + label: 'LLaDA2.0 Flash', + checkpoint: 'inclusionAI/LLaDA2.0-flash', + variants: { BF16: 'BF16 206GB'}, + variantLinks: { BF16: + { + label: 'inclusionAI/LLaDA2.0-flash', + href: 'https://huggingface.co/inclusionAI/LLaDA2.0-flash', + }, + }, + parallelism: ['4'], + }, + 'LLaDA2.1-mini': { + label: 'LLaDA2.1 Mini', + checkpoint: 'inclusionAI/LLaDA2.1-mini', + variants: { BF16: 'BF16 33 GB'}, + variantLinks: { BF16: + { + label: 'inclusionAI/LLaDA2.1-mini', + href: 'https://huggingface.co/inclusionAI/LLaDA2.1-mini', + }, + }, + parallelism: ['1', '4'], + }, + 'LLaDA2.1-flash': { + label: 'LLaDA2.1 Flash', + checkpoint: 'inclusionAI/LLaDA2.1-flash', + variants: { BF16: 'BF16 206GB'}, + variantLinks: { BF16: + { + label: 'inclusionAI/LLaDA2.1-flash', + href: 'https://huggingface.co/inclusionAI/LLaDA2.1-flash', + }, + }, + parallelism: ['4'], + }, + 'diffusion-gemma': { + label: 'Diffusion-Gemma', + checkpoint: 'google/diffusiongemma-26B-A4B-it', + variants: { BF16: 'BF16 52GB', FP8: 'FP8 27GB', NVFP4: 'NVFP4 19GB' }, + variantLinks: { + BF16: { + label: 'google/diffusiongemma-26B-A4B-it', + href: 'https://huggingface.co/google/diffusiongemma-26B-A4B-it', + }, + FP8: { + label: 'RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic', + href: 'https://huggingface.co/RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic', + }, + NVFP4: { + label: 'nvidia/diffusiongemma-26B-A4B-it-NVFP4', + href: 'https://huggingface.co/nvidia/diffusiongemma-26B-A4B-it-NVFP4', + }, + }, + parallelism: ['1', '4'], + }, +}; + +export interface Selection { + model: string; + hardware: string; + variant: string; + backend: string; + parallel: string; +} +export type Setting = keyof Selection; +interface Option { + value: string; + label: string; + verified?: boolean; +} +interface Group { + key: Setting; + label: string; + options: Option[]; +} + +/** Array order controls the order of selector sections. */ +export const groups: Group[] = [ + { + key: 'model', + label: 'Model', + options: Object.entries(models).map(([value, model]) => ({ value, label: model.label })), + }, + { + key: 'hardware', + label: 'Hardware', + options: [ + { value: 'H100', label: 'H100' }, + { value: 'H200', label: 'H200' }, + { value: 'GH200', label: 'GH200', verified: true }, + { value: 'B200', label: 'B200', verified: true }, + ], + }, + { + key: 'variant', + label: 'Variant', + options: ['BF16', 'FP8', 'NVFP4'].map(value => ({ value, label: value })), + }, + { + key: 'backend', + label: 'Attention Backend', + options: [ + { value: 'flashinfer', label: 'FlashInfer' }, + { value: 'fa4', label: 'FA4' }, + ], + }, + { + key: 'parallel', + label: 'Parallelism Settings', + options: ['1', '4'].map(value => ({ value, label: `TP=EP=${value}` })), + }, +]; + +export const defaults: Selection = { + model: 'LLaDA2.0-mini', + hardware: 'H100', + variant: 'BF16', + backend: 'flashinfer', + parallel: '1', +}; + +export function selectOption(current: Selection, key: Setting, value: string): Selection { + const next = { ...current, [key]: value }; + const model = models[next.model]; + if (!(next.variant in model.variants)) next.variant = 'BF16'; + if (key === 'model' || !model.parallelism.includes(next.parallel)) next.parallel = model.parallelism[0]; + return next; +} + +export function optionVisible(selection: Selection, key: Setting, value: string): boolean { + const model = models[selection.model]; + if (key === 'variant') return value in model.variants; + if (key === 'parallel') return model.parallelism.includes(value); + return true; +} + +export function optionLabel(selection: Selection, key: Setting, option: Option): string { + return key === 'variant' ? models[selection.model].variants[option.value] ?? option.label : option.label; +} + +/** Keep command arguments explicit: no replacements against rendered HTML. */ +export function buildCommand(selection: Selection): string { + const args = [ + ['model', models[selection.model].checkpoint], + ['tp-size', selection.parallel], + ['dp-size', '1'], + ['ep-size', selection.parallel], + ['attention-backend', selection.backend], + ]; + return ['fluxserve serve', ...args.map(([key, value]) => ` --${key} ${value}`)].join(' \\\n'); +} + +/** Derive display state in one place for both initial HTML and browser updates. */ +export function recipeView(selection: Selection) { + const reasons: string[] = []; + if (selection.variant !== 'BF16') { + reasons.push(`${selection.variant} quantized checkpoints are not supported by the current FluxServe CLI.`); + } + if (selection.backend === 'fa4' && !selection.model.startsWith('LLaDA2.')) { + reasons.push('FA4 serving is not available for this model recipe.'); + } + const unsupported = reasons.length > 0; + const gemmaBF16 = selection.model === 'diffusion-gemma' && selection.variant === 'BF16'; + const variantLink = models[selection.model].variantLinks[selection.variant]; + const hardwareOptions = groups.find(group => group.key === 'hardware')!.options; + const verified = !unsupported && hardwareOptions.some( + option => option.value === selection.hardware && option.verified, + ); + const note = unsupported + ? 'Select a LLaDA model with BF16 and FlashInfer or FA4 for a runnable command.' + : selection.model === 'diffusion-gemma' + ? 'This recipe uses --scheduler-policy default as requested; it requires a FluxServe version accepting that policy (the local CLI currently lists paged and dynamic).' + : 'Hardware selects the target label; FluxServe detects the installed GPU. Adjust the checkpoint in the command as needed.'; + return { + command: unsupported ? reasons.map(reason => '# ' + reason).join('\n') : buildCommand(selection), + unsupported, + gemmaBF16, + variantLink, + verified, + note, + verificationLabel: 'Verified on NVIDIA ' + selection.hardware, + }; +} diff --git a/src/pages/blog/[...slug].astro b/src/pages/blog/[...slug].astro index d20c7ca..d3691e8 100644 --- a/src/pages/blog/[...slug].astro +++ b/src/pages/blog/[...slug].astro @@ -12,11 +12,21 @@ const { Content, headings } = await render(post); ---

{post.data.author} ·

- {post.id === 'introducing-fluxserve' &&
} + {post.id === 'introducing-fluxserve' && } + {post.id === 'introducing-fluxserve' && }

← All posts

 diff --git a/src/pages/index.astro b/src/pages/index.astro index 2610fa8..13764b2 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -1,27 +1,105 @@ --- import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; -import { Code } from '@astrojs/starlight/components'; -const install = 'docker pull flxoss/fluxserve:v0.1-cu130-fa4'; + +const frontmatter = { + title: 'FluxServe', + head: [{ tag: 'title' as const, content: 'FluxServe' }], + description: + 'FluxServe is a lightweight serving engine for diffusion language models. Get started with block-level scheduling, efficient attention, and multi-GPU execution.', + template: 'splash' as const, + editUrl: false as const, + lastUpdated: false as const, +}; --- - + +
-

FluxServe
Flexible and High-Performance
Diffusion LLM Inference.

-
- Get started - Model Recipes - Serving Parameters +

+ FluxServe + + Flexible and High-Performance +
+ Diffusion LLM Inference. + +

+

+ Open-source, high-throughput and cost-effectient dLLM serving engine for everyone. +

+ +
+ + Get started + + + + View on GitHub +
-
+
+
-

INSIDE FLUXSERVE

Built for the way diffusion generates.

Attention, scheduling, and execution. Designed around the block.

+
+

INSIDE FLUXSERVE

+

Built for the way diffusion generates

+
+
-
01 / ATTENTION

Every block counts.

Block-causal attention for variable-length prefill and decoding, with native FlashInfer and FA4 support.

-
02 / SCHEDULING

Keep requests moving.

A C++ control plane and Python execution plane coordinate requests with fine-grained, block-level scheduling.

-
03 / EXECUTION

Room for larger models.

Tensor, data, and expert parallelism bring diffusion inference to documented multi-GPU configurations.

+
+ 01 / ATTENTION + +

Built for Variable Lengths

+

+ Block-causal attention handles variable-length prefill and decoding efficiently, + with native FlashInfer and FA4 support +

+
+ +
+ 02 / SCHEDULING + +

Keep the Pipeline Moving

+

+ Fine-grained, block-level scheduling keeps requests flowing + across the C++ control plane and Python execution path +

+
+ +
+ 03 / EXECUTION + +

Scale Beyond One GPU

+

+ Tensor, data, and expert parallelism extend diffusion inference across multi-GPU deployments +

+
diff --git a/src/styles/site.css b/src/styles/site.css index e0735a6..4a52fd7 100644 --- a/src/styles/site.css +++ b/src/styles/site.css @@ -18,43 +18,46 @@ --sl-nav-height: 5rem; --sl-content-width: 47rem; --sl-sidebar-width: 17rem; + --page-container: 93.75rem; --brand: #f46d87; --surface: #1b1b1f; --muted: #a4a4ac; --line: #333338; } :root[data-theme='light'] { - --sl-color-accent-low: #fff0f3; + --sl-color-accent-low: #f3dce3; --sl-color-accent: #cc2454; --sl-color-accent-high: #9f1741; - --sl-color-black: #ffffff; - --sl-color-gray-6: #f7f7f8; - --sl-color-gray-5: #e7e7eb; - --sl-color-gray-4: #96969d; - --sl-color-gray-3: #65656e; - --sl-color-gray-2: #414149; - --sl-color-gray-1: #28282d; - --sl-color-white: #18181d; - --sl-color-bg-nav: #ffffff; - --sl-color-bg-sidebar: #fafafa; + --sl-color-black: #dfe8f1; + --sl-color-gray-6: #f2f6fa; + --sl-color-gray-5: #c4d0dc; + --sl-color-gray-4: #7f91a2; + --sl-color-gray-3: #596b7c; + --sl-color-gray-2: #374756; + --sl-color-gray-1: #23303c; + --sl-color-white: #16222d; + --sl-color-bg-nav: #dfe8f1; + --sl-color-bg-sidebar: #e8eff5; --brand: #c72653; - --surface: #f8f8fa; - --muted: #676770; - --line: #e6e6eb; + --surface: #f2f6fa; + --muted: #596b7c; + --line: #9eafbf; } html { -webkit-font-smoothing: antialiased; } body { font-size: 1rem; } +header.header { border-bottom-color: transparent; box-shadow: none; transition: box-shadow .18s ease; } +header.header.is-scrolled { box-shadow: 0 1px 0 color-mix(in srgb, var(--line) 55%, transparent); } a, button, select { -webkit-tap-highlight-color: transparent; } a:focus-visible, button:focus-visible, select:focus-visible { outline: 2px solid var(--brand); outline-offset: 5px; } -.flux-header { display: flex; align-items: center; gap: 2.75rem; height: 100%; max-width: 94rem; margin: auto; } +.flux-header { display: flex; align-items: center; gap: 2.75rem; width: 100%; height: 100%; } .header-navigation { display: contents; } .header-search { order: 2; } .header-navigation > .header-tools { order: 3; } .nav-toggle { display: none; } .brand { display: block; flex: 0 0 auto; line-height: 0; border-radius: 4px; } -.brand img { width: 155px; height: auto; display: block; } -[data-theme='dark'] .brand { background: transparent; padding: 7px 10px; } -[data-theme='dark'] .brand img { width: 135px; } +.brand img { width: 64px; height: 64px; object-fit: contain; display: block; } +[data-theme='dark'] .brand { background: transparent; padding: 0; } +[data-theme='dark'] .brand img { width: 64px; } .primary-nav { display: flex; align-items: center; gap: 1.75rem; margin-left: auto; white-space: nowrap; } .primary-nav a { color: var(--sl-color-gray-2); font-weight: 500; text-decoration: none; font-size: .9rem; padding: .5rem 0; } .primary-nav a:hover, .primary-nav a[aria-current] { color: var(--brand); } @@ -62,6 +65,7 @@ a:focus-visible, button:focus-visible, select:focus-visible { outline: 2px solid .header-tools site-search { min-width: 0; } .header-tools site-search > button { width: 11rem; } html:not([data-has-sidebar]) { --sl-content-width: 72rem; } +html:has(.landing) { --sl-content-width: var(--page-container); } html:not([data-has-sidebar]) .content-panel + .content-panel { border-top: 0; padding-top: 0; } html:has(.landing) .content-panel:first-child { display: none; } html:has(.landing) main { padding-top: 0; } @@ -71,16 +75,18 @@ html:has(.landing) main { padding-top: 0; } .eyebrow { font-family: var(--sl-font-mono); font-size: .75rem; font-weight: 500; letter-spacing: .1em; color: var(--muted); } .status-dot, .small-dot { display: inline-block; width: 7px; height: 7px; border-radius: 50%; background: var(--brand); } .status-dot { margin-right: 8px; } -.hero-section { display: grid; grid-template-columns: 1fr; gap: 3.4rem; align-items: center; padding: 6.25rem 0 5.5rem; } -.hero-copy { min-width: 0; } -.hero-copy h1 { font-size: clamp(2.9rem, 4.65vw, 4.4rem); letter-spacing: -.057em; line-height: 1.045; font-weight: 730; margin: 1.6rem 0 1.5rem; } +.hero-section { display: grid; grid-template-columns: 1fr; align-items: center; padding: 6.25rem 3.5rem; } +.hero-copy { width: 100%; max-width: 60rem; min-width: 0; } +.hero-copy h1 { max-width: 60rem; font-size: clamp(3.625rem, 4vw, 4.25rem); letter-spacing: -.045em; line-height: .98; font-weight: 730; margin: 1.6rem 0 1.5rem; } +.hero-brand { display: block; width: fit-content; margin-bottom: .75rem; padding-inline-end: .1em; margin-inline-end: -.1em; background: linear-gradient(115deg, #f46e58, #da2057); -webkit-background-clip: text; background-clip: text; color: transparent; -webkit-text-fill-color: transparent; } /* Leave room for the final glyph beyond the tightly tracked text box. */ .hero-copy h1 > span:not(.hero-tagline) { padding-inline-end: .1em; margin-inline-end: -.1em; background: linear-gradient(115deg,#f46e58,#da2057); background-clip: text; color: transparent; } [data-theme='dark'] .hero-copy h1 > span:not(.hero-tagline) { background-image: linear-gradient(115deg,#ff9379,#fc6c9a); } +[data-theme='dark'] .hero-brand { background-image: linear-gradient(115deg, #ff9379, #fc6c9a); } .hero-copy h1 > .hero-tagline { background: none; color: #d1d0d0; } [data-theme='light'] .hero-copy h1 > .hero-tagline { color: #292929; } .hero-description { max-width: 29rem; font-size: 1.14rem; line-height: 1.75; color: var(--muted); } -.hero-actions { display: flex; gap: .85rem; flex-wrap: wrap; margin: 1.9rem 0 1.1rem; } +.hero-actions { display: flex; gap: .85rem; flex-wrap: wrap; margin: 2.5rem 0 1.1rem; } .button { display: inline-flex; align-items: center; justify-content: center; gap: 1.4rem; min-height: 48px; padding: .7rem 1.15rem; border-radius: 7px; font-size: .94rem; font-weight: 600; } .hero-actions .button { font-size: .875rem; text-align: center; } .hero-actions .button > span:empty { display: none; } @@ -88,6 +94,8 @@ html:has(.landing) main { padding-top: 0; } .button.primary:hover { background: #ad1e46; } .button.secondary { border: 1px solid var(--line); color: var(--sl-color-white); } .button.secondary:hover { background: var(--surface); } +.github-button { gap: .6rem; } +.github-button svg { width: 1.15rem; height: 1.15rem; flex-shrink: 0; } .hero-install { margin-top: 1.6rem; min-width: 0; } .hero-install > p { font-size: .75rem; color: var(--muted); margin-bottom: .65rem; } .hero-install a { color: var(--brand); } @@ -117,8 +125,10 @@ html:has(.landing) main { padding-top: 0; } .blog-empty { text-align: center; padding: 5rem 1.5rem; margin-top: 2.5rem; background: var(--surface); border: 1px solid var(--line); border-radius: 10px; }.empty-mark { color: var(--brand); font: 2.5rem var(--sl-font-mono); }.blog-empty h2 { font-size: 1.8rem; letter-spacing: -.035em; margin: 1rem 0 .7rem; }.blog-empty p { color: var(--muted); line-height: 1.8; }.blog-empty > a { display: inline-flex; margin-top: 1.5rem; color: var(--brand); gap: 1rem; }.blog-empty > a.primary { color: #fff; } .post-list article { padding: 2.5rem 0; border-bottom: 1px solid var(--line); }.post-list time,.post-list article > span,.post-meta { font-size: .875rem; color: var(--muted); }.post-list h2 { font-size: 1rem; margin: .5rem 0; }.post-list a { color: var(--sl-color-white); }.post-list p { color: var(--muted); margin-bottom: 1rem; } html:has(.post-meta) .sl-markdown-content h3 { font-size: 1.1rem; line-height: 1.4; } -@media(min-width: 80rem) { .flux-header { padding-inline: .6rem; } } -@media(max-width: 65rem) { .flux-header { gap: 1.5rem; }.primary-nav { gap: 1rem; }.header-tools { gap: .6rem; }.header-tools site-search > button { width: auto; }.hero-section { gap: 2rem; padding-block: 4rem; }.hero-copy h1 { font-size: 3.5rem; }.capability-grid article { padding: 1.4rem; } } +@media(min-width: 80rem) { .flux-header { padding-inline: 0; } } +@media(min-width: 65.01rem) { .hero-tagline { display: block; white-space: nowrap; } } +@media(max-width: 75rem) { .hero-section { padding-inline: 0; } } +@media(max-width: 65rem) { .flux-header { gap: 1.5rem; }.primary-nav { gap: 1rem; }.header-tools { gap: .6rem; }.header-tools site-search > button { width: auto; }.hero-section { padding-block: 4rem; }.hero-copy h1 { font-size: 3.5rem; }.capability-grid article { padding: 1.4rem; } } @media(max-width: 49.99rem) { :root { --sl-nav-height: 5rem; } .flux-header { display: flex; gap: .6rem; padding-block: .3rem; } @@ -132,7 +142,7 @@ html:has(.post-meta) .sl-markdown-content h3 { font-size: 1.1rem; line-height: 1 .header-navigation .header-tools .github-link { display: grid; } .header-search site-search > button { width: 40px; padding: 0; justify-content: center; } .header-search site-search > button > :not(svg) { display: none; } - .brand img { width: 125px; }[data-theme='dark'] .brand img { width: 105px; } + .brand img { width: 54px; height: 54px; }[data-theme='dark'] .brand img { width: 54px; } .primary-nav { grid-row: 2; grid-column: 1 / -1; justify-self: end; max-width: 100%; gap: 1rem; min-width: 0; overflow-x: auto; padding: .3rem; }.primary-nav a { flex-shrink: 0; font-size: .85rem; padding: .3rem 0; } .header-tools { grid-row: 1; grid-column: 2; justify-self: end; gap: .6rem; }.github-link { display: none; } .sl-menu-button { display: none !important; } @@ -140,7 +150,7 @@ html:has(.post-meta) .sl-markdown-content h3 { font-size: 1.1rem; line-height: 1 .capability-grid { grid-template-columns: 1fr; }.capability-grid article { padding: 1.8rem; }.capability-grid article + article { border-left: 0; border-top: 1px solid var(--line); }.feature-symbol { margin-block: 1.3rem 1rem; } .start-strip { grid-template-columns: 1fr; gap: 1rem; }.start-strip h2 { max-width: none; } } -@media(max-width: 380px) { .brand img { width: 105px; }[data-theme='dark'] .brand img { width: 85px; }.header-tools { gap: .25rem; }.primary-nav { gap: 1rem; }.hero-actions { gap: .6rem; }.button { gap: .6rem; padding-inline: .85rem; } } +@media(max-width: 380px) { .brand img { width: 50px; height: 50px; }[data-theme='dark'] .brand img { width: 50px; }.header-tools { gap: .25rem; }.primary-nav { gap: 1rem; }.hero-actions { gap: .6rem; }.button { gap: .6rem; padding-inline: .85rem; } } @media(prefers-reduced-motion: reduce) { *,*::before,*::after { scroll-behavior: auto !important; animation: none !important; transition: none !important; } } .sidebar-content summary .large { text-transform: capitalize; } diff --git a/tests/model-recipes.test.mjs b/tests/model-recipes.test.mjs new file mode 100644 index 0000000..66f4abe --- /dev/null +++ b/tests/model-recipes.test.mjs @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; +import ts from 'typescript'; + +// Transpile the pure helper without needing Astro or a browser. +const source = await readFile(new URL('../src/lib/model-recipes.ts', import.meta.url), 'utf8'); +const { outputText } = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, +}); +const { defaults, groups, models, selectOption, optionVisible, optionLabel, recipeView } = + await import('data:text/javascript;base64,' + Buffer.from(outputText).toString('base64')); + +test('section order and default command are preserved', () => { + assert.deepEqual(groups.map(group => group.key), ['model', 'hardware', 'variant', 'backend', 'parallel']); + const lines = recipeView(defaults).command.split('\n'); + assert.equal(lines.length, 6); + assert.equal(lines[0], 'fluxserve serve ' + String.fromCharCode(92)); + assert.ok(lines[1].includes('--model inclusionAI/LLaDA2.0-mini')); + assert.ok(lines[2].includes('--tp-size 1')); + assert.ok(lines[3].includes('--dp-size 1')); + assert.ok(lines[4].includes('--ep-size 1')); + assert.equal(lines[5], ' --attention-backend flashinfer'); +}); + +test('model changes enforce available variants and parallelism', () => { + for (const model of Object.keys(models)) { + const selection = selectOption({ ...defaults, variant: 'FP8' }, 'model', model); + const llada = model.startsWith('LLaDA2.'); + const flash = model.endsWith('-flash'); + assert.equal(selection.variant, llada ? 'BF16' : 'FP8'); + assert.equal(selection.parallel, flash ? '4' : '1'); + assert.equal(optionVisible(selection, 'variant', 'NVFP4'), !llada); + assert.equal(optionVisible(selection, 'parallel', '1'), !flash); + } + assert.equal(defaults.parallel, '1'); +}); + +test('memory labels and Gemma detail match the selected model', () => { + for (const [model, memory] of [ + ['LLaDA2.0-mini', '33'], ['LLaDA2.1-mini', '33'], + ['LLaDA2.0-flash', '206'], ['LLaDA2.1-flash', '206'], + ['diffusion-gemma', '52'], + ]) { + const selection = selectOption(defaults, 'model', model); + const label = optionLabel(selection, 'variant', { value: 'BF16', label: 'BF16' }); + assert.equal(label.replace(/\s/g, ''), 'BF16' + memory + 'GB'); + assert.equal(recipeView(selection).gemmaBF16, model === 'diffusion-gemma'); + assert.ok(recipeView(selection).command.includes(models[model].checkpoint)); + } +}); + +test('every available model variant has a Hugging Face link', () => { + for (const [model, recipe] of Object.entries(models)) { + for (const variant of Object.keys(recipe.variants)) { + const view = recipeView({ ...defaults, model, variant }); + assert.match(view.variantLink.href, /^https:\/\/huggingface\.co\//); + assert.ok(view.variantLink.label); + } + } +}); + +test('unsupported combinations disable copying and verification', () => { + for (const hardware of ['H100', 'GH200', 'B200']) { + const selection = { ...defaults, hardware }; + assert.equal(recipeView(selection).verified, hardware !== 'H100'); + for (const change of [ + { variant: 'FP8' }, + { variant: 'NVFP4' }, + { model: 'diffusion-gemma', backend: 'fa4' }, + ]) { + const view = recipeView({ ...selection, ...change }); + assert.equal(view.unsupported, true); + assert.equal(view.verified, false); + assert.ok(view.command.startsWith('# ')); + } + } +}); + +test('all LLaDA2 recipes generate copyable FA4 commands', () => { + for (const model of Object.keys(models).filter(model => model.startsWith('LLaDA2.'))) { + const selection = selectOption(defaults, 'model', model); + for (const parallel of models[model].parallelism) { + const view = recipeView({ ...selection, parallel, backend: 'fa4' }); + assert.equal(view.unsupported, false); + assert.ok(view.command.includes('--attention-backend fa4')); + assert.ok(view.command.includes('--tp-size ' + parallel)); + assert.ok(view.command.includes('--ep-size ' + parallel)); + assert.ok(view.command.includes(models[model].checkpoint)); + } + } +}); From 0badfee2b6f44d4e95804fdacba61b8d32e0a99d Mon Sep 17 00:00:00 2001 From: Youpeng Date: Thu, 24 Sep 2026 15:17:48 -0400 Subject: [PATCH 4/5] fix: update blog --- .gitignore | 1 - README.md | 37 ++--- package.json | 7 +- src/components/BenchmarkChart.astro | 98 ------------ src/components/BlockDiffusionAnimation.astro | 116 ++++++++++++++ src/components/ModelRecipeSelector.astro | 6 +- src/components/SpeedBenchmarkChart.astro | 153 +++++++++---------- src/content/blog/introducing-fluxserve.md | 71 +++++++-- src/content/docs/docs/benchmark-guide.md | 72 +++++++++ src/content/docs/docs/configuration.md | 82 ++++++++++ src/content/docs/docs/getting_started.md | 104 +++++++++++++ src/content/docs/docs/model-recipes.md | 7 + src/lib/model-recipes.ts | 42 +++-- src/pages/blog/[...slug].astro | 9 ++ src/pages/blog/index.astro | 2 +- src/styles/site.css | 2 + tests/model-recipes.test.mjs | 56 ++++++- 17 files changed, 634 insertions(+), 231 deletions(-) create mode 100644 src/components/BlockDiffusionAnimation.astro create mode 100644 src/content/docs/docs/benchmark-guide.md create mode 100644 src/content/docs/docs/configuration.md create mode 100644 src/content/docs/docs/getting_started.md create mode 100644 src/content/docs/docs/model-recipes.md diff --git a/.gitignore b/.gitignore index ffd2f50..8341d11 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ node_modules/ dist/ .astro/ .cache/ -src/content/docs/ public/fluxserve/ output/ .playwright-cli/ diff --git a/README.md b/README.md index f8b1e49..8a7209d 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,7 @@ npm ci npm run dev ``` -Development imports documentation from the sibling `../FluxServe` checkout, including uncommitted documentation edits. To use another checkout: - -```sh -FLUXSERVE_SOURCE=/absolute/path/to/FluxServe npm run dev -``` +Development reads documentation directly from `src/content/docs/` in this repository. Search indexes are built for production, so use the static preview to test search: @@ -24,7 +20,7 @@ npm run build:local npm run preview ``` -`build:local` is a preview only. Do not upload its output to production. +`build:local` uses the same checked-in content and validation as the production build. ## Production build and verification @@ -36,7 +32,7 @@ npm test npm run preview ``` -Every production build fetches the latest FluxServe `main`. The resolved commit is used for source attribution and repository-file links; no revision pin needs updating. It does not need a sibling checkout and rejects `FLUXSERVE_SOURCE`. Generated pages, imported assets, source caches, and build outputs are ignored by Git. +Production builds use the documentation committed to this repository. They do not fetch documentation from the FluxServe repository and do not require a sibling checkout or network access beyond dependency installation. The build validates page titles, descriptions, canonical URLs, local links, heading anchors, assets, sitemap, and search output. Tests cover import failures, URL rewriting, stale-page removal, and blog publication. Blog integration tests create an isolated temporary site and verify that draft articles appear in neither routes nor search. @@ -44,21 +40,13 @@ Browser verification covers responsive navigation, theme selection and persisten ## Updating documentation -Technical content belongs in **FLX-OSS/FluxServe**, not this repository. - -1. Edit the Markdown in FluxServe and preview it using `npm run dev` or `npm run build:local`. -2. Merge the documentation changes into FluxServe `main`. -3. Run the website’s **Build and deploy website** workflow manually, or let the next website `main` push rebuild it. - -FluxServe merges do not currently trigger a website build automatically. The deployed site remains a static snapshot until the next successful build. The initial docs cleanup must be merged into FluxServe `main` before production builds can succeed. - -Every Markdown file in FluxServe’s `docs/` folder is published automatically, preserving folders and filenames: `docs/serving/llada2.1.md` becomes `/docs/serving/llada2.1/`. An `index.md` becomes its folder’s entry page; `docs/index.md` is required. Titles come from the first level-one heading and descriptions from the first paragraph. The sidebar follows the folders. Keep internal planning notes outside `docs/`; there is no website page allowlist. - -The importer rewrites links between public docs, copies referenced image/PDF assets, and points other repository-file links to the commit resolved for that build. Missing source files, asset files, and imported-page anchors fail the build. Each imported page links to its exact source revision. +Technical content is maintained directly in `src/content/docs/` in this repository. -The disposable source cache is under `.cache/fluxserve/main`. The importer only resets this cache; it never resets the local FluxServe checkout. +1. Edit or add the Markdown under `src/content/docs/docs/`. +2. Preview changes with `npm run dev`, or use `npm run build:local` followed by `npm run preview` to verify the production output. +3. Commit the documentation changes with the website changes. A successful push to `main` deploys that committed snapshot. -The three LLaDA serving guides are combined during import into `src/content/docs/docs/serving/model-recipes.md`, published at `/docs/serving/model-recipes/`. Each model has its own section; links to the original guides are rewritten to the corresponding sections. Edit the upstream recipe files to update their contents. +The directory beneath the content root determines the public route. For example, `src/content/docs/docs/configuration.md` is published at `/docs/configuration/`. Each page supplies its title, description, sidebar metadata, and optional edit link in YAML frontmatter. ## Writing a blog post @@ -83,15 +71,14 @@ Set `draft: false` when ready to publish. Drafts are excluded from the blog inde GitHub Pages must use **GitHub Actions** as its build source. The workflow checks pull requests without deploying and publishes successful `main` builds; it can also be started manually from Actions. It uses the built-in GitHub token, with Pages write permissions limited to the deployment job. -The canonical origin is `https://flx-oss.github.io` with no repository-name prefix. No custom domain is configured. To roll back documentation, revert the relevant change in FluxServe main and rebuild the website. Website code can be rolled back by reverting its commit. +The canonical origin is `https://flx-oss.github.io` with no repository-name prefix. No custom domain is configured. To roll back documentation or website code, revert the relevant commit in this repository and rebuild the website. ## Structure -- `fluxserve-docs.json`: source repository and shared brand assets. -- `scripts/`: Markdown importer and built-site link/metadata validation. -- `src/pages/`: custom homepage, blog, and 404; all documentation, including deployment guides, is imported under `/docs/`. +- `scripts/`: build validation and maintenance utilities. +- `src/pages/`: custom homepage, blog, and 404 pages. - `src/components/`, `src/styles/`: shared navigation and FluxServe styling. - `src/content/blog/`: authored blog Markdown. -- `src/content/docs/`, `public/fluxserve/`: generated and untracked. +- `src/content/docs/`: locally authored and tracked documentation Markdown. The site is English-only and publishes one documentation revision at a time. There is no server, CMS, analytics, account system, or runtime dependency on a running FluxServe engine. diff --git a/package.json b/package.json index ddb63f9..b5f5069 100644 --- a/package.json +++ b/package.json @@ -8,10 +8,9 @@ "npm": ">=10.8.2" }, "scripts": { - "docs:import": "node scripts/import-docs.mjs", - "dev": "npm run docs:import -- --local && astro dev --host 127.0.0.1", - "build": "npm run docs:import && astro build && node scripts/check-site.mjs", - "build:local": "npm run docs:import -- --local && astro build && node scripts/check-site.mjs", + "dev": "astro dev --host 127.0.0.1", + "build": "astro build && node scripts/check-site.mjs", + "build:local": "npm run build", "preview": "astro preview --host 127.0.0.1", "check": "astro check", "test": "node --test tests/*.test.mjs" diff --git a/src/components/BenchmarkChart.astro b/src/components/BenchmarkChart.astro index b35d3a7..dfe4f9d 100644 --- a/src/components/BenchmarkChart.astro +++ b/src/components/BenchmarkChart.astro @@ -1,103 +1,5 @@ --- import SpeedBenchmarkChart from './SpeedBenchmarkChart.astro'; - -// Illustrative placeholder data for the blog design. Replace with measured results. -const series = [ - { id: 'flux', name: 'FluxServe', values: [[40, 1510], [80, 1320], [120, 1110], [180, 850], [260, 610], [340, 430]] }, - { id: 'baseline', name: 'SGLang', values: [[40, 1320], [80, 1120], [120, 920], [180, 690], [260, 470], [340, 310]] }, -] as const; -const xTicks = [0, 100, 200, 300, 400]; -const yTicks = [0, 400, 800, 1200, 1600]; -const x = (value: number) => 88 + value / 400 * 850; -const y = (value: number) => 370 - value / 1600 * 320; --- -
- - -
- {series.map(item => {item.name})} -
- -
- - Single-model scaling performance - Placeholder data comparing FluxServe with a baseline. Both lines slope downward as user input tokens per second increase, while FluxServe maintains higher output tokens per second. - - - {yTicks.map(tick => - - {tick.toLocaleString('en-US')} - )} - {xTicks.map(tick => - - {tick} - )} - - - User/Input/Tok/S - Output/Tok/S - - - - - - {series.map(item => - `${x(input)},${y(output)}`).join(' ')} /> - {item.values.map(([input, output], index) => )} - )} - - -
- -
- Design preview: values are placeholders intended to demonstrate the visualization and animation. - Benchmark Guide ↗ -
-
- - diff --git a/src/components/BlockDiffusionAnimation.astro b/src/components/BlockDiffusionAnimation.astro new file mode 100644 index 0000000..96e0f93 --- /dev/null +++ b/src/components/BlockDiffusionAnimation.astro @@ -0,0 +1,116 @@ +
+
PROMPTDescribe a fox crossing a river just before sunrise in one short sentence.Step 0 of 12
+
Block diffusionMultiple positions decode in each pass
+
+ {[0, 1, 2].map((blockIndex) =>
+
{['01 / FIRST BLOCK', '02 / SECOND BLOCK', '03 / THIRD BLOCK'][blockIndex]}{blockIndex === 0 ? 'Refining' : 'Waiting'}
+
{[0, 1, 2, 3].map(() => [MASK])}
+
)} +
+
+
Autoregressive generationOne token decodes in each pass
+
+ {Array.from({ length: 12 }, () => ·)} +
+
+
Illustrative comparison of decoding steps, not a timing benchmark. Undecoded diffusion positions remain [MASK]; dots mark future AR positions. <eos> marks the end of the sequence.
+
+ + + + diff --git a/src/components/ModelRecipeSelector.astro b/src/components/ModelRecipeSelector.astro index d2a4901..1db835d 100644 --- a/src/components/ModelRecipeSelector.astro +++ b/src/components/ModelRecipeSelector.astro @@ -1,5 +1,5 @@ --- -import { defaults, groups, optionLabel, optionVisible, recipeView } from '../lib/model-recipes'; +import { defaults, groups, optionEnabled, optionLabel, optionVisible, recipeView } from '../lib/model-recipes'; const initialView = recipeView(defaults); --- @@ -36,6 +36,7 @@ const initialView = recipeView(defaults); aria-pressed={defaults[group.key] === option.value} aria-label={option.verified ? option.label + ', verified hardware' : undefined} hidden={!optionVisible(defaults, group.key, option.value)} + disabled={!optionEnabled(defaults, group.key, option.value)} > {option.verified && } {optionLabel(defaults, group.key, option)} @@ -66,7 +67,7 @@ const initialView = recipeView(defaults); + diff --git a/src/content/blog/introducing-fluxserve.md b/src/content/blog/introducing-fluxserve.md index 5c68006..c9fd2e0 100644 --- a/src/content/blog/introducing-fluxserve.md +++ b/src/content/blog/introducing-fluxserve.md @@ -1,30 +1,81 @@ --- title: "FluxServe: A Flexible and High-Performance Inference Engine for Open Diffusion Language Models" -description: We are execited to announce FluxServe, a new inference engine for open diffusion langauge models. FluxServe is desigend and implemented to deliver low-latency and high-throughput for autoregressive (AR) diffusion models with optimized attention runtime, dynamic block-level scheduling and efficient multi-GPU serving. -date: 2026-09-15 +description: We are excited to announce FluxServe, a new inference engine built specifically for open diffusion language models. FluxServe is designed and implemented to deliver low latency and high throughput for autoregressive (AR) diffusion models through an optimized attention runtime, dynamic block-level scheduling, and efficient multi-GPU serving. +date: 2026-09-27 author: FluxServe Team draft: false --- -Recently, diffusion large language models (dLLMs) have emerged as a highly promising alternative paradigm to their AR-based LLM counterparts. -The rapidly growing popularity of dLLMs stems from their unique architectural advantages of combining long-horizon casual sequencing with high-fidelity iterative refinement, where multiple tokens can be denoised at once in a bidirectional manner. -Exemplified by models such as LLaDA, Diffusion-Gemma, and Mercury, dLLMs offer compelling generative capabilities with better compute utilization that promotes a more efficient token economy for modern AI workloads. +Recently, diffusion large language models (dLLMs) have emerged as a highly promising alternative to traditional autoregressive (AR) LLMs. The rapidly growing popularity of dLLMs stems from their unique architectural advantage: combining long-horizon causal sequencing with high-fidelity iterative refinement, allowing multiple tokens to be denoised simultaneously in a bidirectional manner. Exemplified by recent open-weight releases, dLLMs offer compelling generative capabilities with superior compute utilization, promoting a more efficient token economy for modern AI workloads. + +### What is a Diffusion Language Model (dLLM)? + +Diffusion models are a well-established class of generative models that learn to transform noise into data through an iterative denoising process. While widely adopted in image and video generation—where models progressively refine random noise into high-quality visuals—applying diffusion to language is a rapidly emerging frontier. + +Instead of predicting text token-by-token, dLLMs take a block of masked tokens and gradually refine them into coherent text. This unique parallel decoding structure provides dLLMs with bidirectional context and enables block-level parallelism during generation. However, this also introduces significant challenges for existing AR serving stacks. Because current inference components are heavily optimized for sequential token generation, they are fundamentally sub-optimal for the block-level workloads required by dLLMs. + +
### FluxServe Overview -Diffusion language models refine multiple token positions over a sequence of steps. That workload behaves differently from traditional autoregressive decoding, where generation advances one token at a time. +[FluxServe](https://github.com/FLX-OSS/FluxServe) is a lightweight and high-performance serving engine engineered specifically for diffusion language models. It is designed to deliver low-latency and high-throughput inference for autoregressive diffusion models across a variety of hardware setups, ranging from single-GPU batched inference to multi-GPU distributed serving. -FluxServe uses block-level scheduling to keep requests moving efficiently while supporting variable-length prefill and decoding. Paged KV caching and CUDA graph support help reduce overhead along the way. +At launch, FluxServe’s core features include: + +- **Native Block-Causal Attention**: FluxServe implements an efficient block-causal attention runtime tailored for AR diffusion in real-world scenarios. It supports both variable-length (varlen) prefill and varlen block-decode, with backend support for both FlashInfer and FA4. +- **Dynamic Request Scheduler**: FluxServe features a hybrid scheduling architecture, pairing a low-overhead C++ control plane with a Python execution plane. This enables the fine-grained, block-level request management essential for diffusion models. +- **Unified Diffusion Playground**: FluxServe provides native support for a wide range of open diffusion language models, such as [LLaDA 2.X](https://github.com/inclusionAI/LLaDA2.X) and [Diffusion-Gemma](https://huggingface.co/google/diffusiongemma-26B-A4B-it), establishing a standardized benchmarking platform for both academic researchers and industry practitioners. ### Performance Results +Here, we present preliminary benchmark results comparing FluxServe against SGLang. To ensure a fair comparison, we launched both engines as API endpoints and utilized the third-party evaluation tool [evalscope](https://github.com/modelscope/evalscope) to measure system performance across multiple datasets. + +The figure below highlights the LLaDA 2.0 and 2.1 performance of FluxServe versus SGLang. Across four different model configurations, FluxServe achieves a consistent decode throughput improvement over AR-oriented serving stacks, delivering an average speedup of 1.6x. + +
+ +### Roadmaps + +### Short-term Implementations +- Extensive Model Support: [Nemotron-Labs-Diffusion](https://github.com/FLX-OSS/FluxServe/pull/14) +- Advanced Quantization: FP8 & NVFP4 +- NVIDIA Blackwell GPU Support +- Production Model Gateway: gRPC + + +### Long-term Goals +- AMD ROCm Support +- Multi-Modal Diffusion Support + + +### External Contributions +FluxServe is built as a lightweight and performance-oriented serving infrastructure project. Due the widespread use of AI agents, we will be intentionally selective about the submitted PRs, and conduct thorough discussion and validation before merging. +We appreciate everyone's ideas, support and feedback to our project. +We welcome external contributions, especially: +- Obvious bug fixes +- Performance optimizations that fit the existing codebase style without additional unnecessary complexity +- Documentation, tooling, and benchmarking improvements + ### Acknowledgements +Our system design was inspired by, and incorporates reused code from, the following incredible projects: [vLLM](https://github.com/vllm-project/vllm), [SGLang](https://github.com/sgl-project/sglang), [TokenSpeed](https://github.com/lightseekorg/tokenspeed), [dInfer](https://github.com/inclusionAI/dInfer), [FlashInfer](https://github.com/flashinfer-ai/flashinfer/pull/2722), and [Flash-Attention](https://github.com/dao-ailab/flash-attention). +### Citation + +```bibtex +@misc{fluxserve2026, + author = {{FluxServe Team}}, + title = {FluxServe: A Flexible and High-Performance Inference Engine for Open Diffusion Language Models}, + year = {2026}, + month = {September}, + howpublished = {\url{[https://github.com/FLX-OSS/FluxServe](https://github.com/FLX-OSS/FluxServe)}} +} +``` -### What comes next +### Contributors -We plan to use this blog for engineering notes, release updates, benchmarks, and deeper explanations of the ideas behind FluxServe. +- Project Lead & Creator: [Youpeng Zhao](https://kennethzhao24.github.io/) +- Model Runtime: [Meiling Wang](https://meiling0131.github.io/), [Depng Zhu](https://github.com/zhudp3) +- Benchmark & Documentation: [Zhiben Chen](https://www.linkedin.com/in/zhiben-chen/), [Ziyan Wang](https://www.linkedin.com/in/ziyan-wang-00a163228/) -For a closer look at the system today, read the [getting started guide](/docs/getting_started/). diff --git a/src/content/docs/docs/benchmark-guide.md b/src/content/docs/docs/benchmark-guide.md new file mode 100644 index 0000000..b426a55 --- /dev/null +++ b/src/content/docs/docs/benchmark-guide.md @@ -0,0 +1,72 @@ +--- +title: Benchmark Guide +sidebar: + order: 4 +description: Measure FluxServe latency and throughput with online or offline benchmarks. +editUrl: false +--- + +FluxServe provides both internal online serving offline batched inference benchmark. + +## Prepare a dataset + +Both benchmarks accept a JSONL file containing one JSON object per line. Each row requires a non-empty `messages` array. `max_tokens`, request parameters, and optional metadata may also be included. + +```json +{"messages":[{"role":"user","content":"Explain diffusion language models briefly."}],"max_tokens":128,"metadata":{"task_id":"example-1"}} +{"messages":[{"role":"user","content":"Write a Python function that reverses a list."}],"max_tokens":256,"metadata":{"task_id":"example-2"}} +``` +Save these lines as `data/benchmark.jsonl`. Each `metadata.task_id` must be unique when provided. + +## Online benchmark + +The online benchmark measures the full HTTP serving path, including scheduling, execution, and transport overhead. +Use a command from [Model Recipes](/docs/model-recipes/). + +```bash +fluxserve launch \ + --model inclusionAI/LLaDA2.0-mini \ + --host 127.0.0.1 \ + --port 8000 \ + --tp-size 1 \ + --dp-size 1 \ + --ep-size 1 \ + --max-num-seqs 4 \ + --attention-backend flashinfer + +curl -fsS http://127.0.0.1:8000/health + +fluxserve bench serve \ + --model inclusionAI/LLaDA2.0-mini \ + --dataset ./data/benchmark.jsonl \ + --num-prompts 100 \ + --dataset-output-len 128 \ + --request-rate 1 \ + --max-concurrency 4 \ + --metric-percentiles 50,90,95,99 \ + --save-result +``` + +## Offline benchmark + +The offline runner loads the model directly and runs batched inference: + +```bash +fluxserve bench_offline \ + --model inclusionAI/LLaDA2.0-mini \ + --dataset ./data/benchmark.jsonl \ + --batch-size 4 \ + --mini-batch-size 4 \ + --gen-len 128 \ + --block-length 64 \ + --tp-size 1 \ + --dp-size 1 \ + --ep-size 1 \ + --attention-backend flashinfer \ + --output-dir runs/detailed_results \ + --log-file runs/benchmark.log +``` + +## Third-Party benchmark + +We also provide scripts to benchmark FluxServe using third-party evalscope to directly measure the serving throughput through openai-compaitible API. diff --git a/src/content/docs/docs/configuration.md b/src/content/docs/docs/configuration.md new file mode 100644 index 0000000..205345f --- /dev/null +++ b/src/content/docs/docs/configuration.md @@ -0,0 +1,82 @@ +--- +title: Serving Parameters +sidebar: + label: Configuration + order: 2 +head: + - tag: title + content: Confguration +description: CLI parameter configurations for FluxServe. +editUrl: false +--- + +These options are defined for `fluxserve launch` in `python/fluxserve/cli.py`. Defaults below are CLI defaults; the recipes further down override some of them. `bench` and `bench_offline` define their options in separate modules. + +## Engine arguments + +| Parameter | Purpose | +| --- | --- | +| `--host` | Server bind address. Default: `0.0.0.0`. | +| `--port` | Server port. Default: `8000`. | +| `--device` | GPU device specification. Default: `cuda`. | +| `--gpu-memory-utilization` | GPU memory utilization fraction passed to automatic KV-page profiling. Default: `0.90`. | +| `--gpu-memory-safety-reserve` | Safety reserve fraction passed to automatic KV-page profiling. Default: `0.05`. | +| `--kv-cache-layout` | KV-cache layout: `dense` or `paged`. Default: `paged`. | +| `--page-size` | KV-cache page size. Default: unset; scheduler page size falls back to `--block-length`. Paged scheduler policies require it to equal `--block-length`. | + + +## Model arguments + +| Parameter | Purpose | +| --- | --- | +| `--model` / `--model-name` | Required model checkpoint directory or Hugging Face model ID. Loads both the model configuration and tokenizer from this location. | +| `--max-model-len` | Maximum model sequence length. Default: `2048`. Must be divisible by `--block-length` for paged scheduler policies. | +| `--max-new-tokens` | Runner generation-length setting. Default: `128`. | +| `--trust-remote-code` | Allow remote model/tokenizer code when loading the checkpoint. Already `true` by default; this parser provides no disabling flag. | + + +## Scheduler arguments + +| Parameter | Purpose | +| --- | --- | +| `--max-num-seqs` | Maximum scheduler batch size / concurrent sequences. Default: `8`. Also bounds decode CUDA graph batch sizes. | +| `--max-scheduled-tokens` | Scheduler token budget. Default: `512`. Must be divisible by `--block-length` for paged scheduler policies. | +| `--scheduler-policy` | Select `paged` or `dynamic`. Default: `paged`. Both require FlashInfer with paged prefill, paged cache mode, and paged KV layout. | +| `--scheduler-num-device-pages` | Number of device KV-cache pages. Default: `0`; nonpositive values trigger automatic profiling when paged KV pages are needed. | + +## Attention arguments + +| Parameter | Purpose | +| --- | --- | +| `--attention-backend` | Select `sdpa`, `flashinfer`, or `fa4`. Default: `flashinfer`. | +| `--use-cuda-graph` | Enable the runner's general CUDA graph flag. Default: `False`. | +| `--use-prefill-cuda-graph` | Enable prefill CUDA graphs. Default: `False`. (experiemental). | +| `--use-decode-cuda-graph` | Enable decode CUDA graphs. Default: `False`. | +| `--cuda-graph-decode-mode` | Decode graph mode: `decomposed` or `padded`. Default: `padded`. | +| `--cuda-graph-capture-bs` | Space-separated decode graph batch sizes, for example `1 2 4 8`. Defaults to `1` plus every positive even size up to `--max-num-seqs`. Values cannot exceed `--max-num-seqs`. | +| `--cuda-graph-capture-sizes` | Space-separated prefill sequence-length buckets. Default: `64 128 256 512 1024`. Only positive, block-aligned values no greater than `--max-model-len` are retained (experiemental). | + +## Decoding arguments + +| Parameter | Purpose | +| --- | --- | +| `--block-length` | Diffusion decoding block length for LLaDA2.X. Default: `64`. For checkpoints with MoE block routing, it must be a multiple of the checkpoint's routing block size. | +| `--canvas-length` | Diffusion canvas length for Diffusion-Gemma. Default: `128`. | +| `--max-denoising-steps` | Override checkpoint denoising steps, primarily for smoke tests. Default: unset. | +| `--parallel-decoding` | Parallel decoder name. Default: `threshold`. This CLI also references `joint_threshold` for LLaDA2.1 and `levenshtein_joint` for LLaDA2.2; the parser does not restrict choices. | +| `--threshold` | Decoding confidence threshold. Default: `0.9`. | +| `--low-threshold` | Low confidence threshold passed to the decoder. Default: `0.3`. | +| `--editing-threshold` | LLaDA2.1 Token-to-Token editing threshold for `joint_threshold`. Default: `0.5` (Quality preset); the Speed preset uses `0.0`. | +| `--max-post-steps` | Maximum post-mask editing iterations per block for `joint_threshold`. Default: `16`. | + + + +## Parallelism arguments + +| Parameter | Purpose | +| --- | --- | +| `--tp-size` | Tensor-parallel size. Default: `1`. Used to decide whether to launch local distributed workers. | +| `--dp-size` | Data-parallel size passed to the server configuration. Default: `1`. | +| `--ep-size` | Expert-parallel size passed to the server configuration. Default: `1`. | +| `--enable-dp-attention` | Enable data-parallel attention in the server configuration. Off by default. | +| `--distributed-backend` | Backend used to initialize distributed execution. Default: `nccl`. | diff --git a/src/content/docs/docs/getting_started.md b/src/content/docs/docs/getting_started.md new file mode 100644 index 0000000..e46e67a --- /dev/null +++ b/src/content/docs/docs/getting_started.md @@ -0,0 +1,104 @@ +--- +title: "Getting Started" +sidebar: + order: 1 +editUrl: "https://github.com/FLX-OSS/FLX-OSS.github.io/edit/main/src/content/docs/docs/getting_started.md" +--- + +## Prerequisites + +- A Linux host with NVIDIA GPUs of compute capability 9.0 or newer. +- An NVIDIA driver compatible with CUDA 12.9+, Docker, and NVIDIA Container Toolkit configured for GPU access. + +--- + +## Build the docker + +```bash +docker pull flxoss/fluxserve:v0.1-cu130-fa4 +``` + +--- + +## Start the workspace + +```bash +docker run -itd \ + --shm-size 32g \ + --gpus all \ + --ipc=host \ + --network=host \ + --pid=host \ + --privileged \ + --name flux_workspace \ + fluxserve:v0.1-cu130-fa4 \ + /bin/bash + +docker exec -it flux_workspace /bin/bash +``` + +--- + +## Install FluxServe inside the container + +The image provides the CUDA and Python dependencies. Clone the source inside the container and install the kernel, scheduler, and runtime in that order: + +```bash +git clone https://github.com/FLX-OSS/FluxServe +cd FluxServe +export PIP_BREAK_SYSTEM_PACKAGES=1 +pip install -e flux-kernel/python/ --no-build-isolation +pip install -e flux-scheduler +pip install -e . +``` + +--- + +## Verify Installation + +```bash +fluxserve env +fluxserve launch --help +``` + +--- + +## Launch +```bash +fluxserve launch \ + --model inclusionAI/LLaDA2.1-mini \ + --host 127.0.0.1 \ + --port 8000 \ + --tp-size 1 \ + --dp-size 1 \ + --ep-size 1 +``` +For model-specific examples, follow [Model Recipes](/docs/model-recipes/). + +--- + +## Check readiness + +In another shell in the same environment: + +```bash +curl -fsS http://127.0.0.1:8000/health +``` + +--- + +## Send a request + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "inclusionAI/LLaDA2.0-mini", + "messages": [{"role": "user", "content": "Explain diffusion language models in a few sentences."}], + "max_tokens": 128, + "temperature": 0, + "stream": false + }' +``` + +*** diff --git a/src/content/docs/docs/model-recipes.md b/src/content/docs/docs/model-recipes.md new file mode 100644 index 0000000..9836e89 --- /dev/null +++ b/src/content/docs/docs/model-recipes.md @@ -0,0 +1,7 @@ +--- +title: Model Recipes +description: Configure parallelism, attention backend, and target hardware for a FluxServe launch command. +editUrl: false +--- + +See [Configuration](/docs/configuration/) for all serving parameters. diff --git a/src/lib/model-recipes.ts b/src/lib/model-recipes.ts index 9a6937d..66ecc45 100644 --- a/src/lib/model-recipes.ts +++ b/src/lib/model-recipes.ts @@ -19,7 +19,7 @@ const placeholderLink = (model: string, variant: string): VariantLink => ({ export const models: Record = { 'LLaDA2.0-mini': { - label: 'LLaDA2.0 Mini', + label: 'LLaDA2.0-Mini', checkpoint: 'inclusionAI/LLaDA2.0-mini', variants: { BF16: 'BF16 33GB'}, variantLinks: { BF16: @@ -31,7 +31,7 @@ export const models: Record = { parallelism: ['1', '4'], }, 'LLaDA2.0-flash': { - label: 'LLaDA2.0 Flash', + label: 'LLaDA2.0-Flash', checkpoint: 'inclusionAI/LLaDA2.0-flash', variants: { BF16: 'BF16 206GB'}, variantLinks: { BF16: @@ -43,7 +43,7 @@ export const models: Record = { parallelism: ['4'], }, 'LLaDA2.1-mini': { - label: 'LLaDA2.1 Mini', + label: 'LLaDA2.1-Mini', checkpoint: 'inclusionAI/LLaDA2.1-mini', variants: { BF16: 'BF16 33 GB'}, variantLinks: { BF16: @@ -55,7 +55,7 @@ export const models: Record = { parallelism: ['1', '4'], }, 'LLaDA2.1-flash': { - label: 'LLaDA2.1 Flash', + label: 'LLaDA2.1-Flash', checkpoint: 'inclusionAI/LLaDA2.1-flash', variants: { BF16: 'BF16 206GB'}, variantLinks: { BF16: @@ -118,8 +118,8 @@ export const groups: Group[] = [ key: 'hardware', label: 'Hardware', options: [ - { value: 'H100', label: 'H100' }, - { value: 'H200', label: 'H200' }, + { value: 'H100', label: 'H100', verified: true }, + { value: 'H200', label: 'H200', verified: true }, { value: 'GH200', label: 'GH200', verified: true }, { value: 'B200', label: 'B200', verified: true }, ], @@ -153,9 +153,11 @@ export const defaults: Selection = { }; export function selectOption(current: Selection, key: Setting, value: string): Selection { + if (!optionEnabled(current, key, value)) return current; const next = { ...current, [key]: value }; const model = models[next.model]; - if (!(next.variant in model.variants)) next.variant = 'BF16'; + if (!(next.variant in model.variants) || !optionEnabled(next, 'variant', next.variant)) next.variant = 'BF16'; + if (!optionEnabled(next, 'backend', next.backend)) next.backend = 'flashinfer'; if (key === 'model' || !model.parallelism.includes(next.parallel)) next.parallel = model.parallelism[0]; return next; } @@ -167,20 +169,42 @@ export function optionVisible(selection: Selection, key: Setting, value: string) return true; } +export function optionEnabled(selection: Selection, key: Setting, value: string): boolean { + if (key === 'variant') return value === 'BF16'; + if (key === 'backend' && value === 'fa4') return selection.model.startsWith('LLaDA2.'); + return true; +} + export function optionLabel(selection: Selection, key: Setting, option: Option): string { return key === 'variant' ? models[selection.model].variants[option.value] ?? option.label : option.label; } /** Keep command arguments explicit: no replacements against rendered HTML. */ export function buildCommand(selection: Selection): string { - const args = [ + const isLlada21 = selection.model.startsWith('LLaDA2.1-'); + const isGemma = selection.model === 'diffusion-gemma'; + const args: [string, string][] = [ ['model', models[selection.model].checkpoint], ['tp-size', selection.parallel], ['dp-size', '1'], ['ep-size', selection.parallel], + ...(isLlada21 ? [ + ['parallel-decoding', 'joint_threshold'], + ['threshold', '0.7'], + ['editing-threshold', '0.5'], + ['max-post-steps', '16'], + ] as [string, string][] : []), + ...(isGemma ? [ + ['max-num-seqs', '4'], + ['max-model-len', '8192'], + ['block-length', '256'], + ['canvas-length', '256'], + ['page-size', '256'], + ] as [string, string][] : []), ['attention-backend', selection.backend], + ...(isGemma ? [['scheduler-policy', 'default']] as [string, string][] : []), ]; - return ['fluxserve serve', ...args.map(([key, value]) => ` --${key} ${value}`)].join(' \\\n'); + return ['fluxserve launch', ...args.map(([key, value]) => ` --${key} ${value}`)].join(' \\\n'); } /** Derive display state in one place for both initial HTML and browser updates. */ diff --git a/src/pages/blog/[...slug].astro b/src/pages/blog/[...slug].astro index d3691e8..72fd173 100644 --- a/src/pages/blog/[...slug].astro +++ b/src/pages/blog/[...slug].astro @@ -2,6 +2,7 @@ import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; import { getCollection, render, type CollectionEntry } from 'astro:content'; import BenchmarkChart from '../../components/BenchmarkChart.astro'; +import BlockDiffusionAnimation from '../../components/BlockDiffusionAnimation.astro'; import { publishedPosts } from '../../lib/blog.mjs'; export async function getStaticPaths() { return publishedPosts(await getCollection('blog')).map(post => ({ params: {slug: post.id}, props: {post} })); @@ -13,9 +14,16 @@ const { Content, headings } = await render(post);

{post.data.author} ·

+ {post.id === 'introducing-fluxserve' && } {post.id === 'introducing-fluxserve' && } {post.id === 'introducing-fluxserve' &&