feat(mongodb-runner): locate the DSC bundle automatically with --slsDir

packages/mongodb-runner/README.md

@@ -155,6 +155,18 @@ layer, managing the storage backend's docker compose project alongside the
 mongod processes. See
 [docs/disaggregated-storage.md](./docs/disaggregated-storage.md).
 
+Point it at a disaggregated-storage-capable MongoDB install (or a Server
+source checkout) and everything else is located automatically:
+
+```bash
+npx @mongodb-js/mongodb-runner start -t replset --slsDir=/path/to/dsc-install --binDir=...
+# or, for a Server source checkout:
+npx @mongodb-js/mongodb-runner start -t replset --slsCompose=.../atlas/sls-multicell-docker-compose.yml --binDir=...
+```
+
+Pass `--slsDir` without a value to resolve it relative to `--binDir`
+(`--binDir=.../bin` looks in `.../` for the bundle).
+
 ## License
 
 Apache 2.0

packages/mongodb-runner/docs/disaggregated-storage.md

@@ -64,44 +64,46 @@ override.
   - `binDir`: a local directory containing the binaries, or
   - `downloadUrl`: a URL to a tarball of such a build (cached by URL, standard
     release-tarball layout with a top-level directory containing `bin/`).
-- The SLS multi-cell compose file — mongodb-runner does not ship one; point
-  it at the file you want to use, typically
-  `buildscripts/modules/atlas/sls-multicell-docker-compose.yml` in a mongodb
-  server checkout. Files it references (`slsbackup.proto`,
-  `flags-state.json`) are resolved relative to it, and the services/ports are
-  parsed from it, so any version of the file works as-is.
+- The SLS bundle — mongodb-runner does not ship one. Point it at a bundle one
+  of two ways:
+  - `slsDir` (recommended for an installed build): a path to a
+    disaggregated-storage-capable MongoDB install (the tarball's root, or its
+    `buildscripts/modules/atlas` directory). The runner locates the compose
+    file, validates the bundle, and reads the image tag from it automatically.
+  - `slsCompose` (escape hatch for a Server source checkout): the path to the
+    compose file directly, typically
+    `buildscripts/modules/atlas/sls-multicell-docker-compose.yml` in a mongodb
+    server checkout. Files it references (`slsbackup.proto`,
+    `flags-state.json`) are resolved relative to it, and the services/ports are
+    parsed from it, so any version of the file works as-is. The two options are
+    mutually exclusive.
 - The image tag is read automatically from the `pinned_sls_commit` in
-  `manifest.json` sitting next to the compose file (the server repository's
-  `buildscripts/modules/atlas/manifest.json`). Pass `--slsImageTag` to
+  `manifest.json` sitting next to the compose file. Pass `--slsImageTag` to
   override it.
 
 ## Quick start
 
-Given the compose file, everything else (image tag, compose environment
-variables, readiness polling, per-shard log creation, and the
+Given an install or source checkout, everything else (image tag, compose
+environment variables, readiness polling, per-shard log creation, and the
 `disaggregatedStorageConfig` server parameter) is generated automatically.
-Full sequence, assuming a mongodb server checkout at `$MONGO_REPO`:
+Assuming a disaggregated-storage-capable MongoDB install at `$INSTALL_DIR`:
 
 ```bash
-# The image tag is read from the manifest.json next to the compose file; pass
-# --slsImageTag to override it. Logs in to ECR automatically.
-@mongodb-js/mongodb-runner start -t replset \
-  --slsCompose=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \
-  --binDir=/path/to/dsc-mongod/bin \
-  --debug
-# or, instead of --binDir:
-#   --downloadUrl=https://.../dsc-mongod.tgz
+@mongodb-js/mongodb-runner start --topology=replset --slsDir="$INSTALL_DIR" \
+  --logDir="$LOG_DIR" --id=tools-dsc --json >cluster.json
 ```
 
-This prints the connection string once the cluster is up. To get the allocated
-SLS service ports/URIs as structured output, add `--json` and capture stdout to
-a file instead of scraping it:
+`--slsDir` locates and validates the SLS bundle and reads the image tag from
+its manifest. For a Server source checkout rather than an installed build,
+pass the compose file explicitly with `--slsCompose` instead:
 
 ```bash
-mongodb-runner start -t replset \
+# The image tag is read from the manifest.json next to the compose file; pass
+# --slsImageTag to override it. Logs in to ECR automatically.
+@mongodb-js/mongodb-runner start -t replset \
   --slsCompose=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \
   --binDir=/path/to/dsc-mongod/bin \
-  --json > cluster.json
+  --debug
 ```
 
 The JSON object carries `id` and `connectionString` plus an `sls` field with
@@ -253,13 +255,12 @@ server parameter for one shard; options: `logId`, `cellMetadataService`,
 
 ## CLI use
 
-For an SLS project, `--slsCompose` handles everything (see Quick start);
-`--slsImageTag` is optional and overrides the tag read from the manifest:
+For an SLS project, `--slsDir` (installed build) or `--slsCompose` (source
+checkout) handle everything (see Quick start); `--slsImageTag` is optional and
+overrides the tag read from the manifest:
 
 ```bash
-@mongodb-js/mongodb-runner start -t replset \
-  --slsCompose=/path/to/sls-multicell-docker-compose.yml \
-  --binDir=...
+@mongodb-js/mongodb-runner start -t replset --slsDir=/path/to/dsc-install --binDir=...
 ```
 
 Custom (non-SLS) storage backends are only supported through the programmatic

packages/mongodb-runner/src/cli.ts

@@ -88,10 +88,16 @@ import type { MongoClientOptions } from 'mongodb';
       describe:
         'Path to an SLS multi-cell docker-compose.yml; launches the SLS DSC project and configures mongod to use it (requires a DSC-capable mongod via --binDir or --downloadUrl)',
     })
+    .option('slsDir', {
+      type: 'string',
+      describe:
+        'Path to a disaggregated-storage-capable MongoDB install (or its buildscripts/modules/atlas directory); locates the SLS compose file and image tag automatically. Pass without a value to resolve relative to --binDir. Mutually exclusive with --slsCompose',
+    })
+    .conflicts('slsDir', 'slsCompose')
     .option('slsImageTag', {
       type: 'string',
       describe:
-        'SLS docker image tag to use with --slsCompose (defaults to the pinned_sls_commit from the manifest.json next to the compose file)',
+        'SLS docker image tag to use (defaults to the pinned_sls_commit from the manifest.json next to the compose file)',
     })
     .option('slsSkipEcrLogin', {
       type: 'boolean',
@@ -141,9 +147,24 @@ import type { MongoClientOptions } from 'mongodb';
   }
 
   async function start() {
-    const disaggregatedStorage = argv.slsCompose
+    let composeFile = argv.slsCompose;
+    if (argv.slsDir !== undefined) {
+      // An empty value means "resolve relative to --binDir".
+      const slsDir =
+        argv.slsDir ||
+        (argv.binDir
+          ? path.join(argv.binDir, '..')
+          : (() => {
+              throw new Error(
+                '--slsDir was given without a value and --binDir is not set, so there is nowhere to look for the SLS bundle',
+              );
+            })());
+      const bundle = await utilities.resolveSLSBundle(slsDir);
+      composeFile = bundle.composeFile;
+    }
+    const disaggregatedStorage = composeFile
       ? await utilities.createSLSDisaggregatedStorageOptions({
-          composeFile: argv.slsCompose,
+          composeFile,
           imageTag: argv.slsImageTag,
           ecrLogin: !argv.slsSkipEcrLogin,
         })

packages/mongodb-runner/src/index.ts

@@ -23,6 +23,8 @@ export {
   type SLSDisaggregatedStorageSetupOptions,
   parseSLSComposeServices,
   readPinnedSlsCommit,
+  resolveSLSBundle,
+  SLS_ATLAS_SUBDIR,
   SLS_HOSTNAME,
   SLS_CELL1,
   SLS_CELL2,
@@ -31,6 +33,7 @@ export {
   type SLSServiceInfo,
   type SLSMultiCellEnvironment,
   type SLSMultiCellEnvironmentOptions,
+  type SLSBundle,
 } from './sls';
 export {
   parseEcrRegistry,

packages/mongodb-runner/src/sls.spec.ts

@@ -1,47 +1,114 @@
 import { expect } from 'chai';
 import path from 'path';
-import { readPinnedSlsCommit } from './sls';
+import { readPinnedSlsCommit, resolveSLSBundle } from './sls';
 
 const FIXTURES = path.resolve(__dirname, '..', 'test', 'fixtures', 'sls');
 
-describe('readPinnedSlsCommit', function () {
-  it('reads pinned_sls_commit from a manifest', async function () {
-    expect(
-      await readPinnedSlsCommit(path.join(FIXTURES, 'complete')),
-      'should return the pinned commit verbatim',
-    ).to.equal('abc123def456');
-  });
+describe('sls', function () {
+  describe('readPinnedSlsCommit', function () {
+    it('reads pinned_sls_commit from a manifest', async function () {
+      expect(
+        await readPinnedSlsCommit(path.join(FIXTURES, 'complete')),
+        'should return the pinned commit verbatim',
+      ).to.equal('abc123def456');
+    });
 
-  it('names the path it looked at when the manifest is absent', async function () {
-    const missing = path.join(FIXTURES, 'does-not-exist');
-    const err = await readPinnedSlsCommit(missing).catch((e: Error) => e);
-    expect(
-      (err as Error).message,
-      'error should name the manifest path that was checked',
-    ).to.include(path.join(missing, 'manifest.json'));
-    expect(
-      (err as Error).message,
-      'error should mention the override flag',
-    ).to.include('--slsImageTag');
-  });
+    it('names the path it looked at when the manifest is absent', async function () {
+      const missing = path.join(FIXTURES, 'does-not-exist');
+      const err = await readPinnedSlsCommit(missing).catch((e: Error) => e);
+      expect(
+        (err as Error).message,
+        'error should name the manifest path that was checked',
+      ).to.include(path.join(missing, 'manifest.json'));
+      expect(
+        (err as Error).message,
+        'error should mention the override flag',
+      ).to.include('--slsImageTag');
+    });
+
+    it('reports a manifest that is missing the key', async function () {
+      const err = await readPinnedSlsCommit(
+        path.join(FIXTURES, 'no-key'),
+      ).catch((e: Error) => e);
+      expect(
+        (err as Error).message,
+        'error should name the missing key',
+      ).to.include('pinned_sls_commit');
+    });
 
-  it('reports a manifest that is missing the key', async function () {
-    const err = await readPinnedSlsCommit(path.join(FIXTURES, 'no-key')).catch(
-      (e: Error) => e,
-    );
-    expect(
-      (err as Error).message,
-      'error should name the missing key',
-    ).to.include('pinned_sls_commit');
+    it('reports a malformed manifest', async function () {
+      const err = await readPinnedSlsCommit(
+        path.join(FIXTURES, 'malformed'),
+      ).catch((e: Error) => e);
+      expect(
+        (err as Error).message,
+        'error should say the manifest could not be parsed',
+      ).to.match(/parse/i);
+    });
   });
 
-  it('reports a malformed manifest', async function () {
-    const err = await readPinnedSlsCommit(
-      path.join(FIXTURES, 'malformed'),
-    ).catch((e: Error) => e);
-    expect(
-      (err as Error).message,
-      'error should say the manifest could not be parsed',
-    ).to.match(/parse/i);
+  describe('resolveSLSBundle', function () {
+    it('resolves a bundle addressed by install root', async function () {
+      const root = path.join(FIXTURES, 'bundle-complete');
+      const bundle = await resolveSLSBundle(root);
+      expect(
+        bundle.composeFile,
+        'compose file should be found under buildscripts/modules/atlas',
+      ).to.equal(
+        path.join(
+          root,
+          'buildscripts',
+          'modules',
+          'atlas',
+          'sls-multicell-docker-compose.yml',
+        ),
+      );
+      expect(
+        bundle.manifestFile,
+        'manifest should sit next to the compose file',
+      ).to.equal(path.join(bundle.atlasDir, 'manifest.json'));
+    });
+
+    it('resolves a bundle addressed by its atlas directory', async function () {
+      const atlasDir = path.join(
+        FIXTURES,
+        'bundle-complete',
+        'buildscripts',
+        'modules',
+        'atlas',
+      );
+      const bundle = await resolveSLSBundle(atlasDir);
+      expect(
+        bundle.atlasDir,
+        'passing the atlas dir directly should also work',
+      ).to.equal(atlasDir);
+    });
+
+    it('lists every missing file, not just the first', async function () {
+      const err = await resolveSLSBundle(
+        path.join(FIXTURES, 'bundle-incomplete'),
+      ).catch((e: Error) => e);
+      const message = (err as Error).message;
+      expect(
+        message,
+        'error should say this is not a disagg-capable build',
+      ).to.include('not a disaggregated-storage-capable MongoDB build');
+      expect(message, 'should report the missing proto').to.include(
+        'slsbackup.proto',
+      );
+      expect(message, 'should report the missing flags state too').to.include(
+        'flags-state.json',
+      );
+    });
+
+    it('rejects a directory that is not a MongoDB build', async function () {
+      const err = await resolveSLSBundle(
+        path.join(FIXTURES, 'not-a-build'),
+      ).catch((e: Error) => e);
+      expect(
+        (err as Error).message,
+        'a non-build directory should be rejected clearly',
+      ).to.include('not a disaggregated-storage-capable MongoDB build');
+    });
   });
 });

packages/mongodb-runner/src/sls.ts

@@ -64,6 +64,69 @@ export async function readPinnedSlsCommit(atlasDir: string): Promise<string> {
   return pinned;
 }
 
+/** Relative path of the atlas module directory inside a Server build. */
+export const SLS_ATLAS_SUBDIR = path.join('buildscripts', 'modules', 'atlas');
+
+const SLS_COMPOSE_FILE = 'sls-multicell-docker-compose.yml';
+const SLS_BACKUP_PROTO_FILE = 'slsbackup.proto';
+const SLS_FLAGS_STATE_FILE = 'flags-state.json';
+
+/** Files that make up a disaggregated-storage-capable build's SLS bundle. */
+export interface SLSBundle {
+  /** The `buildscripts/modules/atlas` directory holding the bundle. */
+  atlasDir: string;
+  composeFile: string;
+  manifestFile: string;
+  backupProtoFile: string;
+  flagsStateFile: string;
+}
+
+async function exists(file: string): Promise<boolean> {
+  try {
+    await fs.access(file);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Locate the SLS bundle shipped inside a disaggregated-storage-capable
+ * MongoDB build. `dir` may be either the install root or the
+ * `buildscripts/modules/atlas` directory itself.
+ */
+export async function resolveSLSBundle(dir: string): Promise<SLSBundle> {
+  const nested = path.join(dir, SLS_ATLAS_SUBDIR);
+  const atlasDir = (await exists(nested)) ? nested : dir;
+
+  const bundle: SLSBundle = {
+    atlasDir,
+    composeFile: path.join(atlasDir, SLS_COMPOSE_FILE),
+    manifestFile: path.join(atlasDir, SLS_MANIFEST_FILE),
+    backupProtoFile: path.join(atlasDir, SLS_BACKUP_PROTO_FILE),
+    flagsStateFile: path.join(atlasDir, SLS_FLAGS_STATE_FILE),
+  };
+
+  const missing: string[] = [];
+  for (const file of [
+    bundle.composeFile,
+    bundle.manifestFile,
+    bundle.backupProtoFile,
+    bundle.flagsStateFile,
+  ]) {
+    if (!(await exists(file))) missing.push(path.basename(file));
+  }
+  if (missing.length) {
+    throw new Error(
+      `${dir} is not a disaggregated-storage-capable MongoDB build ` +
+        `(missing: ${missing.join(', ')}; looked in ${atlasDir})`,
+    );
+  }
+
+  debug('resolved SLS bundle', { atlasDir });
+  return bundle;
+}
+
 export interface SLSServiceInfo {
   /** Environment variable through which the compose file receives the host port. */
   portVar: string;

packages/mongodb-runner/test/fixtures/sls/bundle-complete/buildscripts/modules/atlas/flags-state.json

@@ -0,0 +1 @@
+{}

packages/mongodb-runner/test/fixtures/sls/bundle-complete/buildscripts/modules/atlas/manifest.json

@@ -0,0 +1 @@
+{ "pinned_sls_commit": "bundlecommit123" }

packages/mongodb-runner/test/fixtures/sls/bundle-complete/buildscripts/modules/atlas/sls-multicell-docker-compose.yml

@@ -0,0 +1,4 @@
+services:
+  cms-cell1-0:
+    ports:
+      - '${CMS_CELL1_0_PORT:-30001}:27998'