Sanger Ab1 files support

.changeset/tagged-ab1-dataset-type.md

@@ -0,0 +1,8 @@
+---
+"@platforma-open/milaboratories.samples-and-data": minor
+"@platforma-open/milaboratories.samples-and-data.kind": minor
+"@platforma-open/milaboratories.samples-and-data.workflow": patch
+"@platforma-open/milaboratories.samples-and-data.ui": patch
+---
+
+Add the `TaggedAb1` dataset type for Sanger chromatograms. One `.ab1` file per sample per tag, exported as a `File` column with `pl7.app/fileExtension` `ab1` on axes `[pl7.app/sampleId, pl7.app/sequencing/tag]`. A pattern such as `{{Sample}}_{{:Primer}}.ab1` groups the traces of one sample by the tag that tells them apart, for example the sequencing primer.

kind/src/params.ts

@@ -89,6 +89,7 @@ const DATASET_TYPES = {
   Fasta: true,
   Xsv: true,
   TaggedXsv: true,
+  TaggedAb1: true,
   BulkCountMatrix: true,
   CellRangerMTX: true,
   MultiplexedFastq: true,

kind/src/types.ts

@@ -137,6 +137,17 @@ export interface DSContentTaggedXsv extends WithPerSampleData<TaggedXsvDatasetRe
   tags: string[];
 }
 
+export interface TaggedAb1DatasetRecord {
+  tags: Record<string, string>;
+  file: ImportFileHandle;
+}
+
+/** Sanger chromatograms. Tags carry whatever distinguishes traces of one sample, e.g. the sequencing primer. */
+export interface DSContentTaggedAb1 extends WithPerSampleData<TaggedAb1DatasetRecord[]> {
+  type: "TaggedAb1";
+  tags: string[];
+}
+
 export type CellRangerMtxRole = "matrix.mtx" | "features.tsv" | "barcodes.tsv";
 export type CellRangerMtxFileGroup = Partial<Record<CellRangerMtxRole, ImportFileHandle>>;
 
@@ -218,6 +229,7 @@ export type DSContent =
   | DSContentFasta
   | DSContentXsv
   | DSContentTaggedXsv
+  | DSContentTaggedAb1
   | DSContentBulkCountMatrix
   | DSContentCellRangerMtx
   | DSContentMultiplexedFastq
@@ -240,6 +252,7 @@ export type DSMultilaneFastq = Dataset<DSContentMultilaneFastq>;
 export type DSTaggedFastq = Dataset<DSContentTaggedFastq>;
 export type DSXsv = Dataset<DSContentXsv>;
 export type DSTaggedXsv = Dataset<DSContentTaggedXsv>;
+export type DSTaggedAb1 = Dataset<DSContentTaggedAb1>;
 export type DSCellRangerMtx = Dataset<DSContentCellRangerMtx>;
 export type DSBulkCountMatrix = Dataset<DSContentBulkCountMatrix>;
 export type DSMultiplexedFastq = Dataset<DSContentMultiplexedFastq>;

pnpm-workspace.yaml

@@ -9,19 +9,19 @@ packages:
   - workflow
 
 catalog:
-  "@platforma-sdk/block-tools": 2.16.1
+  "@platforma-sdk/block-tools": 2.16.3
   "@platforma-sdk/block-kind": 1.1.0
-  "@platforma-sdk/model": 1.83.17
-  "@platforma-sdk/ui-vue": 1.83.21
-  "@platforma-sdk/tengo-builder": 4.1.1
-  "@platforma-sdk/workflow-tengo": 6.11.1
+  "@platforma-sdk/model": 1.84.1
+  "@platforma-sdk/ui-vue": 1.84.2
+  "@platforma-sdk/tengo-builder": 4.1.4
+  "@platforma-sdk/workflow-tengo": 6.12.1
   "@platforma-sdk/blocks-deps-updater": 2.2.0
   "@platforma-sdk/package-builder": 3.16.0
 
   "@platforma-open/milaboratories.runenv-python-3": ^1.7.5
   "@platforma-open/milaboratories.runenv-r-samples-and-data": ^1.0.1
 
-  "@platforma-sdk/test": 1.83.25
+  "@platforma-sdk/test": 1.84.1
   "@milaboratories/helpers": 1.14.5
   "@milaboratories/pl-model-common": 1.48.0
 

ui/src/dialogs/ImportDatasetDialog.vue

@@ -11,6 +11,7 @@ import type {
   DSContentSeurat,
   DSContentMultiSampleH5ad,
   DSContentMultiSampleSeurat,
+  DSContentTaggedAb1,
   DSContentTaggedFastq,
   DSContentTaggedXsv,
   DSContentXsv,
@@ -186,6 +187,17 @@ watch(isOneOfDialogsOpened, (v) => {
 // Pattern compilation and file name matching
 const { patternError, compiledPattern } = usePatternCompilation(data);
 
+/** Records are stored under the target dataset's tag names; the workflow reads them by those names. */
+const tagMismatchError = computed(() => {
+  if (data.mode !== "add-to-existing") return undefined;
+  const ds = app.model.data.datasets.find((ds) => ds.id === data.targetAddDataset);
+  const pattern = compiledPattern.value;
+  if (!ds || !pattern || !("tags" in ds.content)) return undefined;
+  const expected = [...ds.content.tags].sort();
+  if (_.isEqual(pattern.tags, expected)) return undefined;
+  return `Pattern tags must match the dataset tags: ${expected.join(", ")}`;
+});
+
 function updateDataFromPattern(v: FileNamePattern | undefined) {
   if (v && !addingToFixedDataset.value) {
     data.datasetType = v.datasetType;
@@ -426,6 +438,28 @@ function addTaggedXsvDatasetContent(contentData: DSContentTaggedXsv["data"]) {
   }
 }
 
+/** Tagged AB1 */
+function addTaggedAb1DatasetContent(contentData: DSContentTaggedAb1["data"]) {
+  const pattern = compiledPattern.value;
+  if (!pattern) throw new Error("No pattern");
+
+  for (const f of parsedFiles.value) {
+    if (!f.match) continue;
+    const sample = f.match.sample.value;
+    const sampleId = getOrCreateSample(app, sample);
+    const tags = _.mapValues(f.match.tags!, (v) => v.value);
+
+    let sampleRecords = contentData[sampleId];
+    if (!sampleRecords) {
+      sampleRecords = [];
+      contentData[sampleId] = sampleRecords;
+    }
+
+    if (!sampleRecords.some((r) => _.isEqual(r.tags, tags)))
+      sampleRecords.push({ tags, file: f.handle });
+  }
+}
+
 /** CellRanger MTX */
 function addCellRangerMtxDatasetContent(contentData: DSContentCellRangerMtx["data"]) {
   for (const f of parsedFiles.value) {
@@ -578,6 +612,9 @@ async function addToExistingDataset() {
     case "TaggedXsv":
       addTaggedXsvDatasetContent(dataset.content.data);
       break;
+    case "TaggedAb1":
+      addTaggedAb1DatasetContent(dataset.content.data);
+      break;
     case "CellRangerMTX":
       addCellRangerMtxDatasetContent(dataset.content.data);
       break;
@@ -642,6 +679,21 @@ async function createNewDataset() {
       });
       break;
     }
+    case "TaggedAb1": {
+      const contentData: DSContentTaggedAb1["data"] = {};
+      addTaggedAb1DatasetContent(contentData);
+      app.model.data.datasets.push({
+        label: data.newDatasetLabel,
+        id: newDatasetId,
+        content: {
+          type: "TaggedAb1",
+          gzipped: data.gzipped,
+          tags: pattern.tags,
+          data: contentData,
+        },
+      });
+      break;
+    }
     case "Xsv": {
       const contentData: DSContentXsv["data"] = {};
       addXsvDatasetContent(contentData);
@@ -929,6 +981,7 @@ watch(availableColumnsOptions, (options) => {
 const canCreateOrAdd = computed(() => {
   const basicConditions =
     hasMatchedFiles.value &&
+    tagMismatchError.value === undefined &&
     (data.mode === "create-new-dataset" || data.targetAddDataset !== undefined) &&
     data.datasetType !== undefined &&
     !data.loadingColumns &&
@@ -984,7 +1037,7 @@ const canCreateOrAdd = computed(() => {
       <PlCheckbox v-model="data.gzipped" disabled> Gzipped </PlCheckbox>
     </PlRow>
 
-    <PlTextField v-model="data.pattern" label="Pattern" :error="patternError" />
+    <PlTextField v-model="data.pattern" label="Pattern" :error="patternError ?? tagMismatchError" />
 
     <div v-if="data.datasetType === 'MultiSampleH5AD' || data.datasetType === 'MultiSampleSeurat'">
       <div v-if="data.loadingColumns">Parsing files to extract column information...</div>

ui/src/dialogs/datasets.ts

@@ -49,6 +49,11 @@ export const datasetTypes: Record<
     fileType: "Xsv",
     hasTags: true,
   },
+  TaggedAb1: {
+    label: "Tagged Sanger AB1",
+    fileType: "Ab1",
+    hasTags: true,
+  },
   CellRangerMTX: {
     label: "CellRanger MTX",
     fileType: "CellRangerMTX",

ui/src/dialogs/file_name_parser.test.ts

@@ -85,13 +85,90 @@ test.for([
     target: "FebControl10_sampled_R2.fastq.gz",
     match: undefined,
   },
+  {
+    pattern: "{{Sample}}_{{:Primer}}.ab1",
+    target: "A1_Fwd.ab1",
+    match: {
+      sample: { value: "A1" },
+      tags: { Primer: { value: "Fwd" } },
+    },
+  },
+  {
+    pattern: "{{Sample}}_{{:Primer}}.ab1",
+    target: "H12_Rev.ab1",
+    match: {
+      sample: { value: "H12" },
+      tags: { Primer: { value: "Rev" } },
+    },
+  },
+  {
+    pattern: "{{Sample}}_{{:Primer}}.ab1",
+    target: "A1_Fwd.fastq.gz",
+    match: undefined,
+  },
 ])("matching test for $pattern and $target", ({ pattern, target, match }, { expect }) => {
   const fileNamePattern = FileNamePattern.parse(pattern);
   const actualMatch = fileNamePattern.match(target);
   if (match === undefined) expect(actualMatch).toBeUndefined();
   else expect(actualMatch).to.toMatchObject(match);
 });
 
+test("infer ab1 pattern from a two-primer 96-well plate", ({ expect }) => {
+  const wells = [];
+  for (const row of ["A", "B", "C", "D", "E", "F", "G", "H"])
+    for (let col = 1; col <= 12; col++) wells.push(`${row}${col}`);
+
+  const fileNames = [...wells.map((w) => `${w}_Fwd.ab1`), ...wells.map((w) => `${w}_Rev.ab1`)];
+  expect(fileNames.length).toBe(192);
+
+  const inferred = inferFileNamePattern(fileNames);
+  expect(inferred).toBeDefined();
+  expect(inferred!.pattern.rawPattern).toBe("{{Sample}}_{{:Primer}}.ab1");
+  expect(inferred!.pattern.datasetType).toBe("TaggedAb1");
+  expect(inferred!.pattern.tags).toEqual(["Primer"]);
+
+  // 96 wells, not 192 — the primer must land in the tag, not in the sample name.
+  const samples = new Set(fileNames.map((f) => inferred!.pattern.match(f)!.sample.value));
+  expect(samples.size).toBe(96);
+});
+
+test("infer ab1 pattern with a dash separator", ({ expect }) => {
+  const inferred = inferFileNamePattern(["A1-M13F.ab1", "A1-M13R.ab1", "B2-M13F.ab1"]);
+  expect(inferred?.pattern.rawPattern).toBe("{{Sample}}-{{:Primer}}.ab1");
+  expect(inferred?.pattern.datasetType).toBe("TaggedAb1");
+});
+
+test("adding tags to the dedup key leaves fastq inference alone", ({ expect }) => {
+  const inferred = inferFileNamePattern([
+    "SampleA_L001_R1_001.fastq.gz",
+    "SampleA_L001_R2_001.fastq.gz",
+    "SampleB_L001_R1_001.fastq.gz",
+    "SampleB_L001_R2_001.fastq.gz",
+  ]);
+  expect(inferred?.pattern.rawPattern).toBe("{{Sample}}_L{{n}}_{{RR}}_{{n}}.fastq.gz");
+  expect(inferred?.readIndices).toEqual(["R1", "R2"]);
+});
+
+test("ab1 pattern resolves to the TaggedAb1 dataset type", ({ expect }) => {
+  const pattern = FileNamePattern.parse("{{Sample}}_{{:Primer}}.ab1");
+  expect(pattern.fileContentType).toBe("Ab1");
+  expect(pattern.hasTagMatchers).toBe(true);
+  expect(pattern.tags).toEqual(["Primer"]);
+  expect(pattern.datasetType).toBe("TaggedAb1");
+});
+
+test("compressed ab1 pattern is rejected", ({ expect }) => {
+  expect(() => FileNamePattern.parse("{{Sample}}_{{:Primer}}.ab1.gz")).toThrow(
+    "Compressed AB1 files are not supported",
+  );
+});
+
+test("ab1 pattern without tags has no dataset type", ({ expect }) => {
+  const pattern = FileNamePattern.parse("{{Sample}}.ab1");
+  expect(pattern.fileContentType).toBe("Ab1");
+  expect(pattern.datasetType).toBeUndefined();
+});
+
 test("wrapped string builder - pattern", ({ expect }) => {
   const fileNamePattern = FileNamePattern.parse("{{s}}{{R}}L{{n}}D{{*}}.fastq.gz");
   const wrapped = fileNamePattern.buildFormattedPattern({

ui/src/dialogs/file_name_parser.ts

@@ -10,7 +10,8 @@ export type FileContentType =
   | "CellRangerMTX"
   | "H5AD"
   | "H5"
-  | "Seurat";
+  | "Seurat"
+  | "Ab1";
 
 function extractFileContentType(pattern: string): FileContentType {
   if (pattern.includes("CellRangerFileRole")) return "CellRangerMTX";
@@ -23,7 +24,11 @@ function extractFileContentType(pattern: string): FileContentType {
   else if (pt.endsWith("h5ad")) return "H5AD";
   else if (pt.endsWith("h5")) return "H5";
   else if (["rds", "RDS"].some((rds) => pt.endsWith(rds))) return "Seurat";
-  else if (
+  else if (["ab1", "AB1"].some((ab1) => pt.endsWith(ab1))) {
+    // Readers of the dataset column expect a raw trace, so there is no ab1.gz to publish.
+    if (pt !== pattern) throw new Error("Compressed AB1 files are not supported");
+    return "Ab1";
+  } else if (
     pt.endsWith("matrix.mtx") ||
     pt.endsWith("features.tsv") ||
     pt.endsWith("genes.tsv") ||
@@ -122,6 +127,8 @@ export class FileNamePattern {
         return "H5";
       case "Seurat":
         return "Seurat";
+      case "Ab1":
+        return this.hasTagMatchers ? "TaggedAb1" : undefined;
       case "CellRangerMTX":
         return "CellRangerMTX";
       default:
@@ -483,6 +490,20 @@ const wellKnownPattern: WellKnownPattern[] = [
     extensions: ["rds", "RDS"],
     minimalPercent: 0.9,
   },
+  // One Sanger reaction is one template plus one primer, so a sample read with several
+  // primers yields one file per primer. The primer name is the usual separator.
+  {
+    patternWithoutExtension: "{{Sample}}_{{:Primer}}",
+    defaultReadIndices: [],
+    extensions: ["ab1", "AB1"],
+    minimalPercent: 0.9,
+  },
+  {
+    patternWithoutExtension: "{{Sample}}-{{:Primer}}",
+    defaultReadIndices: [],
+    extensions: ["ab1", "AB1"],
+    minimalPercent: 0.9,
+  },
 ];
 
 export type InferFileNamePatternOps = {
@@ -527,6 +548,9 @@ export function inferFileNamePattern(
           if (match.lane) sample += "___" + match.lane.value;
           if (match.readIndex) sample += "___" + match.readIndex.value;
           if (match.cellRangerFileRole) sample += "___" + match.cellRangerFileRole.value;
+          // Tags separate files of one sample just as lane and read index do. Without them a
+          // tagged candidate looks like a duplicate on its second file and is always rejected.
+          for (const tag of pattern.tags) sample += "___" + (match.tags?.[tag]?.value ?? "");
           if (samples.has(sample)) continue outer;
           samples.add(sample);
           matchedFiles++;

ui/src/pages/DatasetPage.vue

@@ -16,6 +16,7 @@ import FastaDatasetPage from "../pages/FastaDatasetPage.vue";
 import FastqDatasetPage from "../pages/FastqDatasetPage.vue";
 import MultilaneFastqDatasetPage from "../pages/MultilaneFastqDatasetPage.vue";
 import MultiplexedFastqDatasetPage from "../pages/MultiplexedFastqDatasetPage.vue";
+import TaggedAb1DatasetPage from "../pages/TaggedAb1DatasetPage.vue";
 import TaggedFastqDatasetPage from "../pages/TaggedFastqDatasetPage.vue";
 import TaggedXsvDatasetPage from "../pages/TaggedXsvDatasetPage.vue";
 import XsvDatasetPage from "../pages/XsvDatasetPage.vue";
@@ -120,6 +121,9 @@ const datasetTypeLabel = datasetTypeLabels[dataset.content.type];
     <template v-else-if="dataset.content.type === 'TaggedXsv'">
       <TaggedXsvDatasetPage />
     </template>
+    <template v-else-if="dataset.content.type === 'TaggedAb1'">
+      <TaggedAb1DatasetPage />
+    </template>
     <template v-else-if="dataset.content.type === 'CellRangerMTX'">
       <CellRangerMtxDatasetPage />
     </template>