From a666fb7ea6e7ac723494fc712072e5e799af18f9 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Fri, 4 Sep 2026 11:00:22 +0000 Subject: [PATCH 1/7] Extract ProjectConfigOptions from cli/index.ts --- cli/BUILD | 1 + cli/index.ts | 163 +-------------------------------- cli/project_config_options.ts | 166 ++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 162 deletions(-) create mode 100644 cli/project_config_options.ts diff --git a/cli/BUILD b/cli/BUILD index 053df52d4..3572f8e80 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -12,6 +12,7 @@ ts_library( "console.ts", "credentials.ts", "index.ts", + "project_config_options.ts", "util.ts", "yargswrapper.ts", ], diff --git a/cli/index.ts b/cli/index.ts index 6d3507a5e..f088f1fee 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -28,6 +28,7 @@ import { printWarning } from "df/cli/console"; import { getBigQueryCredentials } from "df/cli/credentials"; +import { ProjectConfigOptions } from "df/cli/project_config_options"; import { actuallyResolve, assertPathExists, @@ -1003,165 +1004,3 @@ function createLineageEmitter( readCredentials }); } - -class ProjectConfigOptions { - public static defaultDatabase: INamedOption = { - name: "default-database", - option: { - describe: - "The default database to use, equivalent to Google Cloud Project ID. If unset, " + - "the value from workflow_settings.yaml is used.", - type: "string" - } - }; - - public static defaultSchema: INamedOption = { - name: "default-schema", - option: { - describe: - "Override for the default schema name. If unset, the value from workflow_settings.yaml is used." - } - }; - - public static defaultLocation: INamedOption = { - name: "default-location", - option: { - describe: - "The default location to use. See " + - "https://cloud.google.com/bigquery/docs/locations for supported values. If unset, the " + - "value from workflow_settings.yaml is used." - } - }; - - public static assertionSchema: INamedOption = { - name: "assertion-schema", - option: { - describe: "Default assertion schema. If unset, the value from workflow_settings.yaml is used." - } - }; - - public static databaseSuffix: INamedOption = { - name: "database-suffix", - option: { - describe: "Default assertion schema. If unset, the value from workflow_settings.yaml is used." - } - }; - - public static vars: INamedOption = { - name: "vars", - option: { - describe: - "Override for variables to inject via '--vars=someKey=someValue,a=b', referenced by " + - "`dataform.projectConfig.vars.someValue`. If unset, the value from workflow_settings.yaml is used.", - type: "string", - default: null, - coerce: (rawVarsString: string | null) => { - const variables: { [key: string]: string } = {}; - rawVarsString?.split(",").forEach(keyValueStr => { - const [key, value] = keyValueStr.split("="); - variables[key] = value; - }); - return variables; - } - } - }; - - public static schemaSuffix: INamedOption = { - name: "schema-suffix", - option: { - describe: - "A suffix to be appended to output schema names. If unset, the value from workflow_settings.yaml " + - "is used." - }, - check: (argv: yargs.Arguments) => { - if ( - argv[ProjectConfigOptions.schemaSuffix.name] && - !/^[a-zA-Z_0-9]+$/.test(argv[ProjectConfigOptions.schemaSuffix.name]) - ) { - throw new Error( - `--${ProjectConfigOptions.schemaSuffix.name} should contain only ` + - `alphanumeric characters and/or underscores.` - ); - } - } - }; - - public static tablePrefix: INamedOption = { - name: "table-prefix", - option: { - describe: - "Adds a prefix for all table names. If unset, the value from workflow_settings.yaml is used." - } - }; - - public static disableAssertions: INamedOption = { - name: "disable-assertions", - option: { - describe: - "Disables all assertions including built-in assertions (uniqueKey, nonNull, rowConditions) and manual assertions (type: assertion).", - type: "boolean", - default: false - } - }; - - public static defaultReservation: INamedOption = { - name: "default-reservation", - option: { - describe: - "The default BigQuery reservation to use for execution. If unset, the value from " + - "workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.", - type: "string" - } - }; - - public static allYargsOptions = [ - ProjectConfigOptions.defaultDatabase, - ProjectConfigOptions.defaultSchema, - ProjectConfigOptions.defaultLocation, - ProjectConfigOptions.assertionSchema, - ProjectConfigOptions.vars, - ProjectConfigOptions.databaseSuffix, - ProjectConfigOptions.schemaSuffix, - ProjectConfigOptions.tablePrefix, - ProjectConfigOptions.disableAssertions, - ProjectConfigOptions.defaultReservation - ]; - - public static constructProjectConfigOverride( - argv: yargs.Arguments - ): dataform.IProjectConfig { - const projectConfigOptions: dataform.IProjectConfig = {}; - - if (argv[ProjectConfigOptions.defaultDatabase.name]) { - projectConfigOptions.defaultDatabase = argv[ProjectConfigOptions.defaultDatabase.name]; - } - if (argv[ProjectConfigOptions.defaultSchema.name]) { - projectConfigOptions.defaultSchema = argv[ProjectConfigOptions.defaultSchema.name]; - } - if (argv[ProjectConfigOptions.defaultLocation.name]) { - projectConfigOptions.defaultLocation = argv[ProjectConfigOptions.defaultLocation.name]; - } - if (argv[ProjectConfigOptions.assertionSchema.name]) { - projectConfigOptions.assertionSchema = argv[ProjectConfigOptions.assertionSchema.name]; - } - if (argv[ProjectConfigOptions.vars.name]) { - projectConfigOptions.vars = argv[ProjectConfigOptions.vars.name]; - } - if (argv[ProjectConfigOptions.databaseSuffix.name]) { - projectConfigOptions.databaseSuffix = argv[ProjectConfigOptions.databaseSuffix.name]; - } - if (argv[ProjectConfigOptions.schemaSuffix.name]) { - projectConfigOptions.schemaSuffix = argv[ProjectConfigOptions.schemaSuffix.name]; - } - if (argv[ProjectConfigOptions.tablePrefix.name]) { - projectConfigOptions.tablePrefix = argv[ProjectConfigOptions.tablePrefix.name]; - } - if (argv[ProjectConfigOptions.disableAssertions.name]) { - projectConfigOptions.disableAssertions = argv[ProjectConfigOptions.disableAssertions.name]; - } - if (argv[ProjectConfigOptions.defaultReservation.name]) { - projectConfigOptions.defaultReservation = argv[ProjectConfigOptions.defaultReservation.name]; - } - return projectConfigOptions; - } -} diff --git a/cli/project_config_options.ts b/cli/project_config_options.ts new file mode 100644 index 000000000..f8c07a430 --- /dev/null +++ b/cli/project_config_options.ts @@ -0,0 +1,166 @@ +import yargs from "yargs"; + +import { INamedOption } from "df/cli/yargswrapper"; +import { dataform } from "df/protos/ts"; + +export class ProjectConfigOptions { + public static defaultDatabase: INamedOption = { + name: "default-database", + option: { + describe: + "The default database to use, equivalent to Google Cloud Project ID. If unset, " + + "the value from workflow_settings.yaml is used.", + type: "string" + } + }; + + public static defaultSchema: INamedOption = { + name: "default-schema", + option: { + describe: + "Override for the default schema name. If unset, the value from workflow_settings.yaml is used." + } + }; + + public static defaultLocation: INamedOption = { + name: "default-location", + option: { + describe: + "The default location to use. See " + + "https://cloud.google.com/bigquery/docs/locations for supported values. If unset, the " + + "value from workflow_settings.yaml is used." + } + }; + + public static assertionSchema: INamedOption = { + name: "assertion-schema", + option: { + describe: "Default assertion schema. If unset, the value from workflow_settings.yaml is used." + } + }; + + public static databaseSuffix: INamedOption = { + name: "database-suffix", + option: { + describe: "Default assertion schema. If unset, the value from workflow_settings.yaml is used." + } + }; + + public static vars: INamedOption = { + name: "vars", + option: { + describe: + "Override for variables to inject via '--vars=someKey=someValue,a=b', referenced by " + + "`dataform.projectConfig.vars.someValue`. If unset, the value from workflow_settings.yaml is used.", + type: "string", + default: null, + coerce: (rawVarsString: string | null) => { + const variables: { [key: string]: string } = {}; + rawVarsString?.split(",").forEach(keyValueStr => { + const [key, value] = keyValueStr.split("="); + variables[key] = value; + }); + return variables; + } + } + }; + + public static schemaSuffix: INamedOption = { + name: "schema-suffix", + option: { + describe: + "A suffix to be appended to output schema names. If unset, the value from workflow_settings.yaml " + + "is used." + }, + check: (argv: yargs.Arguments) => { + if ( + argv[ProjectConfigOptions.schemaSuffix.name] && + !/^[a-zA-Z_0-9]+$/.test(argv[ProjectConfigOptions.schemaSuffix.name]) + ) { + throw new Error( + `--${ProjectConfigOptions.schemaSuffix.name} should contain only ` + + `alphanumeric characters and/or underscores.` + ); + } + } + }; + + public static tablePrefix: INamedOption = { + name: "table-prefix", + option: { + describe: + "Adds a prefix for all table names. If unset, the value from workflow_settings.yaml is used." + } + }; + + public static disableAssertions: INamedOption = { + name: "disable-assertions", + option: { + describe: + "Disables all assertions including built-in assertions (uniqueKey, nonNull, rowConditions) and manual assertions (type: assertion).", + type: "boolean", + default: false + } + }; + + public static defaultReservation: INamedOption = { + name: "default-reservation", + option: { + describe: + "The default BigQuery reservation to use for execution. If unset, the value from " + + "workflow_settings.yaml is used. If neither is set, default BigQuery behavior applies.", + type: "string" + } + }; + + public static allYargsOptions = [ + ProjectConfigOptions.defaultDatabase, + ProjectConfigOptions.defaultSchema, + ProjectConfigOptions.defaultLocation, + ProjectConfigOptions.assertionSchema, + ProjectConfigOptions.vars, + ProjectConfigOptions.databaseSuffix, + ProjectConfigOptions.schemaSuffix, + ProjectConfigOptions.tablePrefix, + ProjectConfigOptions.disableAssertions, + ProjectConfigOptions.defaultReservation + ]; + + public static constructProjectConfigOverride( + argv: yargs.Arguments + ): dataform.IProjectConfig { + const projectConfigOptions: dataform.IProjectConfig = {}; + + if (argv[ProjectConfigOptions.defaultDatabase.name]) { + projectConfigOptions.defaultDatabase = argv[ProjectConfigOptions.defaultDatabase.name]; + } + if (argv[ProjectConfigOptions.defaultSchema.name]) { + projectConfigOptions.defaultSchema = argv[ProjectConfigOptions.defaultSchema.name]; + } + if (argv[ProjectConfigOptions.defaultLocation.name]) { + projectConfigOptions.defaultLocation = argv[ProjectConfigOptions.defaultLocation.name]; + } + if (argv[ProjectConfigOptions.assertionSchema.name]) { + projectConfigOptions.assertionSchema = argv[ProjectConfigOptions.assertionSchema.name]; + } + if (argv[ProjectConfigOptions.vars.name]) { + projectConfigOptions.vars = argv[ProjectConfigOptions.vars.name]; + } + if (argv[ProjectConfigOptions.databaseSuffix.name]) { + projectConfigOptions.databaseSuffix = argv[ProjectConfigOptions.databaseSuffix.name]; + } + if (argv[ProjectConfigOptions.schemaSuffix.name]) { + projectConfigOptions.schemaSuffix = argv[ProjectConfigOptions.schemaSuffix.name]; + } + if (argv[ProjectConfigOptions.tablePrefix.name]) { + projectConfigOptions.tablePrefix = argv[ProjectConfigOptions.tablePrefix.name]; + } + if (argv[ProjectConfigOptions.disableAssertions.name]) { + projectConfigOptions.disableAssertions = argv[ProjectConfigOptions.disableAssertions.name]; + } + if (argv[ProjectConfigOptions.defaultReservation.name]) { + projectConfigOptions.defaultReservation = argv[ProjectConfigOptions.defaultReservation.name]; + } + return projectConfigOptions; + } +} From 440b4da84a5529913f80fe79d1a39939ca6a6f67 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Mon, 7 Sep 2026 07:01:06 +0000 Subject: [PATCH 2/7] Extract help, init, and install commands from cli/index.ts --- cli/BUILD | 5 ++ cli/commands/help_command.ts | 13 ++++ cli/commands/index.ts | 3 + cli/commands/init_command.ts | 79 +++++++++++++++++++ cli/commands/install_command.ts | 17 +++++ cli/common_options.ts | 34 +++++++++ cli/index.ts | 129 ++------------------------------ 7 files changed, 157 insertions(+), 123 deletions(-) create mode 100644 cli/commands/help_command.ts create mode 100644 cli/commands/index.ts create mode 100644 cli/commands/init_command.ts create mode 100644 cli/commands/install_command.ts create mode 100644 cli/common_options.ts diff --git a/cli/BUILD b/cli/BUILD index 3572f8e80..a1d5b8228 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -9,6 +9,11 @@ package(default_visibility = ["//visibility:public"]) ts_library( name = "cli", srcs = [ + "commands/help_command.ts", + "commands/index.ts", + "commands/init_command.ts", + "commands/install_command.ts", + "common_options.ts", "console.ts", "credentials.ts", "index.ts", diff --git a/cli/commands/help_command.ts b/cli/commands/help_command.ts new file mode 100644 index 000000000..4d79ddb4c --- /dev/null +++ b/cli/commands/help_command.ts @@ -0,0 +1,13 @@ +import { ICommand } from "df/cli/yargswrapper"; + +// This dummy command is a hack with the only goal of displaying "help" as a command in the CLI +// and we need it because of the limitations of yargs considering "help" as an option and not as a command. +export const helpCommand: ICommand = { + format: "help [command]", + description: "Show help. If [command] is specified, the help is for the given command.", + positionalOptions: [], + options: [], + processFn: async () => { + return 0; + } +}; diff --git a/cli/commands/index.ts b/cli/commands/index.ts new file mode 100644 index 000000000..e7ce5b117 --- /dev/null +++ b/cli/commands/index.ts @@ -0,0 +1,3 @@ +export { helpCommand } from "df/cli/commands/help_command"; +export { initCommand, icebergOption } from "df/cli/commands/init_command"; +export { installCommand } from "df/cli/commands/install_command"; diff --git a/cli/commands/init_command.ts b/cli/commands/init_command.ts new file mode 100644 index 000000000..6cd852b02 --- /dev/null +++ b/cli/commands/init_command.ts @@ -0,0 +1,79 @@ +import yargs from "yargs"; + +import { init } from "df/cli/api"; +import { projectDirOption } from "df/cli/common_options"; +import { print, printInitResult } from "df/cli/console"; +import { ProjectConfigOptions } from "df/cli/project_config_options"; +import { promptForIcebergConfig } from "df/cli/util"; +import { ICommand, INamedOption } from "df/cli/yargswrapper"; +import { dataform } from "df/protos/ts"; + +export const icebergOption: INamedOption = { + name: "iceberg", + option: { + describe: "Initialize the project with workflow-level Iceberg tables configuration.", + type: "boolean", + default: false + } +}; + +export const initCommand: ICommand = { + format: + `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` + + ` [${ProjectConfigOptions.defaultLocation.name}]`, + description: "Create a new dataform project.", + positionalOptions: [ + projectDirOption, + { + name: ProjectConfigOptions.defaultDatabase.name, + option: { + describe: "The default database to use, equivalent to Google Cloud Project ID." + }, + check: (argv: yargs.Arguments) => { + if (!argv[ProjectConfigOptions.defaultDatabase.name]) { + throw new Error( + `The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` + + `required. Use "dataform help init" for more info.` + ); + } + } + }, + { + name: ProjectConfigOptions.defaultLocation.name, + option: { + describe: + "The default location to use. See " + + "https://cloud.google.com/bigquery/docs/locations for supported values." + }, + check: (argv: yargs.Arguments) => { + if (!argv[ProjectConfigOptions.defaultLocation.name]) { + throw new Error( + `The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` + + `required. Use "dataform help init" for more info.` + ); + } + } + } + ], + options: [icebergOption], + processFn: async argv => { + const projectDir = argv[projectDirOption.name]; + const projectConfig: dataform.IProjectConfig = { + defaultDatabase: argv[ProjectConfigOptions.defaultDatabase.name], + defaultLocation: argv[ProjectConfigOptions.defaultLocation.name] + }; + + if (argv[icebergOption.name]) { + const icebergConfig = promptForIcebergConfig(); + if (icebergConfig) { + projectConfig.defaultIcebergConfig = icebergConfig; + } + } + + print("Writing project files...\n"); + + const initResult = await init(projectDir, projectConfig); + printInitResult(initResult); + return 0; + } +}; diff --git a/cli/commands/install_command.ts b/cli/commands/install_command.ts new file mode 100644 index 000000000..8e2746e06 --- /dev/null +++ b/cli/commands/install_command.ts @@ -0,0 +1,17 @@ +import { install } from "df/cli/api"; +import { projectDirMustExistOption } from "df/cli/common_options"; +import { print, printSuccess } from "df/cli/console"; +import { ICommand } from "df/cli/yargswrapper"; + +export const installCommand: ICommand = { + format: `install [${projectDirMustExistOption.name}]`, + description: "Install a project's NPM dependencies.", + positionalOptions: [projectDirMustExistOption], + options: [], + processFn: async argv => { + print("Installing NPM dependencies...\n"); + await install(argv[projectDirMustExistOption.name]); + printSuccess("Project dependencies successfully installed."); + return 0; + } +}; diff --git a/cli/common_options.ts b/cli/common_options.ts new file mode 100644 index 000000000..64d1e0f11 --- /dev/null +++ b/cli/common_options.ts @@ -0,0 +1,34 @@ +import * as fs from "fs"; +import * as path from "path"; +import yargs from "yargs"; + +import { actuallyResolve, assertPathExists } from "df/cli/util"; +import { INamedOption } from "df/cli/yargswrapper"; + +export const projectDirOption: INamedOption = { + name: "project-dir", + option: { + describe: "The Dataform project directory.", + default: ".", + coerce: actuallyResolve + } +}; + +export const projectDirMustExistOption: INamedOption = { + ...projectDirOption, + check: (argv: yargs.Arguments) => { + assertPathExists(argv[projectDirOption.name]); + const dataformJsonPath = path.resolve(argv[projectDirOption.name], "dataform.json"); + const workflowSettingsYamlPath = path.resolve( + argv[projectDirOption.name], + "workflow_settings.yaml" + ); + if (!fs.existsSync(dataformJsonPath) && !fs.existsSync(workflowSettingsYamlPath)) { + throw new Error( + `${ + argv[projectDirOption.name] + } does not appear to be a dataform directory (missing workflow_settings.yaml file).` + ); + } + } +}; diff --git a/cli/index.ts b/cli/index.ts index f088f1fee..9c109fd41 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -5,12 +5,14 @@ import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; -import { build, compile, credentials, init, install, prune, run, test } from "df/cli/api"; +import { build, compile, credentials, prune, run, test } from "df/cli/api"; import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { LineageEmitter } from "df/cli/api/lineage/emitter"; import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; import { prettyJsonStringify } from "df/cli/api/utils"; +import { helpCommand, initCommand, installCommand } from "df/cli/commands"; +import { projectDirMustExistOption, projectDirOption } from "df/cli/common_options"; import { compiledGraphOutputType, Logger, @@ -22,7 +24,6 @@ import { printExecutionGraph, printFormatFilesResult, printInitCredsResult, - printInitResult, printSuccess, printTestResult, printWarning @@ -31,9 +32,7 @@ import { getBigQueryCredentials } from "df/cli/credentials"; import { ProjectConfigOptions } from "df/cli/project_config_options"; import { actuallyResolve, - assertPathExists, compiledGraphHasErrors, - promptForIcebergConfig, } from "df/cli/util"; import { createYargsCli, INamedOption } from "df/cli/yargswrapper"; import { targetAsReadableString } from "df/core/targets"; @@ -54,33 +53,6 @@ process.on("unhandledRejection", async (reason: any) => { // TODO: Since yargs launched an actually well typed API in version 12, let's use it as this file is currently not type checked. -const projectDirOption: INamedOption = { - name: "project-dir", - option: { - describe: "The Dataform project directory.", - default: ".", - coerce: actuallyResolve - } -}; - -const projectDirMustExistOption = { - ...projectDirOption, - check: (argv: yargs.Arguments) => { - assertPathExists(argv[projectDirOption.name]); - const dataformJsonPath = path.resolve(argv[projectDirOption.name], "dataform.json"); - const workflowSettingsYamlPath = path.resolve( - argv[projectDirOption.name], - "workflow_settings.yaml" - ); - if (!fs.existsSync(dataformJsonPath) && !fs.existsSync(workflowSettingsYamlPath)) { - throw new Error( - `${ - argv[projectDirOption.name] - } does not appear to be a dataform directory (missing workflow_settings.yaml file).` - ); - } - } -}; const fullRefreshOption: INamedOption = { name: "full-refresh", @@ -308,15 +280,6 @@ const quietCompileOption: INamedOption = { } }; -const icebergOption: INamedOption = { - name: "iceberg", - option: { - describe: "Initialize the project with workflow-level Iceberg tables configuration.", - type: "boolean", - default: false, - }, -}; - const fmtIgnoreJsOption: INamedOption = { name: "ignore-js-files", option: { @@ -344,89 +307,9 @@ function getCredentialsPath(projectDir: string, credentialsPath: string) { export function runCli() { const builtYargs = createYargsCli({ commands: [ - { - // This dummy command is a hack with the only goal of displaying "help" as a command in the CLI - // and we need it because of the limitations of yargs considering "help" as an option and not as a command. - format: "help [command]", - description: "Show help. If [command] is specified, the help is for the given command.", - positionalOptions: [], - options: [], - processFn: async argv => { - return 0; - } - }, - { - format: - `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` + - ` [${ProjectConfigOptions.defaultLocation.name}]`, - description: "Create a new dataform project.", - positionalOptions: [ - projectDirOption, - { - name: ProjectConfigOptions.defaultDatabase.name, - option: { - describe: "The default database to use, equivalent to Google Cloud Project ID." - }, - check: (argv: yargs.Arguments) => { - if (!argv[ProjectConfigOptions.defaultDatabase.name]) { - throw new Error( - `The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` + - `required. Use "dataform help init" for more info.` - ); - } - } - }, - { - name: ProjectConfigOptions.defaultLocation.name, - option: { - describe: - "The default location to use. See " + - "https://cloud.google.com/bigquery/docs/locations for supported values." - }, - check: (argv: yargs.Arguments) => { - if (!argv[ProjectConfigOptions.defaultLocation.name]) { - throw new Error( - `The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` + - `required. Use "dataform help init" for more info.` - ); - } - } - } - ], - options: [icebergOption], - processFn: async argv => { - const projectDir = argv[projectDirOption.name]; - const projectConfig: dataform.IProjectConfig = { - defaultDatabase: argv[ProjectConfigOptions.defaultDatabase.name], - defaultLocation: argv[ProjectConfigOptions.defaultLocation.name], - }; - - if (argv[icebergOption.name]) { - const icebergConfig = promptForIcebergConfig(); - if(icebergConfig) { - projectConfig.defaultIcebergConfig = icebergConfig; - } - } - - print("Writing project files...\n"); - - const initResult = await init(projectDir, projectConfig); - printInitResult(initResult); - return 0; - } - }, - { - format: `install [${projectDirMustExistOption.name}]`, - description: "Install a project's NPM dependencies.", - positionalOptions: [projectDirMustExistOption], - options: [], - processFn: async argv => { - print("Installing NPM dependencies...\n"); - await install(argv[projectDirMustExistOption.name]); - printSuccess("Project dependencies successfully installed."); - return 0; - } - }, + helpCommand, + initCommand, + installCommand, { format: `init-creds [${projectDirMustExistOption.name}]`, description: From db8c5e866a63c98fd4202810f33982bf976019af Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Mon, 7 Sep 2026 07:35:43 +0000 Subject: [PATCH 3/7] Extract init-creds and format commands from cli/index.ts --- cli/BUILD | 2 + cli/commands/format_command.ts | 106 +++++++++++++++++ cli/commands/index.ts | 4 +- cli/commands/init_command.ts | 2 +- cli/commands/init_creds_command.ts | 62 ++++++++++ cli/common_options.ts | 14 +++ cli/index.ts | 185 +++-------------------------- 7 files changed, 203 insertions(+), 172 deletions(-) create mode 100644 cli/commands/format_command.ts create mode 100644 cli/commands/init_creds_command.ts diff --git a/cli/BUILD b/cli/BUILD index a1d5b8228..e6f20a2d9 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -9,9 +9,11 @@ package(default_visibility = ["//visibility:public"]) ts_library( name = "cli", srcs = [ + "commands/format_command.ts", "commands/help_command.ts", "commands/index.ts", "commands/init_command.ts", + "commands/init_creds_command.ts", "commands/install_command.ts", "common_options.ts", "console.ts", diff --git a/cli/commands/format_command.ts b/cli/commands/format_command.ts new file mode 100644 index 000000000..89f61ae2c --- /dev/null +++ b/cli/commands/format_command.ts @@ -0,0 +1,106 @@ +import * as fs from "fs"; +import * as glob from "glob"; +import * as path from "path"; +import yargs from "yargs"; + +import { actionsOption, projectDirMustExistOption } from "df/cli/common_options"; +import { printError, printFormatFilesResult, printSuccess } from "df/cli/console"; +import { ICommand, INamedOption } from "df/cli/yargswrapper"; +import { formatFile } from "df/sqlx/format"; + +const fmtIgnoreJsOption: INamedOption = { + name: "ignore-js-files", + option: { + describe: "If set, the formatter will not consider javascript files (.js)", + type: "boolean", + default: false + } +}; + +const checkOptionName = "check"; + +const checkOption: INamedOption = { + name: checkOptionName, + option: { + describe: "Check if files are formatted correctly without modifying them.", + type: "boolean", + default: false + } +}; + +export const formatCommand: ICommand = { + format: `format [${projectDirMustExistOption.name}]`, + description: "Format the dataform project's files.", + positionalOptions: [projectDirMustExistOption], + options: [actionsOption, fmtIgnoreJsOption, checkOption], + processFn: async argv => { + const extensions = argv[fmtIgnoreJsOption.name] ? "*.sqlx" : "*.{js,sqlx}"; + let actions = [`{definitions,includes}/**/${extensions}`]; + if (actionsOption.name in argv && argv[actionsOption.name].length > 0) { + actions = argv[actionsOption.name]; + } + const filenames = actions + .map((action: string) => glob.sync(action, { cwd: argv[projectDirMustExistOption.name] })) + .flat(); + + const isCheckMode = argv[checkOptionName]; + const results: Array<{ + filename: string; + err?: Error; + needsFormatting?: boolean; + }> = await Promise.all( + filenames.map(async (filename: string) => { + try { + const filePath = path.resolve(argv[projectDirMustExistOption.name], filename); + if (isCheckMode) { + // In check mode, we don't modify files, just check if they need formatting + const fileContent = fs.readFileSync(filePath).toString(); + const formattedContent = await formatFile(filePath, { + overwriteFile: false + }); + return { + filename, + needsFormatting: fileContent !== formattedContent + }; + } else { + // Normal formatting mode + await formatFile(filePath, { + overwriteFile: true + }); + return { + filename + }; + } + } catch (e) { + return { + filename, + err: e + }; + } + }) + ); + + printFormatFilesResult(results); + + // Return error code if there are any formatting errors + const failedFormatResults = results.filter(result => !!result.err); + if (failedFormatResults.length > 0) { + printError(`${failedFormatResults.length} file(s) failed to format.`); + return 1; + } + + // In check mode, return an error code if any files need formatting + if (isCheckMode) { + const filesNeedingFormatting = results.filter(result => result.needsFormatting); + if (filesNeedingFormatting.length > 0) { + printError( + `${filesNeedingFormatting.length} file(s) would be reformatted. Run the format command without --check to update.` + ); + return 1; + } + printSuccess("All files are formatted correctly!"); + } + + return 0; + } +}; diff --git a/cli/commands/index.ts b/cli/commands/index.ts index e7ce5b117..aa05b7478 100644 --- a/cli/commands/index.ts +++ b/cli/commands/index.ts @@ -1,3 +1,5 @@ +export { formatCommand } from "df/cli/commands/format_command"; export { helpCommand } from "df/cli/commands/help_command"; -export { initCommand, icebergOption } from "df/cli/commands/init_command"; +export { initCommand } from "df/cli/commands/init_command"; +export { initCredsCommand } from "df/cli/commands/init_creds_command"; export { installCommand } from "df/cli/commands/install_command"; diff --git a/cli/commands/init_command.ts b/cli/commands/init_command.ts index 6cd852b02..be45b3103 100644 --- a/cli/commands/init_command.ts +++ b/cli/commands/init_command.ts @@ -8,7 +8,7 @@ import { promptForIcebergConfig } from "df/cli/util"; import { ICommand, INamedOption } from "df/cli/yargswrapper"; import { dataform } from "df/protos/ts"; -export const icebergOption: INamedOption = { +const icebergOption: INamedOption = { name: "iceberg", option: { describe: "Initialize the project with workflow-level Iceberg tables configuration.", diff --git a/cli/commands/init_creds_command.ts b/cli/commands/init_creds_command.ts new file mode 100644 index 000000000..244875b59 --- /dev/null +++ b/cli/commands/init_creds_command.ts @@ -0,0 +1,62 @@ +import * as fs from "fs"; +import * as path from "path"; +import yargs from "yargs"; + +import { credentials } from "df/cli/api"; +import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { prettyJsonStringify } from "df/cli/api/utils"; +import { projectDirMustExistOption } from "df/cli/common_options"; +import { print, printInitCredsResult, printSuccess } from "df/cli/console"; +import { getBigQueryCredentials } from "df/cli/credentials"; +import { ICommand, INamedOption } from "df/cli/yargswrapper"; + +const testConnectionOptionName = "test-connection"; + +const testConnectionOption: INamedOption = { + name: testConnectionOptionName, + option: { + describe: "If true, a test query will be run using your final credentials.", + type: "boolean", + default: true + } +}; + +export const initCredsCommand: ICommand = { + format: `init-creds [${projectDirMustExistOption.name}]`, + description: + `Create a ${credentials.CREDENTIALS_FILENAME} file for Dataform to use when ` + + `accessing BigQuery.`, + positionalOptions: [projectDirMustExistOption], + options: [testConnectionOption], + processFn: async argv => { + const finalCredentials = getBigQueryCredentials(); + if (argv[testConnectionOptionName]) { + print("\nRunning connection test..."); + const dbadapter = new BigQueryDbAdapter(finalCredentials); + const testResult = await credentials.test(dbadapter); + switch (testResult.status) { + case credentials.TestResultStatus.SUCCESSFUL: { + printSuccess("\nCredentials test query completed successfully.\n"); + break; + } + case credentials.TestResultStatus.TIMED_OUT: { + throw new Error("Credentials test connection timed out."); + } + case credentials.TestResultStatus.OTHER_ERROR: { + throw new Error( + `Credentials test query failed: ${testResult.error.stack || testResult.error.message}` + ); + } + } + } else { + print("\nCredentials test query was not run.\n"); + } + const filePath = path.resolve( + argv[projectDirMustExistOption.name], + credentials.CREDENTIALS_FILENAME + ); + fs.writeFileSync(filePath, prettyJsonStringify(finalCredentials)); + printInitCredsResult(filePath); + return 0; + } +}; diff --git a/cli/common_options.ts b/cli/common_options.ts index 64d1e0f11..edf6ff915 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -32,3 +32,17 @@ export const projectDirMustExistOption: INamedOption = } } }; + +// Splits repeated and comma-separated values into a flat list, e.g. +// `--actions a,b --actions c` -> ["a", "b", "c"]. +export const splitCommas = (raw: string[] | null) => raw.map(value => value.split(",")).flat(); + +export const actionsOption: INamedOption = { + name: "actions", + option: { + describe: "A list of action names or patterns to run. Can include '*' wildcards.", + type: "array", + coerce: splitCommas + } +}; + diff --git a/cli/index.ts b/cli/index.ts index 9c109fd41..fc49f331a 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -1,6 +1,5 @@ import * as chokidar from "chokidar"; import * as fs from "fs"; -import * as glob from "glob"; import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; @@ -11,8 +10,19 @@ import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { LineageEmitter } from "df/cli/api/lineage/emitter"; import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; import { prettyJsonStringify } from "df/cli/api/utils"; -import { helpCommand, initCommand, installCommand } from "df/cli/commands"; -import { projectDirMustExistOption, projectDirOption } from "df/cli/common_options"; +import { + formatCommand, + helpCommand, + initCommand, + initCredsCommand, + installCommand +} from "df/cli/commands"; +import { + actionsOption, + projectDirMustExistOption, + projectDirOption, + splitCommas +} from "df/cli/common_options"; import { compiledGraphOutputType, Logger, @@ -22,13 +32,10 @@ import { printError, printExecutedAction, printExecutionGraph, - printFormatFilesResult, - printInitCredsResult, printSuccess, printTestResult, printWarning } from "df/cli/console"; -import { getBigQueryCredentials } from "df/cli/credentials"; import { ProjectConfigOptions } from "df/cli/project_config_options"; import { actuallyResolve, @@ -37,7 +44,6 @@ import { import { createYargsCli, INamedOption } from "df/cli/yargswrapper"; import { targetAsReadableString } from "df/core/targets"; import { dataform } from "df/protos/ts"; -import { formatFile } from "df/sqlx/format"; const RECOMPILE_DELAY = 500; @@ -63,10 +69,6 @@ const fullRefreshOption: INamedOption = { } }; -// Splits repeated and comma-separated values into a flat list, e.g. -// `--actions a,b --actions c` -> ["a", "b", "c"]. -const splitCommas = (raw: string[] | null) => raw.map(value => value.split(",")).flat(); - // It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. const requiresSelection = ( name: string, @@ -80,15 +82,6 @@ const requiresSelection = ( } }; -const actionsOption: INamedOption = { - name: "actions", - option: { - describe: "A list of action names or patterns to run. Can include '*' wildcards.", - type: "array", - coerce: splitCommas - } -}; - const tagsOption: INamedOption = { name: "tags", option: { @@ -280,23 +273,11 @@ const quietCompileOption: INamedOption = { } }; -const fmtIgnoreJsOption: INamedOption = { - name: "ignore-js-files", - option: { - describe: "If set, the formatter will not consider javascript files (.js)", - type: "boolean", - default: false, - }, -}; - -const testConnectionOptionName = "test-connection"; - const watchOptionName = "watch"; const verboseOptionName = "verbose"; const dryRunOptionName = "dry-run"; const runTestsOptionName = "run-tests"; -const checkOptionName = "check"; const actionRetryLimitName = "action-retry-limit"; @@ -310,55 +291,7 @@ export function runCli() { helpCommand, initCommand, installCommand, - { - format: `init-creds [${projectDirMustExistOption.name}]`, - description: - `Create a ${credentials.CREDENTIALS_FILENAME} file for Dataform to use when ` + - `accessing BigQuery.`, - positionalOptions: [projectDirMustExistOption], - options: [ - { - name: testConnectionOptionName, - option: { - describe: "If true, a test query will be run using your final credentials.", - type: "boolean", - default: true - } - } - ], - processFn: async argv => { - const finalCredentials = getBigQueryCredentials(); - if (argv[testConnectionOptionName]) { - print("\nRunning connection test..."); - const dbadapter = new BigQueryDbAdapter(finalCredentials); - const testResult = await credentials.test(dbadapter); - switch (testResult.status) { - case credentials.TestResultStatus.SUCCESSFUL: { - printSuccess("\nCredentials test query completed successfully.\n"); - break; - } - case credentials.TestResultStatus.TIMED_OUT: { - throw new Error("Credentials test connection timed out."); - } - case credentials.TestResultStatus.OTHER_ERROR: { - throw new Error( - `Credentials test query failed: ${testResult.error.stack || - testResult.error.message}` - ); - } - } - } else { - print("\nCredentials test query was not run.\n"); - } - const filePath = path.resolve( - argv[projectDirMustExistOption.name], - credentials.CREDENTIALS_FILENAME - ); - fs.writeFileSync(filePath, prettyJsonStringify(finalCredentials)); - printInitCredsResult(filePath); - return 0; - } - }, + initCredsCommand, { format: `compile [${projectDirMustExistOption.name}]`, description: @@ -760,95 +693,7 @@ export function runCli() { return runResult.status === dataform.RunResult.ExecutionStatus.SUCCESSFUL ? 0 : 1; } }, - { - format: `format [${projectDirMustExistOption.name}]`, - description: "Format the dataform project's files.", - positionalOptions: [projectDirMustExistOption], - options: [ - actionsOption, - fmtIgnoreJsOption, - { - name: checkOptionName, - option: { - describe: "Check if files are formatted correctly without modifying them.", - type: "boolean", - default: false - } - } - ], - processFn: async argv => { - const extensions = argv[fmtIgnoreJsOption.name] ? "*.sqlx" : "*.{js,sqlx}"; - let actions = [`{definitions,includes}/**/${extensions}`]; - if (actionsOption.name in argv && argv[actionsOption.name].length > 0) { - actions = argv[actionsOption.name]; - } - const filenames = actions - .map((action: string) => - glob.sync(action, { cwd: argv[projectDirMustExistOption.name] }) - ) - .flat(); - - const isCheckMode = argv[checkOptionName]; - const results: Array<{ - filename: string; - err?: Error; - needsFormatting?: boolean; - }> = await Promise.all( - filenames.map(async (filename: string) => { - try { - const filePath = path.resolve(argv[projectDirMustExistOption.name], filename); - if (isCheckMode) { - // In check mode, we don't modify files, just check if they need formatting - const fileContent = fs.readFileSync(filePath).toString(); - const formattedContent = await formatFile(filePath, { - overwriteFile: false - }); - return { - filename, - needsFormatting: fileContent !== formattedContent - }; - } else { - // Normal formatting mode - await formatFile(filePath, { - overwriteFile: true - }); - return { - filename - }; - } - } catch (e) { - return { - filename, - err: e - }; - } - }) - ); - - printFormatFilesResult(results); - - // Return error code if there are any formatting errors - const failedFormatResults = results.filter(result => !!result.err); - if (failedFormatResults.length > 0) { - printError(`${failedFormatResults.length} file(s) failed to format.`); - return 1; - } - - // In check mode, return an error code if any files need formatting - if (isCheckMode) { - const filesNeedingFormatting = results.filter(result => result.needsFormatting); - if (filesNeedingFormatting.length > 0) { - printError( - `${filesNeedingFormatting.length} file(s) would be reformatted. Run the format command without --check to update.` - ); - return 1; - } - printSuccess("All files are formatted correctly!"); - } - - return 0; - } - } + formatCommand ] }) .scriptName("dataform") From 28d443ea69ef22a24d5a0334ec6170193b3c5a29 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Mon, 7 Sep 2026 10:00:20 +0000 Subject: [PATCH 4/7] Extract test command and shared options from cli/index.ts --- cli/BUILD | 1 + cli/commands/index.ts | 1 + cli/commands/test_command.ts | 72 +++++++++++++++++++++++++ cli/common_options.ts | 45 ++++++++++++++++ cli/index.ts | 100 ++++------------------------------- 5 files changed, 128 insertions(+), 91 deletions(-) create mode 100644 cli/commands/test_command.ts diff --git a/cli/BUILD b/cli/BUILD index e6f20a2d9..0bd2b8fb4 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -15,6 +15,7 @@ ts_library( "commands/init_command.ts", "commands/init_creds_command.ts", "commands/install_command.ts", + "commands/test_command.ts", "common_options.ts", "console.ts", "credentials.ts", diff --git a/cli/commands/index.ts b/cli/commands/index.ts index aa05b7478..5abc72481 100644 --- a/cli/commands/index.ts +++ b/cli/commands/index.ts @@ -3,3 +3,4 @@ export { helpCommand } from "df/cli/commands/help_command"; export { initCommand } from "df/cli/commands/init_command"; export { initCredsCommand } from "df/cli/commands/init_creds_command"; export { installCommand } from "df/cli/commands/install_command"; +export { testCommand } from "df/cli/commands/test_command"; diff --git a/cli/commands/test_command.ts b/cli/commands/test_command.ts new file mode 100644 index 000000000..3e041c38d --- /dev/null +++ b/cli/commands/test_command.ts @@ -0,0 +1,72 @@ +import { compile, credentials, test } from "df/cli/api"; +import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { prettyJsonStringify } from "df/cli/api/utils"; +import { + credentialsOption, + getCredentialsPath, + jsonOutputOption, + projectDirMustExistOption, + projectDirOption, + quietCompileOption, + timeoutOption +} from "df/cli/common_options"; +import { + print, + printCompiledGraphErrors, + printError, + printSuccess, + printTestResult +} from "df/cli/console"; +import { ProjectConfigOptions } from "df/cli/project_config_options"; +import { compiledGraphHasErrors } from "df/cli/util"; +import { ICommand } from "df/cli/yargswrapper"; + +export const testCommand: ICommand = { + format: `test [${projectDirMustExistOption.name}]`, + description: "Run the dataform project's unit tests.", + positionalOptions: [projectDirMustExistOption], + options: [ + credentialsOption, + timeoutOption, + jsonOutputOption, + ...ProjectConfigOptions.allYargsOptions + ], + processFn: async argv => { + if (!argv[jsonOutputOption.name]) { + print("Compiling...\n"); + } + const compiledGraph = await compile({ + projectDir: argv[projectDirMustExistOption.name], + projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), + timeoutMillis: argv[timeoutOption.name] || undefined + }); + if (compiledGraphHasErrors(compiledGraph)) { + printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); + return 1; + } + if (!argv[jsonOutputOption.name]) { + printSuccess("Compiled successfully.\n"); + } + const readCredentials = credentials.read( + getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) + ); + + if (!compiledGraph.tests.length) { + printError("No unit tests found."); + return 1; + } + + if (!argv[jsonOutputOption.name]) { + print(`Running ${compiledGraph.tests.length} unit tests...\n`); + } + const dbadapter = new BigQueryDbAdapter(readCredentials); + const testResults = await test(dbadapter, compiledGraph.tests); + if (!argv[jsonOutputOption.name]) { + testResults.forEach(testResult => printTestResult(testResult)); + } else { + // Print all results as JSON if the option is set. + print(prettyJsonStringify(testResults)); + } + return testResults.every(testResult => testResult.successful) ? 0 : 1; + } +}; diff --git a/cli/common_options.ts b/cli/common_options.ts index edf6ff915..30080c0b6 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -1,7 +1,9 @@ import * as fs from "fs"; +import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; +import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials"; import { actuallyResolve, assertPathExists } from "df/cli/util"; import { INamedOption } from "df/cli/yargswrapper"; @@ -46,3 +48,46 @@ export const actionsOption: INamedOption = { } }; +export function getCredentialsPath(projectDir: string, credentialsPath: string) { + return actuallyResolve(projectDir, credentialsPath); +} + +export const credentialsOption: INamedOption = { + name: "credentials", + option: { + describe: "The location of the credentials JSON file to use.", + default: CREDENTIALS_FILENAME + }, + check: (argv: yargs.Arguments) => + getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) +}; + +export const jsonOutputOption: INamedOption = { + name: "json", + option: { + describe: "Outputs a JSON representation of the compiled project or test results.", + type: "boolean", + default: false + } +}; + +export const timeoutOption: INamedOption = { + name: "timeout", + option: { + describe: "Duration to allow project compilation to complete. Examples: '1s', '10m', etc.", + type: "string", + default: null, + coerce: (rawTimeoutString: string | null) => + rawTimeoutString ? parseDuration(rawTimeoutString) : null + } +}; + +export const quietCompileOption: INamedOption = { + name: "quiet", + option: { + describe: "Less verbose compilation output. Example usage: 'dataform compile --quiet'", + type: "boolean", + default: false + } +}; + diff --git a/cli/index.ts b/cli/index.ts index fc49f331a..9c985eb47 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -5,7 +5,6 @@ import * as path from "path"; import yargs from "yargs"; import { build, compile, credentials, prune, run, test } from "df/cli/api"; -import { CREDENTIALS_FILENAME } from "df/cli/api/commands/credentials"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { LineageEmitter } from "df/cli/api/lineage/emitter"; import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; @@ -15,13 +14,19 @@ import { helpCommand, initCommand, initCredsCommand, - installCommand + installCommand, + testCommand } from "df/cli/commands"; import { actionsOption, + credentialsOption, + getCredentialsPath, + jsonOutputOption, projectDirMustExistOption, projectDirOption, - splitCommas + quietCompileOption, + splitCommas, + timeoutOption } from "df/cli/common_options"; import { compiledGraphOutputType, @@ -38,7 +43,6 @@ import { } from "df/cli/console"; import { ProjectConfigOptions } from "df/cli/project_config_options"; import { - actuallyResolve, compiledGraphHasErrors, } from "df/cli/util"; import { createYargsCli, INamedOption } from "df/cli/yargswrapper"; @@ -151,16 +155,6 @@ const outputIncludeDependentsOption: INamedOption = { check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption) }; -const credentialsOption: INamedOption = { - name: "credentials", - option: { - describe: "The location of the credentials JSON file to use.", - default: CREDENTIALS_FILENAME - }, - check: (argv: yargs.Arguments) => - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) -}; - const emitLineageOption: INamedOption = { name: "emit-lineage", option: { @@ -171,15 +165,6 @@ const emitLineageOption: INamedOption = { } }; -const jsonOutputOption: INamedOption = { - name: "json", - option: { - describe: "Outputs a JSON representation of the compiled project or test results.", - type: "boolean", - default: false - } -}; - const dotOutputOption: INamedOption = { name: "dot", option: { @@ -195,17 +180,6 @@ const dotOutputOption: INamedOption = { }; -const timeoutOption: INamedOption = { - name: "timeout", - option: { - describe: "Duration to allow project compilation to complete. Examples: '1s', '10m', etc.", - type: "string", - default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null - } -}; - const executionTimeoutOption: INamedOption = { name: "execution-timeout", option: { @@ -264,15 +238,6 @@ const bigqueryJobLabelsOption: INamedOption = { } }; -const quietCompileOption: INamedOption = { - name: "quiet", - option: { - describe: "Less verbose compilation output. Example usage: 'dataform compile --quiet'", - type: "boolean", - default: false - } -}; - const watchOptionName = "watch"; const verboseOptionName = "verbose"; @@ -281,10 +246,6 @@ const runTestsOptionName = "run-tests"; const actionRetryLimitName = "action-retry-limit"; -function getCredentialsPath(projectDir: string, credentialsPath: string) { - return actuallyResolve(projectDir, credentialsPath); -} - export function runCli() { const builtYargs = createYargsCli({ commands: [ @@ -438,50 +399,7 @@ export function runCli() { } } }, - { - format: `test [${projectDirMustExistOption.name}]`, - description: "Run the dataform project's unit tests.", - positionalOptions: [projectDirMustExistOption], - options: [credentialsOption, timeoutOption, jsonOutputOption, ...ProjectConfigOptions.allYargsOptions], - processFn: async argv => { - if (!argv[jsonOutputOption.name]) { - print("Compiling...\n"); - } - const compiledGraph = await compile({ - projectDir: argv[projectDirMustExistOption.name], - projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), - timeoutMillis: argv[timeoutOption.name] || undefined - }); - if (compiledGraphHasErrors(compiledGraph)) { - printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); - return 1; - } - if (!argv[jsonOutputOption.name]) { - printSuccess("Compiled successfully.\n"); - } - const readCredentials = credentials.read( - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) - ); - - if (!compiledGraph.tests.length) { - printError("No unit tests found."); - return 1; - } - - if (!argv[jsonOutputOption.name]) { - print(`Running ${compiledGraph.tests.length} unit tests...\n`); - } - const dbadapter = new BigQueryDbAdapter(readCredentials); - const testResults = await test(dbadapter, compiledGraph.tests); - if (!argv[jsonOutputOption.name]) { - testResults.forEach(testResult => printTestResult(testResult)); - } else { - // Print all results as JSON if the option is set. - print(prettyJsonStringify(testResults)); - } - return testResults.every(testResult => testResult.successful) ? 0 : 1; - } - }, + testCommand, { format: `run [${projectDirMustExistOption.name}]`, description: "Run the dataform project.", From 78bfe48a1629b5c7910ff3e1c66704fb817e7e7f Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Mon, 7 Sep 2026 10:49:51 +0000 Subject: [PATCH 5/7] Extract compile command from cli/index.ts --- cli/BUILD | 1 + cli/commands/compile_command.ts | 230 ++++++++++++++++++++++++++++++++ cli/commands/index.ts | 1 + cli/common_options.ts | 13 ++ cli/index.ts | 229 +------------------------------ 5 files changed, 249 insertions(+), 225 deletions(-) create mode 100644 cli/commands/compile_command.ts diff --git a/cli/BUILD b/cli/BUILD index 0bd2b8fb4..6f4921346 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -9,6 +9,7 @@ package(default_visibility = ["//visibility:public"]) ts_library( name = "cli", srcs = [ + "commands/compile_command.ts", "commands/format_command.ts", "commands/help_command.ts", "commands/index.ts", diff --git a/cli/commands/compile_command.ts b/cli/commands/compile_command.ts new file mode 100644 index 000000000..1733d2479 --- /dev/null +++ b/cli/commands/compile_command.ts @@ -0,0 +1,230 @@ +import * as chokidar from "chokidar"; +import yargs from "yargs"; + +import { compile, prune } from "df/cli/api"; +import { + jsonOutputOption, + projectDirMustExistOption, + quietCompileOption, + requiresSelection, + splitCommas, + timeoutOption +} from "df/cli/common_options"; +import { + compiledGraphOutputType, + Logger, + print, + printCompiledGraph, + printCompiledGraphErrors, + printError +} from "df/cli/console"; +import { ProjectConfigOptions } from "df/cli/project_config_options"; +import { compiledGraphHasErrors } from "df/cli/util"; +import { ICommand, INamedOption } from "df/cli/yargswrapper"; + +const RECOMPILE_DELAY = 500; + +// `compile` reuses the same prune() filtering as run/build, but these flags only +// filter the *printed output* -- the whole project still compiles. The `output-` +// prefix makes that distinction explicit. +const outputActionsOption: INamedOption = { + name: "output-actions", + option: { + // No wildcard support: prune()'s matchPatterns() does exact matching on the + // action name or its fully-qualified `database.schema.name`. + describe: "A list of action names to filter the compiled output to.", + type: "array", + coerce: splitCommas + } +}; + +const outputTagsOption: INamedOption = { + name: "output-tags", + option: { + describe: "A list of tags to filter the compiled output to.", + type: "array", + coerce: splitCommas + } +}; + +const outputIncludeDepsOption: INamedOption = { + name: "output-include-deps", + option: { + describe: "If set, dependencies of the selected actions are also included in the output.", + type: "boolean" + }, + check: requiresSelection("output-include-deps", outputActionsOption, outputTagsOption) +}; + +const outputIncludeDependentsOption: INamedOption = { + name: "output-include-dependents", + option: { + describe: + "If set, dependents (downstream) of the selected actions are also included in the output.", + type: "boolean" + }, + check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption) +}; + +const dotOutputOption: INamedOption = { + name: "dot", + option: { + describe: "Outputs a dot representation of the compiled project.", + type: "boolean", + default: false + }, + check: (argv: yargs.Arguments) => { + if (argv.json && argv.dot) { + throw new Error("Arguments --json and --dot are mutually exclusive."); + } + } +}; + +const watchOptionName = "watch"; +const verboseOptionName = "verbose"; + +export const compileCommand: ICommand = { + format: `compile [${projectDirMustExistOption.name}]`, + description: + "Compile the dataform project. Produces JSON output describing the non-executable graph.", + positionalOptions: [projectDirMustExistOption], + options: [ + { + name: watchOptionName, + option: { + describe: "Whether to watch the changes in the project directory.", + type: "boolean", + default: false + } + }, + jsonOutputOption, + dotOutputOption, + timeoutOption, + quietCompileOption, + outputActionsOption, + outputTagsOption, + outputIncludeDepsOption, + outputIncludeDependentsOption, + { + name: verboseOptionName, + option: { + describe: "Enable verbose compilation output. Example usage: 'dataform compile --verbose'", + type: "boolean", + default: false + }, + check: (argv: yargs.Arguments) => { + if (argv.quiet && argv.verbose) { + throw new Error("Arguments --verbose and --quiet are mutually exclusive."); + } + } + }, + ...ProjectConfigOptions.allYargsOptions + ], + processFn: async argv => { + const projectDir = argv[projectDirMustExistOption.name]; + const logger = new Logger(!argv[jsonOutputOption.name]); + + async function compileAndPrint() { + let outputType = compiledGraphOutputType.Summary; + if (argv[jsonOutputOption.name]) { + outputType = compiledGraphOutputType.Json; + } else if (argv[dotOutputOption.name]) { + outputType = compiledGraphOutputType.Dot; + } + + if (outputType === compiledGraphOutputType.Summary) { + logger.log("Compiling...\n"); + } + const compiledGraph = await compile({ + projectDir, + projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), + timeoutMillis: argv[timeoutOption.name] || undefined, + verbose: argv[verboseOptionName] || false + }); + + // The whole project must compile (ref() resolution needs every action + // registered), but the printed output can be filtered to the selected + // action(s) -- mirroring how `run`/`build` prune the graph. We only prune + // a clean graph; if compilation produced errors we print the full graph + // plus the errors, keeping graph-level errors as-is. + const hasSelector = + argv[outputActionsOption.name]?.length > 0 || argv[outputTagsOption.name]?.length > 0; + const outputGraph = + hasSelector && !compiledGraphHasErrors(compiledGraph) + ? prune(compiledGraph, { + actions: argv[outputActionsOption.name], + tags: argv[outputTagsOption.name], + includeDependencies: argv[outputIncludeDepsOption.name], + includeDependents: argv[outputIncludeDependentsOption.name] + }) + : compiledGraph; + printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]); + if (compiledGraphHasErrors(compiledGraph)) { + print(""); + printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); + return true; + } + return false; + } + + const graphHasErrors = await compileAndPrint(); + + if (!argv[watchOptionName]) { + return graphHasErrors ? 1 : 0; + } + + let watching = true; + + let timeoutID: NodeJS.Timer = null; + let isCompiling = false; + + // Initialize watcher. + const watcher = chokidar.watch(projectDir, { + ignored: /node_modules/, + persistent: true, + ignoreInitial: true, + awaitWriteFinish: { + stabilityThreshold: 1000, + pollInterval: 200 + } + }); + + const printReady = () => { + print("\nWatching for changes...\n"); + }; + // Add event listeners. + watcher + .on("ready", printReady) + .on("error", error => { + // This error is caught not if there is a compilation error, but + // if the watcher fails; this indicates an failure on our side. + printError(`Error: ${error}`); + process.exit(1); + }) + .on("all", () => { + if (timeoutID || isCompiling) { + // don't recompile many times if we changed a lot of files + clearTimeout(timeoutID); + } + + timeoutID = setTimeout(async () => { + clearTimeout(timeoutID); + + if (!isCompiling) { + isCompiling = true; + await compileAndPrint(); + printReady(); + isCompiling = false; + } + }, RECOMPILE_DELAY); + }); + process.on("SIGINT", async () => { + await watcher.close(); + watching = false; + process.exit(1); + }); + while (watching) { + await new Promise((resolve, reject) => setTimeout(() => resolve(), 100)); + } + } +}; diff --git a/cli/commands/index.ts b/cli/commands/index.ts index 5abc72481..04595cbdf 100644 --- a/cli/commands/index.ts +++ b/cli/commands/index.ts @@ -1,3 +1,4 @@ +export { compileCommand } from "df/cli/commands/compile_command"; export { formatCommand } from "df/cli/commands/format_command"; export { helpCommand } from "df/cli/commands/help_command"; export { initCommand } from "df/cli/commands/init_command"; diff --git a/cli/common_options.ts b/cli/common_options.ts index 30080c0b6..2bc7294b5 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -91,3 +91,16 @@ export const quietCompileOption: INamedOption = { } }; +// It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. +export const requiresSelection = ( + name: string, + actions: INamedOption, + tags: INamedOption +): INamedOption["check"] => (argv: yargs.Arguments) => { + if (argv[name] && !(argv[actions.name] || argv[tags.name])) { + throw new Error( + `The --${name} flag should only be supplied along with --${actions.name} or --${tags.name}.` + ); + } +}; + diff --git a/cli/index.ts b/cli/index.ts index 9c985eb47..390136a6a 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -1,15 +1,15 @@ -import * as chokidar from "chokidar"; import * as fs from "fs"; import parseDuration from "parse-duration"; import * as path from "path"; import yargs from "yargs"; -import { build, compile, credentials, prune, run, test } from "df/cli/api"; +import { build, compile, credentials, run, test } from "df/cli/api"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { LineageEmitter } from "df/cli/api/lineage/emitter"; import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; import { prettyJsonStringify } from "df/cli/api/utils"; import { + compileCommand, formatCommand, helpCommand, initCommand, @@ -25,14 +25,13 @@ import { projectDirMustExistOption, projectDirOption, quietCompileOption, + requiresSelection, splitCommas, timeoutOption } from "df/cli/common_options"; import { - compiledGraphOutputType, Logger, print, - printCompiledGraph, printCompiledGraphErrors, printError, printExecutedAction, @@ -49,8 +48,6 @@ import { createYargsCli, INamedOption } from "df/cli/yargswrapper"; import { targetAsReadableString } from "df/core/targets"; import { dataform } from "df/protos/ts"; -const RECOMPILE_DELAY = 500; - // Maximum time to wait for outstanding lineage emissions to complete before // `dataform run` returns. Lineage emission is fail-open — if we don't drain // within this window, in-flight requests are abandoned and the run status is @@ -73,19 +70,6 @@ const fullRefreshOption: INamedOption = { } }; -// It would be nice to use yargs' "implies" to implement this, but it doesn't work for some reason. -const requiresSelection = ( - name: string, - actions: INamedOption, - tags: INamedOption -): INamedOption["check"] => (argv: yargs.Arguments) => { - if (argv[name] && !(argv[actions.name] || argv[tags.name])) { - throw new Error( - `The --${name} flag should only be supplied along with --${actions.name} or --${tags.name}.` - ); - } -}; - const tagsOption: INamedOption = { name: "tags", option: { @@ -113,48 +97,6 @@ const includeDependentsOption: INamedOption = { check: requiresSelection("include-dependents", actionsOption, tagsOption) }; -// `compile` reuses the same prune() filtering as run/build, but these flags only -// filter the *printed output* -- the whole project still compiles. The `output-` -// prefix makes that distinction explicit. -const outputActionsOption: INamedOption = { - name: "output-actions", - option: { - // No wildcard support: prune()'s matchPatterns() does exact matching on the - // action name or its fully-qualified `database.schema.name`. - describe: "A list of action names to filter the compiled output to.", - type: "array", - coerce: splitCommas - } -}; - -const outputTagsOption: INamedOption = { - name: "output-tags", - option: { - describe: "A list of tags to filter the compiled output to.", - type: "array", - coerce: splitCommas - } -}; - -const outputIncludeDepsOption: INamedOption = { - name: "output-include-deps", - option: { - describe: "If set, dependencies of the selected actions are also included in the output.", - type: "boolean" - }, - check: requiresSelection("output-include-deps", outputActionsOption, outputTagsOption) -}; - -const outputIncludeDependentsOption: INamedOption = { - name: "output-include-dependents", - option: { - describe: - "If set, dependents (downstream) of the selected actions are also included in the output.", - type: "boolean" - }, - check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption) -}; - const emitLineageOption: INamedOption = { name: "emit-lineage", option: { @@ -165,21 +107,6 @@ const emitLineageOption: INamedOption = { } }; -const dotOutputOption: INamedOption = { - name: "dot", - option: { - describe: "Outputs a dot representation of the compiled project.", - type: "boolean", - default: false, - }, - check: (argv: yargs.Arguments) => { - if (argv.json && argv.dot) { - throw new Error("Arguments --json and --dot are mutually exclusive."); - } - } - -}; - const executionTimeoutOption: INamedOption = { name: "execution-timeout", option: { @@ -238,9 +165,6 @@ const bigqueryJobLabelsOption: INamedOption = { } }; -const watchOptionName = "watch"; - -const verboseOptionName = "verbose"; const dryRunOptionName = "dry-run"; const runTestsOptionName = "run-tests"; @@ -253,152 +177,7 @@ export function runCli() { initCommand, installCommand, initCredsCommand, - { - format: `compile [${projectDirMustExistOption.name}]`, - description: - "Compile the dataform project. Produces JSON output describing the non-executable graph.", - positionalOptions: [projectDirMustExistOption], - options: [ - { - name: watchOptionName, - option: { - describe: "Whether to watch the changes in the project directory.", - type: "boolean", - default: false - } - }, - jsonOutputOption, - dotOutputOption, - timeoutOption, - quietCompileOption, - outputActionsOption, - outputTagsOption, - outputIncludeDepsOption, - outputIncludeDependentsOption, - { - name: verboseOptionName, - option: { - describe: "Enable verbose compilation output. Example usage: 'dataform compile --verbose'", - type: "boolean", - default: false - }, - check: (argv: yargs.Arguments) => { - if (argv.quiet && argv.verbose) { - throw new Error("Arguments --verbose and --quiet are mutually exclusive."); - } - } - }, - ...ProjectConfigOptions.allYargsOptions - ], - processFn: async argv => { - const projectDir = argv[projectDirMustExistOption.name]; - const logger = new Logger(!argv[jsonOutputOption.name]); - - async function compileAndPrint() { - - let outputType = compiledGraphOutputType.Summary; - if (argv[jsonOutputOption.name]) { - outputType = compiledGraphOutputType.Json; - } else if (argv[dotOutputOption.name]) { - outputType = compiledGraphOutputType.Dot; - } - - if (outputType === compiledGraphOutputType.Summary) { - logger.log("Compiling...\n"); - } - const compiledGraph = await compile({ - projectDir, - projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), - timeoutMillis: argv[timeoutOption.name] || undefined, - verbose: argv[verboseOptionName] || false - }); - - // The whole project must compile (ref() resolution needs every action - // registered), but the printed output can be filtered to the selected - // action(s) -- mirroring how `run`/`build` prune the graph. We only prune - // a clean graph; if compilation produced errors we print the full graph - // plus the errors, keeping graph-level errors as-is. - const hasSelector = - argv[outputActionsOption.name]?.length > 0 || argv[outputTagsOption.name]?.length > 0; - const outputGraph = - hasSelector && !compiledGraphHasErrors(compiledGraph) - ? prune(compiledGraph, { - actions: argv[outputActionsOption.name], - tags: argv[outputTagsOption.name], - includeDependencies: argv[outputIncludeDepsOption.name], - includeDependents: argv[outputIncludeDependentsOption.name] - }) - : compiledGraph; - printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]); - if (compiledGraphHasErrors(compiledGraph)) { - print(""); - printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); - return true; - } - return false; - } - - const graphHasErrors = await compileAndPrint(); - - if (!argv[watchOptionName]) { - return graphHasErrors ? 1 : 0; - } - - let watching = true; - - let timeoutID: NodeJS.Timer = null; - let isCompiling = false; - - // Initialize watcher. - const watcher = chokidar.watch(projectDir, { - ignored: /node_modules/, - persistent: true, - ignoreInitial: true, - awaitWriteFinish: { - stabilityThreshold: 1000, - pollInterval: 200 - } - }); - - const printReady = () => { - print("\nWatching for changes...\n"); - }; - // Add event listeners. - watcher - .on("ready", printReady) - .on("error", error => { - // This error is caught not if there is a compilation error, but - // if the watcher fails; this indicates an failure on our side. - printError(`Error: ${error}`); - process.exit(1); - }) - .on("all", () => { - if (timeoutID || isCompiling) { - // don't recompile many times if we changed a lot of files - clearTimeout(timeoutID); - } - - timeoutID = setTimeout(async () => { - clearTimeout(timeoutID); - - if (!isCompiling) { - isCompiling = true; - await compileAndPrint(); - printReady(); - isCompiling = false; - } - }, RECOMPILE_DELAY); - }); - process.on("SIGINT", async () => { - await watcher.close(); - watching = false; - process.exit(1); - }); - while (watching) { - await new Promise((resolve, reject) => setTimeout(() => resolve(), 100)); - } - } - }, + compileCommand, testCommand, { format: `run [${projectDirMustExistOption.name}]`, From c18baed06332debde065067d30956f4137c03453 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Mon, 7 Sep 2026 11:46:23 +0000 Subject: [PATCH 6/7] Extract run command from cli/index.ts --- cli/BUILD | 1 + cli/commands/index.ts | 1 + cli/commands/run_command.ts | 376 +++++++++++++++++++++++++++++++++++ cli/index.ts | 384 +----------------------------------- 4 files changed, 382 insertions(+), 380 deletions(-) create mode 100644 cli/commands/run_command.ts diff --git a/cli/BUILD b/cli/BUILD index 6f4921346..bb14dcc6f 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -16,6 +16,7 @@ ts_library( "commands/init_command.ts", "commands/init_creds_command.ts", "commands/install_command.ts", + "commands/run_command.ts", "commands/test_command.ts", "common_options.ts", "console.ts", diff --git a/cli/commands/index.ts b/cli/commands/index.ts index 04595cbdf..d23cd1566 100644 --- a/cli/commands/index.ts +++ b/cli/commands/index.ts @@ -4,4 +4,5 @@ export { helpCommand } from "df/cli/commands/help_command"; export { initCommand } from "df/cli/commands/init_command"; export { initCredsCommand } from "df/cli/commands/init_creds_command"; export { installCommand } from "df/cli/commands/install_command"; +export { runCommand } from "df/cli/commands/run_command"; export { testCommand } from "df/cli/commands/test_command"; diff --git a/cli/commands/run_command.ts b/cli/commands/run_command.ts new file mode 100644 index 000000000..46110a56a --- /dev/null +++ b/cli/commands/run_command.ts @@ -0,0 +1,376 @@ +import parseDuration from "parse-duration"; +import yargs from "yargs"; + +import { build, compile, credentials, run, test } from "df/cli/api"; +import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { LineageEmitter } from "df/cli/api/lineage/emitter"; +import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; +import { prettyJsonStringify } from "df/cli/api/utils"; +import { + actionsOption, + credentialsOption, + getCredentialsPath, + jsonOutputOption, + projectDirMustExistOption, + projectDirOption, + quietCompileOption, + requiresSelection, + splitCommas, + timeoutOption +} from "df/cli/common_options"; +import { + Logger, + print, + printCompiledGraphErrors, + printError, + printExecutedAction, + printExecutionGraph, + printTestResult, + printWarning +} from "df/cli/console"; +import { ProjectConfigOptions } from "df/cli/project_config_options"; +import { compiledGraphHasErrors } from "df/cli/util"; +import { ICommand, INamedOption } from "df/cli/yargswrapper"; +import { targetAsReadableString } from "df/core/targets"; +import { dataform } from "df/protos/ts"; + +// Maximum time to wait for outstanding lineage emissions to complete before +// `dataform run` returns. Lineage emission is fail-open — if we don't drain +// within this window, in-flight requests are abandoned and the run status is +// unaffected. +const LINEAGE_DRAIN_TIMEOUT_MS = 15_000; + +const fullRefreshOption: INamedOption = { + name: "full-refresh", + option: { + describe: "Forces incremental tables to be rebuilt from scratch.", + type: "boolean", + default: false + } +}; + +const tagsOption: INamedOption = { + name: "tags", + option: { + describe: "A list of tags to filter the actions to run.", + type: "array", + coerce: splitCommas + } +}; + +const includeDepsOption: INamedOption = { + name: "include-deps", + option: { + describe: "If set, dependencies for selected actions will also be run.", + type: "boolean" + }, + check: requiresSelection("include-deps", actionsOption, tagsOption) +}; + +const includeDependentsOption: INamedOption = { + name: "include-dependents", + option: { + describe: "If set, dependents (downstream) for selected actions will also be run.", + type: "boolean" + }, + check: requiresSelection("include-dependents", actionsOption, tagsOption) +}; + +const emitLineageOption: INamedOption = { + name: "emit-lineage", + option: { + describe: + "If set, emit OpenLineage RunEvents to Knowledge Catalog Lineage for each executed action. " + + "Overrides workflow_settings.yaml lineage.enabled when specified.", + type: "boolean" + } +}; + +const executionTimeoutOption: INamedOption = { + name: "execution-timeout", + option: { + describe: + "Wall-clock deadline for the entire run (compile + all actions). When it fires, " + + "in-flight actions are cancelled and pending actions are skipped. Off by default. " + + "Examples: '10m', '2h'.", + type: "string", + default: null, + coerce: (rawTimeoutString: string | null) => + rawTimeoutString ? parseDuration(rawTimeoutString) : null + } +}; + +const jitTimeoutOption: INamedOption = { + name: "jit-timeout", + option: { + describe: + "Per-model JiT compilation worker timeout. Each action with jitCode gets " + + "its own fresh deadline; independent of --execution-timeout. When unset, no " + + "per-model cap is applied and only --execution-timeout bounds JiT work. " + + "Examples: '30s', '2m'.", + type: "string", + default: null, + coerce: (rawTimeoutString: string | null) => + rawTimeoutString ? parseDuration(rawTimeoutString) : null + } +}; + +const jobPrefixOption: INamedOption = { + name: "job-prefix", + option: { + describe: "Adds an additional prefix in the form of `dataform-${jobPrefix}-`.", + type: "string", + default: null + } +}; + +const bigqueryJobLabelsOption: INamedOption = { + name: "job-labels", + option: { + describe: + "Comma-separated list of labels to add to BigQuery jobs, e.g. 'key1=val1,key2=val2'.", + type: "string", + coerce: (raw: string | null) => { + const labels: { [key: string]: string } = {}; + raw?.split(",").forEach(kv => { + if (!kv) { + return; + } + const [key, ...rest] = kv.split("="); + labels[key] = rest.join("=") || ""; + }); + return labels; + } + } +}; + +const dryRunOptionName = "dry-run"; +const runTestsOptionName = "run-tests"; + +const actionRetryLimitName = "action-retry-limit"; + +function createLineageEmitter( + argv: yargs.Arguments, + executionGraph: dataform.IExecutionGraph, + readCredentials: dataform.IBigQuery | undefined +): LineageEmitter | undefined { + return createLineageEmitterFromFactory({ + cliEmitLineage: argv[emitLineageOption.name] as boolean | undefined, + workflowLineageEnabled: executionGraph.projectConfig?.lineageEnabled ?? undefined, + dryRun: !!argv[dryRunOptionName], + projectDir: argv[projectDirOption.name] || process.cwd(), + readCredentials + }); +} + +export const runCommand: ICommand = { + format: `run [${projectDirMustExistOption.name}]`, + description: "Run the dataform project.", + positionalOptions: [projectDirMustExistOption], + options: [ + { + name: dryRunOptionName, + option: { + describe: + "If set, BigQuery will validate the run SQL without applying changes to the warehouse.", + type: "boolean" + } + }, + { + name: runTestsOptionName, + option: { + describe: + "If set, the project's unit tests are required to pass before running the project.", + type: "boolean" + } + }, + { + name: actionRetryLimitName, + option: { + describe: "If set, idempotent actions will be retried up to the limit.", + type: "number", + default: 0 + } + }, + actionsOption, + credentialsOption, + emitLineageOption, + fullRefreshOption, + includeDepsOption, + includeDependentsOption, + jsonOutputOption, + timeoutOption, + executionTimeoutOption, + jitTimeoutOption, + tagsOption, + bigqueryJobLabelsOption, + ...ProjectConfigOptions.allYargsOptions + ], + processFn: async argv => { + const isJsonOutput = argv[jsonOutputOption.name]; + const logger = new Logger(!isJsonOutput); + + if (isJsonOutput && !argv[dryRunOptionName]) { + printError( + `For execution, the --${jsonOutputOption.name} option is only supported if the ` + + `--${dryRunOptionName} option is enabled` + ); + return; + } + if ( + !isJsonOutput && + argv[timeoutOption.name] != null && + argv[executionTimeoutOption.name] == null + ) { + printWarning( + "Note: --timeout only bounds project compilation. " + + "For a whole-run wall-clock deadline, use --execution-timeout.\n" + ); + } + logger.log("Compiling...\n"); + const compiledGraph = await compile({ + projectDir: argv[projectDirOption.name], + projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), + timeoutMillis: argv[timeoutOption.name] || undefined + }); + if (compiledGraphHasErrors(compiledGraph)) { + printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); + return 1; + } + logger.success("Compiled successfully.\n"); + const readCredentials = credentials.read( + getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) + ); + + const dbadapter = new BigQueryDbAdapter(readCredentials); + const executionGraph = await build( + compiledGraph, + { + fullRefresh: argv[fullRefreshOption.name], + actions: argv[actionsOption.name], + includeDependencies: argv[includeDepsOption.name], + includeDependents: argv[includeDependentsOption.name], + tags: argv[tagsOption.name], + timeoutMillis: argv[executionTimeoutOption.name] || undefined, + jitTimeoutMillis: argv[jitTimeoutOption.name] || undefined + }, + dbadapter + ); + + if ( + argv[dryRunOptionName] && + isJsonOutput && + // Skip the early graph print when JiT actions are present: their compiled + // SQL is only produced once the Runner triggers JiT compilation, so falling + // through ensures the JSON dry-run output includes the generated SQL rather + // than the raw jitCode. + !executionGraph.actions.some(action => !!action.jitCode) + ) { + printExecutionGraph(executionGraph, isJsonOutput); + return; + } + + if (argv[runTestsOptionName]) { + logger.log(`Running ${compiledGraph.tests.length} unit tests...\n`); + const testResults = await test(dbadapter, compiledGraph.tests); + testResults.forEach(testResult => printTestResult(testResult)); + if (testResults.some(testResult => !testResult.successful)) { + printError("\nUnit tests did not pass; aborting run."); + return 1; + } + logger.success("Unit tests completed successfully.\n"); + } + + let bigqueryOptions: {} = { + actionRetryLimit: argv[actionRetryLimitName] + }; + if (argv[dryRunOptionName]) { + bigqueryOptions = { ...bigqueryOptions, dryRun: argv[dryRunOptionName] }; + } + if (argv[jobPrefixOption.name]) { + bigqueryOptions = { ...bigqueryOptions, jobPrefix: argv[jobPrefixOption.name] }; + } + if (argv[bigqueryJobLabelsOption.name]) { + bigqueryOptions = { ...bigqueryOptions, labels: argv[bigqueryJobLabelsOption.name] }; + } + + const actionsByName = new Map(); + executionGraph.actions.forEach(action => { + actionsByName.set(targetAsReadableString(action.target), action); + }); + + if (actionsByName.size === 0) { + logger.log("No actions to run.\n"); + return 0; + } + + if (argv[dryRunOptionName]) { + logger.log("Dry running (no changes to the warehouse will be applied)..."); + } else { + logger.log("Running...\n"); + } + + const lineageEmitter = createLineageEmitter(argv, executionGraph, readCredentials); + + const runner = run( + dbadapter, + executionGraph, + { + projectDir: argv[projectDirOption.name], + bigquery: bigqueryOptions, + lineageEmitter + } + ); + process.on("SIGINT", () => { + runner.cancel(); + }); + + const alreadyPrintedActions = new Set(); + + const printExecutedGraph = (executedGraph: dataform.IRunResult) => { + executedGraph.actions + .filter( + actionResult => + actionResult.status !== dataform.ActionResult.ExecutionStatus.RUNNING + ) + .filter( + executedAction => + !alreadyPrintedActions.has(targetAsReadableString(executedAction.target)) + ) + .forEach(executedAction => { + printExecutedAction( + executedAction, + actionsByName.get(targetAsReadableString(executedAction.target)), + argv[dryRunOptionName] + ); + alreadyPrintedActions.add(targetAsReadableString(executedAction.target)); + }); + }; + + if (!isJsonOutput) { + runner.onChange(printExecutedGraph); + } + const runResult = await runner.result(); + if (lineageEmitter) { + await lineageEmitter.drain(LINEAGE_DRAIN_TIMEOUT_MS); + } + if (!isJsonOutput) { + printExecutedGraph(runResult); + } + if (isJsonOutput) { + print(prettyJsonStringify(runResult)); + } + if (!isJsonOutput) { + if (runResult.status === dataform.RunResult.ExecutionStatus.TIMED_OUT) { + const executionTimeoutMillis = argv[executionTimeoutOption.name]; + const suffix = executionTimeoutMillis + ? ` after ${executionTimeoutMillis / 1000} seconds (--execution-timeout)` + : ""; + printError(`Run timed out${suffix}.`); + } else if (runResult.status === dataform.RunResult.ExecutionStatus.CANCELLED) { + printError("Run cancelled."); + } + } + return runResult.status === dataform.RunResult.ExecutionStatus.SUCCESSFUL ? 0 : 1; + } +}; diff --git a/cli/index.ts b/cli/index.ts index 390136a6a..5ad2f6af2 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -1,13 +1,5 @@ -import * as fs from "fs"; -import parseDuration from "parse-duration"; -import * as path from "path"; import yargs from "yargs"; -import { build, compile, credentials, run, test } from "df/cli/api"; -import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; -import { LineageEmitter } from "df/cli/api/lineage/emitter"; -import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/api/lineage/emitter_factory"; -import { prettyJsonStringify } from "df/cli/api/utils"; import { compileCommand, formatCommand, @@ -15,161 +7,17 @@ import { initCommand, initCredsCommand, installCommand, + runCommand, testCommand } from "df/cli/commands"; -import { - actionsOption, - credentialsOption, - getCredentialsPath, - jsonOutputOption, - projectDirMustExistOption, - projectDirOption, - quietCompileOption, - requiresSelection, - splitCommas, - timeoutOption -} from "df/cli/common_options"; -import { - Logger, - print, - printCompiledGraphErrors, - printError, - printExecutedAction, - printExecutionGraph, - printSuccess, - printTestResult, - printWarning -} from "df/cli/console"; -import { ProjectConfigOptions } from "df/cli/project_config_options"; -import { - compiledGraphHasErrors, -} from "df/cli/util"; -import { createYargsCli, INamedOption } from "df/cli/yargswrapper"; -import { targetAsReadableString } from "df/core/targets"; -import { dataform } from "df/protos/ts"; - -// Maximum time to wait for outstanding lineage emissions to complete before -// `dataform run` returns. Lineage emission is fail-open — if we don't drain -// within this window, in-flight requests are abandoned and the run status is -// unaffected. -const LINEAGE_DRAIN_TIMEOUT_MS = 15_000; +import { printError } from "df/cli/console"; +import { createYargsCli } from "df/cli/yargswrapper"; process.on("unhandledRejection", async (reason: any) => { printError(`Unhandled promise rejection: ${reason?.stack || reason}`); }); // TODO: Since yargs launched an actually well typed API in version 12, let's use it as this file is currently not type checked. - - -const fullRefreshOption: INamedOption = { - name: "full-refresh", - option: { - describe: "Forces incremental tables to be rebuilt from scratch.", - type: "boolean", - default: false - } -}; - -const tagsOption: INamedOption = { - name: "tags", - option: { - describe: "A list of tags to filter the actions to run.", - type: "array", - coerce: splitCommas - } -}; - -const includeDepsOption: INamedOption = { - name: "include-deps", - option: { - describe: "If set, dependencies for selected actions will also be run.", - type: "boolean" - }, - check: requiresSelection("include-deps", actionsOption, tagsOption) -}; - -const includeDependentsOption: INamedOption = { - name: "include-dependents", - option: { - describe: "If set, dependents (downstream) for selected actions will also be run.", - type: "boolean" - }, - check: requiresSelection("include-dependents", actionsOption, tagsOption) -}; - -const emitLineageOption: INamedOption = { - name: "emit-lineage", - option: { - describe: - "If set, emit OpenLineage RunEvents to Knowledge Catalog Lineage for each executed action. " + - "Overrides workflow_settings.yaml lineage.enabled when specified.", - type: "boolean" - } -}; - -const executionTimeoutOption: INamedOption = { - name: "execution-timeout", - option: { - describe: - "Wall-clock deadline for the entire run (compile + all actions). When it fires, " + - "in-flight actions are cancelled and pending actions are skipped. Off by default. " + - "Examples: '10m', '2h'.", - type: "string", - default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null - } -}; - -const jitTimeoutOption: INamedOption = { - name: "jit-timeout", - option: { - describe: - "Per-model JiT compilation worker timeout. Each action with jitCode gets " + - "its own fresh deadline; independent of --execution-timeout. When unset, no " + - "per-model cap is applied and only --execution-timeout bounds JiT work. " + - "Examples: '30s', '2m'.", - type: "string", - default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null - } -}; - -const jobPrefixOption: INamedOption = { - name: "job-prefix", - option: { - describe: "Adds an additional prefix in the form of `dataform-${jobPrefix}-`.", - type: "string", - default: null - } -}; - -const bigqueryJobLabelsOption: INamedOption = { - name: "job-labels", - option: { - describe: - "Comma-separated list of labels to add to BigQuery jobs, e.g. 'key1=val1,key2=val2'.", - type: "string", - coerce: (raw: string | null) => { - const labels: { [key: string]: string } = {}; - raw?.split(",").forEach(kv => { - if (!kv) { - return; - } - const [key, ...rest] = kv.split("="); - labels[key] = rest.join("=") || ""; - }); - return labels; - } - } -}; - -const dryRunOptionName = "dry-run"; -const runTestsOptionName = "run-tests"; - -const actionRetryLimitName = "action-retry-limit"; - export function runCli() { const builtYargs = createYargsCli({ commands: [ @@ -179,217 +27,7 @@ export function runCli() { initCredsCommand, compileCommand, testCommand, - { - format: `run [${projectDirMustExistOption.name}]`, - description: "Run the dataform project.", - positionalOptions: [projectDirMustExistOption], - options: [ - { - name: dryRunOptionName, - option: { - describe: - "If set, BigQuery will validate the run SQL without applying changes to the warehouse.", - type: "boolean" - } - }, - { - name: runTestsOptionName, - option: { - describe: - "If set, the project's unit tests are required to pass before running the project.", - type: "boolean" - } - }, - { - name: actionRetryLimitName, - option: { - describe: "If set, idempotent actions will be retried up to the limit.", - type: "number", - default: 0 - } - }, - actionsOption, - credentialsOption, - emitLineageOption, - fullRefreshOption, - includeDepsOption, - includeDependentsOption, - jsonOutputOption, - timeoutOption, - executionTimeoutOption, - jitTimeoutOption, - tagsOption, - bigqueryJobLabelsOption, - ...ProjectConfigOptions.allYargsOptions - ], - processFn: async argv => { - const isJsonOutput = argv[jsonOutputOption.name]; - const logger = new Logger(!isJsonOutput); - - if (isJsonOutput && !argv[dryRunOptionName]) { - printError( - `For execution, the --${jsonOutputOption.name} option is only supported if the ` + - `--${dryRunOptionName} option is enabled` - ); - return; - } - if ( - !isJsonOutput && - argv[timeoutOption.name] != null && - argv[executionTimeoutOption.name] == null - ) { - printWarning( - "Note: --timeout only bounds project compilation. " + - "For a whole-run wall-clock deadline, use --execution-timeout.\n" - ); - } - logger.log("Compiling...\n"); - const compiledGraph = await compile({ - projectDir: argv[projectDirOption.name], - projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv), - timeoutMillis: argv[timeoutOption.name] || undefined - }); - if (compiledGraphHasErrors(compiledGraph)) { - printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]); - return 1; - } - logger.success("Compiled successfully.\n"); - const readCredentials = credentials.read( - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) - ); - - const dbadapter = new BigQueryDbAdapter(readCredentials); - const executionGraph = await build( - compiledGraph, - { - fullRefresh: argv[fullRefreshOption.name], - actions: argv[actionsOption.name], - includeDependencies: argv[includeDepsOption.name], - includeDependents: argv[includeDependentsOption.name], - tags: argv[tagsOption.name], - timeoutMillis: argv[executionTimeoutOption.name] || undefined, - jitTimeoutMillis: argv[jitTimeoutOption.name] || undefined - }, - dbadapter - ); - - if ( - argv[dryRunOptionName] && - isJsonOutput && - // Skip the early graph print when JiT actions are present: their compiled - // SQL is only produced once the Runner triggers JiT compilation, so falling - // through ensures the JSON dry-run output includes the generated SQL rather - // than the raw jitCode. - !executionGraph.actions.some(action => !!action.jitCode) - ) { - printExecutionGraph(executionGraph, isJsonOutput); - return; - } - - if (argv[runTestsOptionName]) { - logger.log(`Running ${compiledGraph.tests.length} unit tests...\n`); - const testResults = await test(dbadapter, compiledGraph.tests); - testResults.forEach(testResult => printTestResult(testResult)); - if (testResults.some(testResult => !testResult.successful)) { - printError("\nUnit tests did not pass; aborting run."); - return 1; - } - logger.success("Unit tests completed successfully.\n"); - } - - let bigqueryOptions: {} = { - actionRetryLimit: argv[actionRetryLimitName] - }; - if (argv[dryRunOptionName]) { - bigqueryOptions = { ...bigqueryOptions, dryRun: argv[dryRunOptionName] }; - } - if (argv[jobPrefixOption.name]) { - bigqueryOptions = { ...bigqueryOptions, jobPrefix: argv[jobPrefixOption.name] }; - } - if (argv[bigqueryJobLabelsOption.name]) { - bigqueryOptions = { ...bigqueryOptions, labels: argv[bigqueryJobLabelsOption.name] }; - } - - const actionsByName = new Map(); - executionGraph.actions.forEach(action => { - actionsByName.set(targetAsReadableString(action.target), action); - }); - - if (actionsByName.size === 0) { - logger.log("No actions to run.\n"); - return 0; - } - - if (argv[dryRunOptionName]) { - logger.log("Dry running (no changes to the warehouse will be applied)..."); - } else { - logger.log("Running...\n"); - } - - const lineageEmitter = createLineageEmitter(argv, executionGraph, readCredentials); - - const runner = run( - dbadapter, - executionGraph, - { - projectDir: argv[projectDirOption.name], - bigquery: bigqueryOptions, - lineageEmitter - } - ); - process.on("SIGINT", () => { - runner.cancel(); - }); - - const alreadyPrintedActions = new Set(); - - const printExecutedGraph = (executedGraph: dataform.IRunResult) => { - executedGraph.actions - .filter( - actionResult => - actionResult.status !== dataform.ActionResult.ExecutionStatus.RUNNING - ) - .filter( - executedAction => - !alreadyPrintedActions.has(targetAsReadableString(executedAction.target)) - ) - .forEach(executedAction => { - printExecutedAction( - executedAction, - actionsByName.get(targetAsReadableString(executedAction.target)), - argv[dryRunOptionName] - ); - alreadyPrintedActions.add(targetAsReadableString(executedAction.target)); - }); - }; - - if (!isJsonOutput) { - runner.onChange(printExecutedGraph); - } - const runResult = await runner.result(); - if (lineageEmitter) { - await lineageEmitter.drain(LINEAGE_DRAIN_TIMEOUT_MS); - } - if (!isJsonOutput) { - printExecutedGraph(runResult); - } - if (isJsonOutput) { - print(prettyJsonStringify(runResult)); - } - if (!isJsonOutput) { - if (runResult.status === dataform.RunResult.ExecutionStatus.TIMED_OUT) { - const executionTimeoutMillis = argv[executionTimeoutOption.name]; - const suffix = executionTimeoutMillis - ? ` after ${executionTimeoutMillis / 1000} seconds (--execution-timeout)` - : ""; - printError(`Run timed out${suffix}.`); - } else if (runResult.status === dataform.RunResult.ExecutionStatus.CANCELLED) { - printError("Run cancelled."); - } - } - return runResult.status === dataform.RunResult.ExecutionStatus.SUCCESSFUL ? 0 : 1; - } - }, + runCommand, formatCommand ] }) @@ -415,17 +53,3 @@ export function runCli() { yargs.showHelp(); } } - -function createLineageEmitter( - argv: yargs.Arguments, - executionGraph: dataform.IExecutionGraph, - readCredentials: dataform.IBigQuery | undefined -): LineageEmitter | undefined { - return createLineageEmitterFromFactory({ - cliEmitLineage: argv[emitLineageOption.name] as boolean | undefined, - workflowLineageEnabled: executionGraph.projectConfig?.lineageEnabled ?? undefined, - dryRun: !!argv[dryRunOptionName], - projectDir: argv[projectDirOption.name] || process.cwd(), - readCredentials - }); -} From 11571684be47e85f3713e457e0a368c9a9e2fe93 Mon Sep 17 00:00:00 2001 From: Marcin Biernacik Date: Tue, 8 Sep 2026 07:47:02 +0000 Subject: [PATCH 7/7] Address PR feedback: fix CLI options, null check and return codes --- cli/commands/run_command.ts | 16 +++++++--------- cli/commands/test_command.ts | 5 ++--- cli/common_options.ts | 15 +++++++-------- cli/project_config_options.ts | 3 ++- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/cli/commands/run_command.ts b/cli/commands/run_command.ts index 46110a56a..92d759b17 100644 --- a/cli/commands/run_command.ts +++ b/cli/commands/run_command.ts @@ -1,4 +1,3 @@ -import parseDuration from "parse-duration"; import yargs from "yargs"; import { build, compile, credentials, run, test } from "df/cli/api"; @@ -8,8 +7,8 @@ import { createLineageEmitter as createLineageEmitterFromFactory } from "df/cli/ import { prettyJsonStringify } from "df/cli/api/utils"; import { actionsOption, + coerceTimeout, credentialsOption, - getCredentialsPath, jsonOutputOption, projectDirMustExistOption, projectDirOption, @@ -29,7 +28,7 @@ import { printWarning } from "df/cli/console"; import { ProjectConfigOptions } from "df/cli/project_config_options"; -import { compiledGraphHasErrors } from "df/cli/util"; +import { actuallyResolve, compiledGraphHasErrors } from "df/cli/util"; import { ICommand, INamedOption } from "df/cli/yargswrapper"; import { targetAsReadableString } from "df/core/targets"; import { dataform } from "df/protos/ts"; @@ -95,8 +94,7 @@ const executionTimeoutOption: INamedOption = { "Examples: '10m', '2h'.", type: "string", default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null + coerce: coerceTimeout } }; @@ -110,8 +108,7 @@ const jitTimeoutOption: INamedOption = { "Examples: '30s', '2m'.", type: "string", default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null + coerce: coerceTimeout } }; @@ -202,6 +199,7 @@ export const runCommand: ICommand = { timeoutOption, executionTimeoutOption, jitTimeoutOption, + jobPrefixOption, tagsOption, bigqueryJobLabelsOption, ...ProjectConfigOptions.allYargsOptions @@ -215,7 +213,7 @@ export const runCommand: ICommand = { `For execution, the --${jsonOutputOption.name} option is only supported if the ` + `--${dryRunOptionName} option is enabled` ); - return; + return 1; } if ( !isJsonOutput && @@ -239,7 +237,7 @@ export const runCommand: ICommand = { } logger.success("Compiled successfully.\n"); const readCredentials = credentials.read( - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) + actuallyResolve(argv[projectDirOption.name], argv[credentialsOption.name]) ); const dbadapter = new BigQueryDbAdapter(readCredentials); diff --git a/cli/commands/test_command.ts b/cli/commands/test_command.ts index 3e041c38d..3ec541186 100644 --- a/cli/commands/test_command.ts +++ b/cli/commands/test_command.ts @@ -3,7 +3,6 @@ import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { prettyJsonStringify } from "df/cli/api/utils"; import { credentialsOption, - getCredentialsPath, jsonOutputOption, projectDirMustExistOption, projectDirOption, @@ -18,7 +17,7 @@ import { printTestResult } from "df/cli/console"; import { ProjectConfigOptions } from "df/cli/project_config_options"; -import { compiledGraphHasErrors } from "df/cli/util"; +import { actuallyResolve, compiledGraphHasErrors } from "df/cli/util"; import { ICommand } from "df/cli/yargswrapper"; export const testCommand: ICommand = { @@ -48,7 +47,7 @@ export const testCommand: ICommand = { printSuccess("Compiled successfully.\n"); } const readCredentials = credentials.read( - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) + actuallyResolve(argv[projectDirOption.name], argv[credentialsOption.name]) ); if (!compiledGraph.tests.length) { diff --git a/cli/common_options.ts b/cli/common_options.ts index 2bc7294b5..9121bbd13 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -37,7 +37,8 @@ export const projectDirMustExistOption: INamedOption = // Splits repeated and comma-separated values into a flat list, e.g. // `--actions a,b --actions c` -> ["a", "b", "c"]. -export const splitCommas = (raw: string[] | null) => raw.map(value => value.split(",")).flat(); +export const splitCommas = (raw: string[] | null) => + raw ? raw.map(value => value.split(",")).flat() : []; export const actionsOption: INamedOption = { name: "actions", @@ -48,10 +49,6 @@ export const actionsOption: INamedOption = { } }; -export function getCredentialsPath(projectDir: string, credentialsPath: string) { - return actuallyResolve(projectDir, credentialsPath); -} - export const credentialsOption: INamedOption = { name: "credentials", option: { @@ -59,7 +56,7 @@ export const credentialsOption: INamedOption = { default: CREDENTIALS_FILENAME }, check: (argv: yargs.Arguments) => - getCredentialsPath(argv[projectDirOption.name], argv[credentialsOption.name]) + actuallyResolve(argv[projectDirOption.name], argv[credentialsOption.name]) }; export const jsonOutputOption: INamedOption = { @@ -71,14 +68,16 @@ export const jsonOutputOption: INamedOption = { } }; +export const coerceTimeout = (rawTimeoutString: string | null) => + rawTimeoutString ? parseDuration(rawTimeoutString) : null; + export const timeoutOption: INamedOption = { name: "timeout", option: { describe: "Duration to allow project compilation to complete. Examples: '1s', '10m', etc.", type: "string", default: null, - coerce: (rawTimeoutString: string | null) => - rawTimeoutString ? parseDuration(rawTimeoutString) : null + coerce: coerceTimeout } }; diff --git a/cli/project_config_options.ts b/cli/project_config_options.ts index f8c07a430..a1ac101aa 100644 --- a/cli/project_config_options.ts +++ b/cli/project_config_options.ts @@ -42,7 +42,8 @@ export class ProjectConfigOptions { public static databaseSuffix: INamedOption = { name: "database-suffix", option: { - describe: "Default assertion schema. If unset, the value from workflow_settings.yaml is used." + describe: + "A suffix to be appended to output database names. If unset, the value from workflow_settings.yaml is used." } };