refactor(cli): decompose monolithic cli/index.ts into modular command modules

cli/BUILD

@@ -9,9 +9,20 @@ package(default_visibility = ["//visibility:public"])
 ts_library(
     name = "cli",
     srcs = [
+        "commands/compile_command.ts",
+        "commands/format_command.ts",
+        "commands/help_command.ts",
+        "commands/index.ts",
+        "commands/init_command.ts",
+        "commands/init_creds_command.ts",
+        "commands/install_command.ts",
+        "commands/run_command.ts",
+        "commands/test_command.ts",
+        "common_options.ts",
         "console.ts",
         "credentials.ts",
         "index.ts",
+        "project_config_options.ts",
         "util.ts",
         "yargswrapper.ts",
     ],

cli/commands/compile_command.ts

@@ -0,0 +1,230 @@
+import * as chokidar from "chokidar";
+import yargs from "yargs";
+
+import { compile, prune } from "df/cli/api";
+import {
+  jsonOutputOption,
+  projectDirMustExistOption,
+  quietCompileOption,
+  requiresSelection,
+  splitCommas,
+  timeoutOption
+} from "df/cli/common_options";
+import {
+  compiledGraphOutputType,
+  Logger,
+  print,
+  printCompiledGraph,
+  printCompiledGraphErrors,
+  printError
+} from "df/cli/console";
+import { ProjectConfigOptions } from "df/cli/project_config_options";
+import { compiledGraphHasErrors } from "df/cli/util";
+import { ICommand, INamedOption } from "df/cli/yargswrapper";
+
+const RECOMPILE_DELAY = 500;
+
+// `compile` reuses the same prune() filtering as run/build, but these flags only
+// filter the *printed output* -- the whole project still compiles. The `output-`
+// prefix makes that distinction explicit.
+const outputActionsOption: INamedOption<yargs.Options> = {
+  name: "output-actions",
+  option: {
+    // No wildcard support: prune()'s matchPatterns() does exact matching on the
+    // action name or its fully-qualified `database.schema.name`.
+    describe: "A list of action names to filter the compiled output to.",
+    type: "array",
+    coerce: splitCommas
+  }
+};
+
+const outputTagsOption: INamedOption<yargs.Options> = {
+  name: "output-tags",
+  option: {
+    describe: "A list of tags to filter the compiled output to.",
+    type: "array",
+    coerce: splitCommas
+  }
+};
+
+const outputIncludeDepsOption: INamedOption<yargs.Options> = {
+  name: "output-include-deps",
+  option: {
+    describe: "If set, dependencies of the selected actions are also included in the output.",
+    type: "boolean"
+  },
+  check: requiresSelection("output-include-deps", outputActionsOption, outputTagsOption)
+};
+
+const outputIncludeDependentsOption: INamedOption<yargs.Options> = {
+  name: "output-include-dependents",
+  option: {
+    describe:
+      "If set, dependents (downstream) of the selected actions are also included in the output.",
+    type: "boolean"
+  },
+  check: requiresSelection("output-include-dependents", outputActionsOption, outputTagsOption)
+};
+
+const dotOutputOption: INamedOption<yargs.Options> = {
+  name: "dot",
+  option: {
+    describe: "Outputs a dot representation of the compiled project.",
+    type: "boolean",
+    default: false
+  },
+  check: (argv: yargs.Arguments<any>) => {
+    if (argv.json && argv.dot) {
+      throw new Error("Arguments --json and --dot are mutually exclusive.");
+    }
+  }
+};
+
+const watchOptionName = "watch";
+const verboseOptionName = "verbose";
+
+export const compileCommand: ICommand = {
+  format: `compile [${projectDirMustExistOption.name}]`,
+  description:
+    "Compile the dataform project. Produces JSON output describing the non-executable graph.",
+  positionalOptions: [projectDirMustExistOption],
+  options: [
+    {
+      name: watchOptionName,
+      option: {
+        describe: "Whether to watch the changes in the project directory.",
+        type: "boolean",
+        default: false
+      }
+    },
+    jsonOutputOption,
+    dotOutputOption,
+    timeoutOption,
+    quietCompileOption,
+    outputActionsOption,
+    outputTagsOption,
+    outputIncludeDepsOption,
+    outputIncludeDependentsOption,
+    {
+      name: verboseOptionName,
+      option: {
+        describe: "Enable verbose compilation output. Example usage: 'dataform compile --verbose'",
+        type: "boolean",
+        default: false
+      },
+      check: (argv: yargs.Arguments) => {
+        if (argv.quiet && argv.verbose) {
+          throw new Error("Arguments --verbose and --quiet are mutually exclusive.");
+        }
+      }
+    },
+    ...ProjectConfigOptions.allYargsOptions
+  ],
+  processFn: async argv => {
+    const projectDir = argv[projectDirMustExistOption.name];
+    const logger = new Logger(!argv[jsonOutputOption.name]);
+
+    async function compileAndPrint() {
+      let outputType = compiledGraphOutputType.Summary;
+      if (argv[jsonOutputOption.name]) {
+        outputType = compiledGraphOutputType.Json;
+      } else if (argv[dotOutputOption.name]) {
+        outputType = compiledGraphOutputType.Dot;
+      }
+
+      if (outputType === compiledGraphOutputType.Summary) {
+        logger.log("Compiling...\n");
+      }
+      const compiledGraph = await compile({
+        projectDir,
+        projectConfigOverride: ProjectConfigOptions.constructProjectConfigOverride(argv),
+        timeoutMillis: argv[timeoutOption.name] || undefined,
+        verbose: argv[verboseOptionName] || false
+      });
+
+      // The whole project must compile (ref() resolution needs every action
+      // registered), but the printed output can be filtered to the selected
+      // action(s) -- mirroring how `run`/`build` prune the graph. We only prune
+      // a clean graph; if compilation produced errors we print the full graph
+      // plus the errors, keeping graph-level errors as-is.
+      const hasSelector =
+        argv[outputActionsOption.name]?.length > 0 || argv[outputTagsOption.name]?.length > 0;
+      const outputGraph =
+        hasSelector && !compiledGraphHasErrors(compiledGraph)
+          ? prune(compiledGraph, {
+              actions: argv[outputActionsOption.name],
+              tags: argv[outputTagsOption.name],
+              includeDependencies: argv[outputIncludeDepsOption.name],
+              includeDependents: argv[outputIncludeDependentsOption.name]
+            })
+          : compiledGraph;
+      printCompiledGraph(outputGraph, outputType, argv[quietCompileOption.name]);
+      if (compiledGraphHasErrors(compiledGraph)) {
+        print("");
+        printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);
+        return true;
+      }
+      return false;
+    }
+
+    const graphHasErrors = await compileAndPrint();
+
+    if (!argv[watchOptionName]) {
+      return graphHasErrors ? 1 : 0;
+    }
+
+    let watching = true;
+
+    let timeoutID: NodeJS.Timer = null;
+    let isCompiling = false;
+
+    // Initialize watcher.
+    const watcher = chokidar.watch(projectDir, {
+      ignored: /node_modules/,
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 1000,
+        pollInterval: 200
+      }
+    });
+
+    const printReady = () => {
+      print("\nWatching for changes...\n");
+    };
+    // Add event listeners.
+    watcher
+      .on("ready", printReady)
+      .on("error", error => {
+        // This error is caught not if there is a compilation error, but
+        // if the watcher fails; this indicates an failure on our side.
+        printError(`Error: ${error}`);
+        process.exit(1);
+      })
+      .on("all", () => {
+        if (timeoutID || isCompiling) {
+          // don't recompile many times if we changed a lot of files
+          clearTimeout(timeoutID);
+        }
+
+        timeoutID = setTimeout(async () => {
+          clearTimeout(timeoutID);
+
+          if (!isCompiling) {
+            isCompiling = true;
+            await compileAndPrint();
+            printReady();
+            isCompiling = false;
+          }
+        }, RECOMPILE_DELAY);
+      });
+    process.on("SIGINT", async () => {
+      await watcher.close();
+      watching = false;
+      process.exit(1);
+    });
+    while (watching) {
+      await new Promise((resolve, reject) => setTimeout(() => resolve(), 100));
+    }
+  }
+};

cli/commands/format_command.ts

@@ -0,0 +1,106 @@
+import * as fs from "fs";
+import * as glob from "glob";
+import * as path from "path";
+import yargs from "yargs";
+
+import { actionsOption, projectDirMustExistOption } from "df/cli/common_options";
+import { printError, printFormatFilesResult, printSuccess } from "df/cli/console";
+import { ICommand, INamedOption } from "df/cli/yargswrapper";
+import { formatFile } from "df/sqlx/format";
+
+const fmtIgnoreJsOption: INamedOption<yargs.Options> = {
+  name: "ignore-js-files",
+  option: {
+    describe: "If set, the formatter will not consider javascript files (.js)",
+    type: "boolean",
+    default: false
+  }
+};
+
+const checkOptionName = "check";
+
+const checkOption: INamedOption<yargs.Options> = {
+  name: checkOptionName,
+  option: {
+    describe: "Check if files are formatted correctly without modifying them.",
+    type: "boolean",
+    default: false
+  }
+};
+
+export const formatCommand: ICommand = {
+  format: `format [${projectDirMustExistOption.name}]`,
+  description: "Format the dataform project's files.",
+  positionalOptions: [projectDirMustExistOption],
+  options: [actionsOption, fmtIgnoreJsOption, checkOption],
+  processFn: async argv => {
+    const extensions = argv[fmtIgnoreJsOption.name] ? "*.sqlx" : "*.{js,sqlx}";
+    let actions = [`{definitions,includes}/**/${extensions}`];
+    if (actionsOption.name in argv && argv[actionsOption.name].length > 0) {
+      actions = argv[actionsOption.name];
+    }
+    const filenames = actions
+      .map((action: string) => glob.sync(action, { cwd: argv[projectDirMustExistOption.name] }))
+      .flat();
+
+    const isCheckMode = argv[checkOptionName];
+    const results: Array<{
+      filename: string;
+      err?: Error;
+      needsFormatting?: boolean;
+    }> = await Promise.all(
+      filenames.map(async (filename: string) => {
+        try {
+          const filePath = path.resolve(argv[projectDirMustExistOption.name], filename);
+          if (isCheckMode) {
+            // In check mode, we don't modify files, just check if they need formatting
+            const fileContent = fs.readFileSync(filePath).toString();
+            const formattedContent = await formatFile(filePath, {
+              overwriteFile: false
+            });
+            return {
+              filename,
+              needsFormatting: fileContent !== formattedContent
+            };
+          } else {
+            // Normal formatting mode
+            await formatFile(filePath, {
+              overwriteFile: true
+            });
+            return {
+              filename
+            };
+          }
+        } catch (e) {
+          return {
+            filename,
+            err: e
+          };
+        }
+      })
+    );
+
+    printFormatFilesResult(results);
+
+    // Return error code if there are any formatting errors
+    const failedFormatResults = results.filter(result => !!result.err);
+    if (failedFormatResults.length > 0) {
+      printError(`${failedFormatResults.length} file(s) failed to format.`);
+      return 1;
+    }
+
+    // In check mode, return an error code if any files need formatting
+    if (isCheckMode) {
+      const filesNeedingFormatting = results.filter(result => result.needsFormatting);
+      if (filesNeedingFormatting.length > 0) {
+        printError(
+          `${filesNeedingFormatting.length} file(s) would be reformatted. Run the format command without --check to update.`
+        );
+        return 1;
+      }
+      printSuccess("All files are formatted correctly!");
+    }
+
+    return 0;
+  }
+};

cli/commands/help_command.ts

@@ -0,0 +1,13 @@
+import { ICommand } from "df/cli/yargswrapper";
+
+// This dummy command is a hack with the only goal of displaying "help" as a command in the CLI
+// and we need it because of the limitations of yargs considering "help" as an option and not as a command.
+export const helpCommand: ICommand = {
+  format: "help [command]",
+  description: "Show help. If [command] is specified, the help is for the given command.",
+  positionalOptions: [],
+  options: [],
+  processFn: async () => {
+    return 0;
+  }
+};

cli/commands/index.ts

@@ -0,0 +1,8 @@
+export { compileCommand } from "df/cli/commands/compile_command";
+export { formatCommand } from "df/cli/commands/format_command";
+export { helpCommand } from "df/cli/commands/help_command";
+export { initCommand } from "df/cli/commands/init_command";
+export { initCredsCommand } from "df/cli/commands/init_creds_command";
+export { installCommand } from "df/cli/commands/install_command";
+export { runCommand } from "df/cli/commands/run_command";
+export { testCommand } from "df/cli/commands/test_command";