Filter the stateless-install project copy using .gitignore

cli/api/BUILD

@@ -41,6 +41,7 @@ ts_library(
         "@npm//fs-extra",
         "@npm//glob",
         "@npm//google-sql-syntax-ts",
+        "@npm//ignore",
         "@npm//js-beautify",
         "@npm//js-yaml",
         "@npm//promise-pool-executor",
@@ -62,6 +63,7 @@ ts_test_suite(
     srcs = [
         "tasks_test.ts",
         "utils_test.ts",
+        "commands/compile_copy_filter_test.ts",
         "commands/jit/rpc_test.ts",
         "dbadapters/bigquery_test.ts",
         "execution_sql_test.ts",

cli/api/commands/compile.ts

@@ -5,6 +5,7 @@ import * as tmp from "tmp";
 import { promisify } from "util";
 
 import { BaseWorker } from "df/cli/api/commands/base_worker";
+import { buildProjectCopyFilter } from "df/cli/api/commands/compile_copy_filter";
 import { MISSING_CORE_VERSION_ERROR } from "df/cli/api/commands/install";
 import { readConfigFromWorkflowSettings } from "df/cli/api/utils";
 import { DEFAULT_COMPILATION_TIMEOUT_MILLIS } from "df/cli/api/utils/constants";
@@ -52,9 +53,12 @@ export async function compile(
     if (compileConfig.verbose) {
       print(`Using isolated environment for @dataform/core@${workflowSettingsDataformCoreVersion}\n`);
       print(`Copying project to temporary directory: ${temporaryProjectPath}\n`);
+      print(`Excluding .git, node_modules, and paths matched by the project's .gitignore\n`);
     }
     const copyStartTime = performance.now();
-    fs.copySync(resolvedProjectPath, temporaryProjectPath);
+    fs.copySync(resolvedProjectPath, temporaryProjectPath, {
+      filter: buildProjectCopyFilter(resolvedProjectPath)
+    });
     if (compileConfig.verbose) {
       print(`Project copy completed in ${performance.now() - copyStartTime}ms\n`);
     }

cli/api/commands/compile_copy_filter.ts

@@ -0,0 +1,74 @@
+import * as fs from "fs-extra";
+import ignore from "ignore";
+import * as path from "path";
+
+// Excluded whatever the project's .gitignore says. `.git` holds no Dataform project
+// files. A top-level `node_modules` can't be present at all here -- `compile()` rejects
+// the project before copying if it finds one -- so that entry covers nested ones, which
+// are likewise never part of a Dataform project.
+//
+// Checked independently of the `ignore` instance below, rather than seeded into it, so
+// that a project's .gitignore cannot override this floor: `ignore` lets later patterns
+// override earlier ones by design, so a `!node_modules` negation would otherwise
+// un-ignore it.
+const ALWAYS_IGNORED_NAMES = new Set([".git", "node_modules"]);
+
+/**
+ * Builds a filter for fs-extra's `copySync`, so the stateless-install copy in `compile()`
+ * skips files that can't be part of the Dataform project -- most commonly a large
+ * `.venv`, build-output or cache directory sitting alongside `definitions/`, whose size
+ * the copy would otherwise pay for.
+ *
+ * Exclusions come from the project's own `.gitignore` rather than from a hardcoded list
+ * of directory names: no fixed list covers every ecosystem's junk directories (`.venv`,
+ * `target/`, `__pycache__/`, `vendor/`, `coverage/`, ...), whereas a project's
+ * `.gitignore` already states exactly what that project treats as disposable, and
+ * `dataform init` writes one.
+ *
+ * Only the project root's `.gitignore` is read. Nested `.gitignore` files,
+ * `.git/info/exclude` and the user's global excludes file are not consulted, so a
+ * project relying on those has more copied than `git status` would suggest. A project
+ * with no `.gitignore` at all gets only the ALWAYS_IGNORED_NAMES floor.
+ *
+ * Note that a gitignored file is never copied, so it is also never compiled: a project
+ * that generates definitions into a gitignored path needs that path unignored.
+ */
+export function buildProjectCopyFilter(resolvedProjectPath: string): (src: string) => boolean {
+  const ig = ignore();
+  const gitignorePath = path.join(resolvedProjectPath, ".gitignore");
+  if (fs.existsSync(gitignorePath)) {
+    ig.add(fs.readFileSync(gitignorePath, "utf8"));
+  }
+
+  return (src: string) => {
+    const relative = path.relative(resolvedProjectPath, src);
+    // The project root itself (relative === ""), or something outside the project
+    // root (shouldn't happen in practice for a copySync(resolvedProjectPath, ...)
+    // call, but not this function's place to decide) is always copied/recursed into.
+    if (
+      !relative ||
+      relative === ".." ||
+      relative.startsWith(`..${path.sep}`) ||
+      path.isAbsolute(relative)
+    ) {
+      return true;
+    }
+
+    const relativeSegments = relative.split(path.sep);
+    if (relativeSegments.some(segment => ALWAYS_IGNORED_NAMES.has(segment))) {
+      return false;
+    }
+
+    // `ignore` needs to know whether a path is a directory to correctly match
+    // patterns like `.venv/` (trailing slash = directories only), and fs-extra's
+    // copySync filter callback isn't given that -- only `src`. Use lstatSync so
+    // dangling symlinks remain copyable, matching copySync's default behavior of
+    // copying links rather than dereferencing them.
+    let posixRelative = relativeSegments.join("/");
+    if (fs.lstatSync(src).isDirectory()) {
+      posixRelative += "/";
+    }
+
+    return !ig.ignores(posixRelative);
+  };
+}

cli/api/commands/compile_copy_filter_test.ts

@@ -0,0 +1,121 @@
+import { expect } from "chai";
+import * as fs from "fs-extra";
+import * as path from "path";
+
+import { buildProjectCopyFilter } from "df/cli/api/commands/compile_copy_filter";
+import { suite, test } from "df/testing";
+import { TmpDirFixture } from "df/testing/fixtures";
+
+suite("buildProjectCopyFilter", ({ afterEach }) => {
+  const tmpDirFixture = new TmpDirFixture(afterEach);
+
+  test("with no .gitignore, only .git and node_modules are excluded", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    fs.ensureDirSync(path.join(projectDir, "definitions"));
+    fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1");
+    fs.ensureDirSync(path.join(projectDir, ".venv"));
+    fs.ensureDirSync(path.join(projectDir, ".git"));
+    fs.ensureDirSync(path.join(projectDir, "node_modules"));
+    fs.ensureDirSync(path.join(projectDir, "definitions", "nested", "node_modules"));
+    const filter = buildProjectCopyFilter(projectDir);
+
+    expect(filter(projectDir)).to.equal(true);
+    expect(filter(path.join(projectDir, "definitions"))).to.equal(true);
+    expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true);
+    expect(filter(path.join(projectDir, ".venv"))).to.equal(true);
+
+    expect(filter(path.join(projectDir, ".git"))).to.equal(false);
+    expect(filter(path.join(projectDir, "node_modules"))).to.equal(false);
+    // Excluded at any depth, not just at the project root.
+    expect(filter(path.join(projectDir, "definitions", "nested", "node_modules"))).to.equal(false);
+  });
+
+  test("does not dereference symlinks while filtering", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    const danglingSymlink = path.join(projectDir, "dangling-link");
+    fs.symlinkSync(path.join(projectDir, "missing-target"), danglingSymlink);
+
+    const filter = buildProjectCopyFilter(projectDir);
+
+    expect(filter(danglingSymlink)).to.equal(true);
+  });
+
+  test("applies ignore rules to in-project paths beginning with two dots", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    const ignoredDir = path.join(projectDir, "..cache");
+    fs.ensureDirSync(ignoredDir);
+    fs.writeFileSync(path.join(projectDir, ".gitignore"), "..cache/\n");
+
+    const filter = buildProjectCopyFilter(projectDir);
+
+    expect(filter(ignoredDir)).to.equal(false);
+  });
+
+  test("the always-ignored floor cannot be overridden by a negation pattern", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    fs.ensureDirSync(path.join(projectDir, "node_modules"));
+    // A project .gitignore is user-controlled and could (unusually, but validly)
+    // contain a negation pattern for something we always want to exclude.
+    fs.writeFileSync(path.join(projectDir, ".gitignore"), "!node_modules\n");
+
+    const filter = buildProjectCopyFilter(projectDir);
+
+    expect(filter(path.join(projectDir, "node_modules"))).to.equal(false);
+  });
+
+  test("filters an actual project copy", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    const destinationDir = tmpDirFixture.createNewTmpDir();
+    fs.ensureDirSync(path.join(projectDir, "definitions"));
+    fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1");
+    fs.ensureDirSync(path.join(projectDir, ".venv"));
+    fs.writeFileSync(path.join(projectDir, ".venv", "ignored"), "junk");
+    fs.ensureDirSync(path.join(projectDir, "node_modules"));
+    fs.writeFileSync(path.join(projectDir, "node_modules", "ignored"), "junk");
+    fs.writeFileSync(path.join(projectDir, ".gitignore"), ".venv/\n");
+
+    fs.copySync(projectDir, destinationDir, {
+      filter: buildProjectCopyFilter(projectDir)
+    });
+
+    expect(fs.readFileSync(path.join(destinationDir, "definitions", "foo.sqlx"), "utf8")).to.equal(
+      "SELECT 1"
+    );
+    expect(fs.existsSync(path.join(destinationDir, ".venv"))).to.equal(false);
+    expect(fs.existsSync(path.join(destinationDir, "node_modules"))).to.equal(false);
+  });
+
+  test("respects a project .gitignore, in addition to the always-ignored floor", () => {
+    const projectDir = tmpDirFixture.createNewTmpDir();
+    fs.writeFileSync(
+      path.join(projectDir, ".gitignore"),
+      [".venv/", "__pycache__/", "*.pyc"].join("\n")
+    );
+    fs.ensureDirSync(path.join(projectDir, ".venv", "lib"));
+    fs.writeFileSync(path.join(projectDir, ".venv", "lib", "mod.py"), "# stub");
+    fs.ensureDirSync(path.join(projectDir, "definitions"));
+    fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1");
+    fs.writeFileSync(path.join(projectDir, "foo.pyc"), "junk");
+    fs.ensureDirSync(path.join(projectDir, ".git"));
+    fs.ensureDirSync(path.join(projectDir, "node_modules"));
+
+    const filter = buildProjectCopyFilter(projectDir);
+
+    // Dataform-relevant paths are still copied.
+    expect(filter(projectDir)).to.equal(true);
+    expect(filter(path.join(projectDir, "definitions"))).to.equal(true);
+    expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true);
+    expect(filter(path.join(projectDir, ".gitignore"))).to.equal(true);
+
+    // gitignore'd paths are excluded -- including the bare directory itself (not
+    // just its contents), which requires correctly detecting it as a directory to
+    // match a trailing-slash-only pattern like `.venv/`.
+    expect(filter(path.join(projectDir, ".venv"))).to.equal(false);
+    expect(filter(path.join(projectDir, ".venv", "lib", "mod.py"))).to.equal(false);
+    expect(filter(path.join(projectDir, "foo.pyc"))).to.equal(false);
+
+    // The always-ignored floor still applies even when a .gitignore is present.
+    expect(filter(path.join(projectDir, ".git"))).to.equal(false);
+    expect(filter(path.join(projectDir, "node_modules"))).to.equal(false);
+  });
+});

package.json

@@ -40,6 +40,7 @@
     "fs-extra": "^9.0.0",
     "glob": "13.0.6",
     "google-sql-syntax-ts": "^1.0.3",
+    "ignore": "^5.2.0",
     "js-beautify": "^1.10.2",
     "js-yaml": "^4.2.0",
     "jsdoc": "^3.6.11",

packages/@dataform/cli/BUILD

@@ -38,6 +38,7 @@ externals = [
     "fs-extra",
     "glob",
     "google-sql-syntax-ts",
+    "ignore",
     "js-beautify",
     "js-yaml",
     "moo",