diff --git a/cli/api/BUILD b/cli/api/BUILD index 2fadbc8d1..02d9e2a83 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -41,6 +41,7 @@ ts_library( "@npm//fs-extra", "@npm//glob", "@npm//google-sql-syntax-ts", + "@npm//ignore", "@npm//js-beautify", "@npm//js-yaml", "@npm//promise-pool-executor", @@ -62,6 +63,7 @@ ts_test_suite( srcs = [ "tasks_test.ts", "utils_test.ts", + "commands/compile_copy_filter_test.ts", "commands/jit/rpc_test.ts", "dbadapters/bigquery_test.ts", "execution_sql_test.ts", diff --git a/cli/api/commands/compile.ts b/cli/api/commands/compile.ts index 38d32592a..67f84db7b 100644 --- a/cli/api/commands/compile.ts +++ b/cli/api/commands/compile.ts @@ -5,6 +5,7 @@ import * as tmp from "tmp"; import { promisify } from "util"; import { BaseWorker } from "df/cli/api/commands/base_worker"; +import { buildProjectCopyFilter } from "df/cli/api/commands/compile_copy_filter"; import { MISSING_CORE_VERSION_ERROR } from "df/cli/api/commands/install"; import { readConfigFromWorkflowSettings } from "df/cli/api/utils"; import { DEFAULT_COMPILATION_TIMEOUT_MILLIS } from "df/cli/api/utils/constants"; @@ -52,9 +53,12 @@ export async function compile( if (compileConfig.verbose) { print(`Using isolated environment for @dataform/core@${workflowSettingsDataformCoreVersion}\n`); print(`Copying project to temporary directory: ${temporaryProjectPath}\n`); + print(`Excluding .git, node_modules, and paths matched by the project's .gitignore\n`); } const copyStartTime = performance.now(); - fs.copySync(resolvedProjectPath, temporaryProjectPath); + fs.copySync(resolvedProjectPath, temporaryProjectPath, { + filter: buildProjectCopyFilter(resolvedProjectPath) + }); if (compileConfig.verbose) { print(`Project copy completed in ${performance.now() - copyStartTime}ms\n`); } diff --git a/cli/api/commands/compile_copy_filter.ts b/cli/api/commands/compile_copy_filter.ts new file mode 100644 index 000000000..a03099cef --- /dev/null +++ b/cli/api/commands/compile_copy_filter.ts @@ -0,0 +1,74 @@ +import * as fs from "fs-extra"; +import ignore from "ignore"; +import * as path from "path"; + +// Excluded whatever the project's .gitignore says. `.git` holds no Dataform project +// files. A top-level `node_modules` can't be present at all here -- `compile()` rejects +// the project before copying if it finds one -- so that entry covers nested ones, which +// are likewise never part of a Dataform project. +// +// Checked independently of the `ignore` instance below, rather than seeded into it, so +// that a project's .gitignore cannot override this floor: `ignore` lets later patterns +// override earlier ones by design, so a `!node_modules` negation would otherwise +// un-ignore it. +const ALWAYS_IGNORED_NAMES = new Set([".git", "node_modules"]); + +/** + * Builds a filter for fs-extra's `copySync`, so the stateless-install copy in `compile()` + * skips files that can't be part of the Dataform project -- most commonly a large + * `.venv`, build-output or cache directory sitting alongside `definitions/`, whose size + * the copy would otherwise pay for. + * + * Exclusions come from the project's own `.gitignore` rather than from a hardcoded list + * of directory names: no fixed list covers every ecosystem's junk directories (`.venv`, + * `target/`, `__pycache__/`, `vendor/`, `coverage/`, ...), whereas a project's + * `.gitignore` already states exactly what that project treats as disposable, and + * `dataform init` writes one. + * + * Only the project root's `.gitignore` is read. Nested `.gitignore` files, + * `.git/info/exclude` and the user's global excludes file are not consulted, so a + * project relying on those has more copied than `git status` would suggest. A project + * with no `.gitignore` at all gets only the ALWAYS_IGNORED_NAMES floor. + * + * Note that a gitignored file is never copied, so it is also never compiled: a project + * that generates definitions into a gitignored path needs that path unignored. + */ +export function buildProjectCopyFilter(resolvedProjectPath: string): (src: string) => boolean { + const ig = ignore(); + const gitignorePath = path.join(resolvedProjectPath, ".gitignore"); + if (fs.existsSync(gitignorePath)) { + ig.add(fs.readFileSync(gitignorePath, "utf8")); + } + + return (src: string) => { + const relative = path.relative(resolvedProjectPath, src); + // The project root itself (relative === ""), or something outside the project + // root (shouldn't happen in practice for a copySync(resolvedProjectPath, ...) + // call, but not this function's place to decide) is always copied/recursed into. + if ( + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + return true; + } + + const relativeSegments = relative.split(path.sep); + if (relativeSegments.some(segment => ALWAYS_IGNORED_NAMES.has(segment))) { + return false; + } + + // `ignore` needs to know whether a path is a directory to correctly match + // patterns like `.venv/` (trailing slash = directories only), and fs-extra's + // copySync filter callback isn't given that -- only `src`. Use lstatSync so + // dangling symlinks remain copyable, matching copySync's default behavior of + // copying links rather than dereferencing them. + let posixRelative = relativeSegments.join("/"); + if (fs.lstatSync(src).isDirectory()) { + posixRelative += "/"; + } + + return !ig.ignores(posixRelative); + }; +} diff --git a/cli/api/commands/compile_copy_filter_test.ts b/cli/api/commands/compile_copy_filter_test.ts new file mode 100644 index 000000000..98eb82e3a --- /dev/null +++ b/cli/api/commands/compile_copy_filter_test.ts @@ -0,0 +1,121 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; + +import { buildProjectCopyFilter } from "df/cli/api/commands/compile_copy_filter"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; + +suite("buildProjectCopyFilter", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + test("with no .gitignore, only .git and node_modules are excluded", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.ensureDirSync(path.join(projectDir, ".venv")); + fs.ensureDirSync(path.join(projectDir, ".git")); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + fs.ensureDirSync(path.join(projectDir, "definitions", "nested", "node_modules")); + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(projectDir)).to.equal(true); + expect(filter(path.join(projectDir, "definitions"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, ".venv"))).to.equal(true); + + expect(filter(path.join(projectDir, ".git"))).to.equal(false); + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + // Excluded at any depth, not just at the project root. + expect(filter(path.join(projectDir, "definitions", "nested", "node_modules"))).to.equal(false); + }); + + test("does not dereference symlinks while filtering", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const danglingSymlink = path.join(projectDir, "dangling-link"); + fs.symlinkSync(path.join(projectDir, "missing-target"), danglingSymlink); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(danglingSymlink)).to.equal(true); + }); + + test("applies ignore rules to in-project paths beginning with two dots", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const ignoredDir = path.join(projectDir, "..cache"); + fs.ensureDirSync(ignoredDir); + fs.writeFileSync(path.join(projectDir, ".gitignore"), "..cache/\n"); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(ignoredDir)).to.equal(false); + }); + + test("the always-ignored floor cannot be overridden by a negation pattern", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + // A project .gitignore is user-controlled and could (unusually, but validly) + // contain a negation pattern for something we always want to exclude. + fs.writeFileSync(path.join(projectDir, ".gitignore"), "!node_modules\n"); + + const filter = buildProjectCopyFilter(projectDir); + + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + }); + + test("filters an actual project copy", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const destinationDir = tmpDirFixture.createNewTmpDir(); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.ensureDirSync(path.join(projectDir, ".venv")); + fs.writeFileSync(path.join(projectDir, ".venv", "ignored"), "junk"); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + fs.writeFileSync(path.join(projectDir, "node_modules", "ignored"), "junk"); + fs.writeFileSync(path.join(projectDir, ".gitignore"), ".venv/\n"); + + fs.copySync(projectDir, destinationDir, { + filter: buildProjectCopyFilter(projectDir) + }); + + expect(fs.readFileSync(path.join(destinationDir, "definitions", "foo.sqlx"), "utf8")).to.equal( + "SELECT 1" + ); + expect(fs.existsSync(path.join(destinationDir, ".venv"))).to.equal(false); + expect(fs.existsSync(path.join(destinationDir, "node_modules"))).to.equal(false); + }); + + test("respects a project .gitignore, in addition to the always-ignored floor", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, ".gitignore"), + [".venv/", "__pycache__/", "*.pyc"].join("\n") + ); + fs.ensureDirSync(path.join(projectDir, ".venv", "lib")); + fs.writeFileSync(path.join(projectDir, ".venv", "lib", "mod.py"), "# stub"); + fs.ensureDirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions", "foo.sqlx"), "SELECT 1"); + fs.writeFileSync(path.join(projectDir, "foo.pyc"), "junk"); + fs.ensureDirSync(path.join(projectDir, ".git")); + fs.ensureDirSync(path.join(projectDir, "node_modules")); + + const filter = buildProjectCopyFilter(projectDir); + + // Dataform-relevant paths are still copied. + expect(filter(projectDir)).to.equal(true); + expect(filter(path.join(projectDir, "definitions"))).to.equal(true); + expect(filter(path.join(projectDir, "definitions", "foo.sqlx"))).to.equal(true); + expect(filter(path.join(projectDir, ".gitignore"))).to.equal(true); + + // gitignore'd paths are excluded -- including the bare directory itself (not + // just its contents), which requires correctly detecting it as a directory to + // match a trailing-slash-only pattern like `.venv/`. + expect(filter(path.join(projectDir, ".venv"))).to.equal(false); + expect(filter(path.join(projectDir, ".venv", "lib", "mod.py"))).to.equal(false); + expect(filter(path.join(projectDir, "foo.pyc"))).to.equal(false); + + // The always-ignored floor still applies even when a .gitignore is present. + expect(filter(path.join(projectDir, ".git"))).to.equal(false); + expect(filter(path.join(projectDir, "node_modules"))).to.equal(false); + }); +}); diff --git a/package.json b/package.json index de57a513d..53762526a 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "fs-extra": "^9.0.0", "glob": "13.0.6", "google-sql-syntax-ts": "^1.0.3", + "ignore": "^5.2.0", "js-beautify": "^1.10.2", "js-yaml": "^4.2.0", "jsdoc": "^3.6.11", diff --git a/packages/@dataform/cli/BUILD b/packages/@dataform/cli/BUILD index 31309c581..b625504f0 100644 --- a/packages/@dataform/cli/BUILD +++ b/packages/@dataform/cli/BUILD @@ -38,6 +38,7 @@ externals = [ "fs-extra", "glob", "google-sql-syntax-ts", + "ignore", "js-beautify", "js-yaml", "moo",