diff --git a/cli/BUILD b/cli/BUILD index bb14dcc6f..7eb8e3a51 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -57,6 +57,7 @@ ts_library( name = "index_test_base", srcs = ["index_test_base.ts"], deps = [ + "//cli", "//core", "//protos:ts", "//testing", diff --git a/cli/index_compile_test.ts b/cli/index_compile_test.ts index c27ac47eb..db9b30a54 100644 --- a/cli/index_compile_test.ts +++ b/cli/index_compile_test.ts @@ -4,7 +4,7 @@ import * as fs from "fs-extra"; import { dump as dumpYaml, load as loadYaml } from "js-yaml"; import * as path from "path"; -import { cliEntryPointPath, DEFAULT_DATABASE, DEFAULT_LOCATION } from "df/cli/index_test_base"; +import { cliEntryPointPath, INTEGRATION_TEST_LOCATION, INTEGRATION_TEST_PROJECT } from "df/cli/index_test_base"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; import { corePackageTarPath, getProcessResult, nodePath, npmPath, suite, test } from "df/testing"; @@ -49,7 +49,7 @@ suite("compile command", ({ afterEach }) => { "defaultDatabase": "tada-analytics", "defaultSchema": "df_integration_test", "assertionSchema": "df_integration_test_assertions", - "defaultLocation": "${DEFAULT_LOCATION}" + "defaultLocation": "${INTEGRATION_TEST_LOCATION}" } ` ); @@ -105,8 +105,8 @@ suite("compile command", ({ afterEach }) => { fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), dumpYaml({ - defaultProject: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, + defaultProject: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, defaultDataset: "dataform", dataformCoreVersion: "3.0.50" }) @@ -166,7 +166,7 @@ suite("disable-assertions flag (compilation)", ({ afterEach, beforeEach }) => { const packageJsonPath = path.join(projectDir, "package.json"); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); @@ -239,13 +239,13 @@ SELECT 1 as id assertions: [ { canonicalTarget: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "dataform_example_table_assertions_uniqueKey_0", schema: "dataform_assertions" }, dependencyTargets: [ { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" } @@ -253,22 +253,22 @@ SELECT 1 as id disabled: true, fileName: "definitions/example_table.sqlx", parentAction: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" }, query: // tslint:disable-next-line:tsr-detect-sql-literal-injection - `\nSELECT\n *\nFROM (\n SELECT\n id,\n COUNT(1) AS index_row_count\n FROM \`${DEFAULT_DATABASE}.dataform.example_table\`\n GROUP BY id\n ) AS data\nWHERE index_row_count > 1\n`, + `\nSELECT\n *\nFROM (\n SELECT\n id,\n COUNT(1) AS index_row_count\n FROM \`${INTEGRATION_TEST_PROJECT}.dataform.example_table\`\n GROUP BY id\n ) AS data\nWHERE index_row_count > 1\n`, target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "dataform_example_table_assertions_uniqueKey_0", schema: "dataform_assertions" } }, { canonicalTarget: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "test_assertion", schema: "dataform_assertions" }, @@ -276,7 +276,7 @@ SELECT 1 as id fileName: "definitions/test_assertion.sqlx", query: "\n\nSELECT 1 WHERE FALSE\n", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "test_assertion", schema: "dataform_assertions" } @@ -287,8 +287,8 @@ SELECT 1 as id jitData: {}, projectConfig: { assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, defaultSchema: "dataform", disableAssertions: true, warehouse: "bigquery" @@ -296,7 +296,7 @@ SELECT 1 as id tables: [ { canonicalTarget: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" }, @@ -306,7 +306,7 @@ SELECT 1 as id hermeticity: "NON_HERMETIC", query: "\n\nSELECT 1 as id\n", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" }, @@ -315,17 +315,17 @@ SELECT 1 as id ], targets: [ { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "dataform_example_table_assertions_uniqueKey_0", schema: "dataform_assertions" }, { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" }, { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "test_assertion", schema: "dataform_assertions" } @@ -370,7 +370,7 @@ suite("compile node selection", ({ afterEach }) => { const npmCacheDir = tmpDirFixture.createNewTmpDir(); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); @@ -508,8 +508,8 @@ suite("extension config", ({ afterEach }) => { fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), dumpYaml({ - defaultProject: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, + defaultProject: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, defaultDataset: "dataform", defaultAssertionDataset: "dataform_assertions", extension: { diff --git a/cli/index_project_test.ts b/cli/index_project_test.ts index d7b38f4e4..50fc71098 100644 --- a/cli/index_project_test.ts +++ b/cli/index_project_test.ts @@ -4,7 +4,7 @@ import * as fs from "fs-extra"; import { dump as dumpYaml, load as loadYaml } from "js-yaml"; import * as path from "path"; -import { cliEntryPointPath, DEFAULT_DATABASE, DEFAULT_LOCATION } from "df/cli/index_test_base"; +import { cliEntryPointPath, INTEGRATION_TEST_LOCATION, INTEGRATION_TEST_PROJECT } from "df/cli/index_test_base"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; import { corePackageTarPath, getProcessResult, nodePath, npmPath, suite, test } from "df/testing"; @@ -46,7 +46,7 @@ suite("project ops", ({ afterEach }) => { // Initialize a project using the CLI, don't install packages. await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Install packages manually to get around bazel read-only sandbox issues. @@ -134,7 +134,7 @@ SELECT 1 as test // Initialize a project using the CLI, don't install packages. await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Install packages manually to get around bazel read-only sandbox issues. diff --git a/cli/index_run_e2e_test.ts b/cli/index_run_e2e_test.ts index 226e5833b..452796d6b 100644 --- a/cli/index_run_e2e_test.ts +++ b/cli/index_run_e2e_test.ts @@ -7,9 +7,9 @@ import * as path from "path"; import { cliEntryPointPath, CREDENTIALS_PATH, - DEFAULT_DATABASE, - DEFAULT_LOCATION, - DEFAULT_RESERVATION + INTEGRATION_TEST_LOCATION, + INTEGRATION_TEST_PROJECT, + INTEGRATION_TEST_RESERVATION } from "df/cli/index_test_base"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -27,7 +27,7 @@ suite("run e2e", ({ afterEach }) => { // Initialize a project using the CLI, don't install packages. await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Install packages manually to get around bazel read-only sandbox issues. @@ -86,14 +86,14 @@ select 1 as \${dataform.projectConfig.vars.testVar2} type: "table", enumType: "TABLE", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, schema: "dataform_test_schema_suffix", name: "example" }, canonicalTarget: { schema: "dataform", name: "example", - database: DEFAULT_DATABASE + database: INTEGRATION_TEST_PROJECT }, query: "\n\nselect 1 as testValue2\n", disabled: false, @@ -106,8 +106,8 @@ select 1 as \${dataform.projectConfig.vars.testVar2} warehouse: "bigquery", defaultSchema: "dataform", assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, vars: { testVar1: "testValue1", testVar2: "testValue2" @@ -119,7 +119,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} dataformCoreVersion: version, targets: [ { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, schema: "dataform", name: "example" } @@ -156,7 +156,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} hermeticity: "NON_HERMETIC", tableType: "table", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example", schema: "dataform" }, @@ -164,7 +164,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} { statement: // tslint:disable-next-line:tsr-detect-sql-literal-injection - `create or replace table \`${DEFAULT_DATABASE}.dataform.example\` as \n\nselect 1 as testValue2`, + `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example\` as \n\nselect 1 as testValue2`, type: "statement" } ], @@ -174,7 +174,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} jitData: {}, projectConfig: { assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, + defaultDatabase: INTEGRATION_TEST_PROJECT, defaultLocation: "europe", defaultSchema: "dataform", warehouse: "bigquery", @@ -200,7 +200,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2} const packageJsonPath = path.join(projectDir, "package.json"); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); @@ -274,7 +274,7 @@ SELECT 1 as id hermeticity: "NON_HERMETIC", tableType: "table", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_table", schema: "dataform" }, @@ -282,7 +282,7 @@ SELECT 1 as id { statement: // tslint:disable-next-line:tsr-detect-sql-literal-injection - `create or replace table \`${DEFAULT_DATABASE}.dataform.example_table\` as \n\nSELECT 1 as id`, + `create or replace table \`${INTEGRATION_TEST_PROJECT}.dataform.example_table\` as \n\nSELECT 1 as id`, type: "statement" } ], @@ -292,7 +292,7 @@ SELECT 1 as id fileName: "definitions/test_assertion.sqlx", hermeticity: "HERMETIC", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "test_assertion", schema: "dataform_assertions" }, @@ -302,8 +302,8 @@ SELECT 1 as id jitData: {}, projectConfig: { assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, defaultSchema: "dataform", disableAssertions: true, warehouse: "bigquery" @@ -401,7 +401,7 @@ SELECT 1 as id const packageJsonPath = path.join(projectDir, "package.json"); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Remove dataformCoreVersion so we can use the local package. @@ -448,7 +448,7 @@ SELECT 1 as id "compile", projectDir, "--json", - `--default-reservation=${DEFAULT_RESERVATION}` + `--default-reservation=${INTEGRATION_TEST_RESERVATION}` ]) ); @@ -458,9 +458,9 @@ SELECT 1 as id warehouse: "bigquery", defaultSchema: "dataform", assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, - defaultReservation: DEFAULT_RESERVATION + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, + defaultReservation: INTEGRATION_TEST_RESERVATION }); }); @@ -474,7 +474,7 @@ SELECT 1 as id CREDENTIALS_PATH, "--dry-run", "--json", - `--default-reservation=${DEFAULT_RESERVATION}`, + `--default-reservation=${INTEGRATION_TEST_RESERVATION}`, "--actions=example_table" ]) ); @@ -485,9 +485,9 @@ SELECT 1 as id warehouse: "bigquery", defaultSchema: "dataform", assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION, - defaultReservation: DEFAULT_RESERVATION + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION, + defaultReservation: INTEGRATION_TEST_RESERVATION }); }); }); @@ -500,7 +500,7 @@ SELECT 1 as id // Initialize a project using the CLI, don't install packages. await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Install packages manually to get around bazel read-only sandbox issues. @@ -577,7 +577,7 @@ select 1 // Initialize a project using the CLI, don't install packages. await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); // Install packages manually to get around bazel read-only sandbox issues. @@ -659,7 +659,7 @@ select 2 const packageJsonPath = path.join(projectDir, "package.json"); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); const workflowSettings = dataform.WorkflowSettings.create( @@ -762,8 +762,8 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p warehouse: "bigquery", defaultSchema: uniqueDataset, assertionSchema: "dataform_assertions", - defaultDatabase: DEFAULT_DATABASE, - defaultLocation: DEFAULT_LOCATION + defaultDatabase: INTEGRATION_TEST_PROJECT, + defaultLocation: INTEGRATION_TEST_LOCATION }, runConfig: { actions: ["example_incremental"], @@ -775,7 +775,7 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p hermeticity: "NON_HERMETIC", tableType: "incremental", target: { - database: DEFAULT_DATABASE, + database: INTEGRATION_TEST_PROJECT, name: "example_incremental", schema: uniqueDataset }, @@ -820,7 +820,7 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), - `defaultProject: ${DEFAULT_DATABASE}\ndefaultLocation: ${DEFAULT_LOCATION}\n` + `defaultProject: ${INTEGRATION_TEST_PROJECT}\ndefaultLocation: ${INTEGRATION_TEST_LOCATION}\n` ); const runResult = await getProcessResult( @@ -844,7 +844,7 @@ DROP SCHEMA IF EXISTS \`\${dataform.projectConfig.defaultDatabase}.\${dataform.p const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), - `defaultProject: ${DEFAULT_DATABASE}\ndefaultLocation: ${DEFAULT_LOCATION}\n` + `defaultProject: ${INTEGRATION_TEST_PROJECT}\ndefaultLocation: ${INTEGRATION_TEST_LOCATION}\n` ); const runResult = await getProcessResult( diff --git a/cli/index_test_base.ts b/cli/index_test_base.ts index 5c150404b..861e36bee 100644 --- a/cli/index_test_base.ts +++ b/cli/index_test_base.ts @@ -4,14 +4,14 @@ import * as fs from "fs-extra"; import { dump as dumpYaml, load as loadYaml } from "js-yaml"; import * as path from "path"; +import { Logger } from "df/cli/console"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; import { corePackageTarPath, getProcessResult, nodePath, npmPath } from "df/testing"; import { TmpDirFixture } from "df/testing/fixtures"; -export const DEFAULT_DATABASE = "dataform-open-source"; -export const DEFAULT_LOCATION = "US"; -export const DEFAULT_RESERVATION = "projects/dataform-open-source/locations/us/reservations/dataform-test"; +const DEFAULT_PROJECT = "dataform-open-source"; +const DEFAULT_LOCATION = "US"; const runfilesDir = process.env.RUNFILES; let workspaceName = "df"; @@ -21,6 +21,42 @@ if (!fs.existsSync(path.resolve(runfilesDir, "df"))) { export const CREDENTIALS_PATH = path.resolve(runfilesDir, workspaceName, "test_credentials/bigquery.json"); +const logger = new Logger(true); + +function getCredentialsProjectId(): string { + try { + if (fs.existsSync(CREDENTIALS_PATH)) { + const parsed = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, "utf8")); + if (parsed?.projectId) { + return parsed.projectId; + } + } + } catch (e) { + // Fall back to default + } + logger.log(`Project name not specified; defaulting to ${DEFAULT_PROJECT}`); + return DEFAULT_PROJECT; +} + +function getCredentialsLocation(): string { + try { + if (fs.existsSync(CREDENTIALS_PATH)) { + const parsed = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, "utf8")); + if (parsed?.location) { + return parsed.location; + } + } + } catch (e) { + // Fall back to default + } + logger.log(`Location not specified; defaulting to ${DEFAULT_LOCATION}`); + return DEFAULT_LOCATION; +} + +export const INTEGRATION_TEST_PROJECT = getCredentialsProjectId(); +export const INTEGRATION_TEST_LOCATION = getCredentialsLocation(); +export const INTEGRATION_TEST_RESERVATION = `projects/${INTEGRATION_TEST_PROJECT}/locations/${INTEGRATION_TEST_LOCATION.toLowerCase()}/reservations/dataform-test`; + export const cliEntryPointPath = "cli/node_modules/@dataform/cli/bundle.js"; export async function setupJitProject( @@ -31,7 +67,7 @@ export async function setupJitProject( const packageJsonPath = path.join(projectDir, "package.json"); await getProcessResult( - execFile(nodePath, [cliEntryPointPath, "init", projectDir, DEFAULT_DATABASE, DEFAULT_LOCATION]) + execFile(nodePath, [cliEntryPointPath, "init", projectDir, INTEGRATION_TEST_PROJECT, INTEGRATION_TEST_LOCATION]) ); const workflowSettingsPath = path.join(projectDir, "workflow_settings.yaml"); diff --git a/cli/tests/jit/index_jit_dependency_test.ts b/cli/tests/jit/index_jit_dependency_test.ts index b3f1393d3..3d50c65d9 100644 --- a/cli/tests/jit/index_jit_dependency_test.ts +++ b/cli/tests/jit/index_jit_dependency_test.ts @@ -6,6 +6,7 @@ import * as path from "path"; import { cliEntryPointPath, CREDENTIALS_PATH, + INTEGRATION_TEST_PROJECT, setupJitProject } from "df/cli/index_test_base"; import { getProcessResult, nodePath, suite, test } from "df/testing"; @@ -51,7 +52,7 @@ suite("JiT support dependencies", ({ afterEach }) => { expect(executedGraph.actions.some((a: any) => a.target.name === "table_a")).to.equal(true); const actionB = executedGraph.actions.find((a: any) => a.target.name === "table_b"); expect(actionB).to.not.equal(undefined); - expect(actionB.tasks[0].compiledSql).to.include("SELECT '`dataform-open-source.dataform.table_a`' as ref_name"); + expect(actionB.tasks[0].compiledSql).to.include(`SELECT '\`${INTEGRATION_TEST_PROJECT}.dataform.table_a\`' as ref_name`); }); test("JiT to JiT dependency chain", async () => { @@ -88,6 +89,6 @@ suite("JiT support dependencies", ({ afterEach }) => { const executedGraph = JSON.parse(runResult.stdout); expect(executedGraph.actions.length).to.equal(2); const actionB = executedGraph.actions.find((a: any) => a.target.name === "jit_b"); - expect(actionB.tasks[0].compiledSql).to.include("SELECT '`dataform-open-source.dataform.jit_a`' as ref_name"); + expect(actionB.tasks[0].compiledSql).to.include(`SELECT '\`${INTEGRATION_TEST_PROJECT}.dataform.jit_a\`' as ref_name`); }); }); diff --git a/cli/tests/jit/index_jit_main_test.ts b/cli/tests/jit/index_jit_main_test.ts index 015581b38..3a86732b2 100644 --- a/cli/tests/jit/index_jit_main_test.ts +++ b/cli/tests/jit/index_jit_main_test.ts @@ -6,7 +6,7 @@ import * as path from "path"; import { cliEntryPointPath, CREDENTIALS_PATH, - DEFAULT_DATABASE, + INTEGRATION_TEST_PROJECT, setupJitProject } from "df/cli/index_test_base"; import { getProcessResult, nodePath, suite, test } from "df/testing"; @@ -182,7 +182,7 @@ suite("JiT support main", ({ afterEach }) => { `publish("rpc_jit", {type: "table"}).jitCode(async (jctx) => { // This will fail because the table does not exist in the warehouse, // and jctx.adapter.getTable throws an error in this case. - const table = await jctx.adapter.getTable({target: {database: "${DEFAULT_DATABASE}", schema: "sch", name: "tab"}}); + const table = await jctx.adapter.getTable({target: {database: "${INTEGRATION_TEST_PROJECT}", schema: "sch", name: "tab"}}); return "SELECT 1 as id"; })` ); @@ -210,7 +210,7 @@ suite("JiT support main", ({ afterEach }) => { expect(rpcAction.tasks[0].status).to.equal(3); expect(rpcAction.tasks[0].errorMessage).to.include("JiT compilation error"); expect(rpcAction.tasks[0].errorMessage).to.include("Table not found"); - expect(rpcAction.tasks[0].errorMessage).to.include(DEFAULT_DATABASE); + expect(rpcAction.tasks[0].errorMessage).to.include(INTEGRATION_TEST_PROJECT); expect(rpcAction.tasks[0].errorMessage).to.include('"schema":"sch"'); expect(rpcAction.tasks[0].errorMessage).to.include('"name":"tab"'); }); diff --git a/contributing.md b/contributing.md index 36831d064..cde67f956 100644 --- a/contributing.md +++ b/contributing.md @@ -40,25 +40,23 @@ The following command runs tests for @dataform/core: bazel test //core/... ``` -### Integration Test - -To run the CLI integration test against your own GCP project: +### Unit Test -1. Comment out the following dependency in `cli/BUILD`: +You can run all unit tests across the project by running this script: - - `//test_credentials:bigquery.json` +```bash +./scripts/run_tests +``` -2. Update the following constants in `cli/index_test.ts` to match your project: +### Integration Test - - `DEFAULT_DATABASE` - - `DEFAULT_LOCATION` - - `CREDENTIALS_PATH` +To run the CLI integration test against your own GCP project: - Prepare a credentials JSON file referenced by `CREDENTIALS_PATH`. Set values as follows: +1. Prepare a credentials JSON file at `test_credentials/bigquery.json`. Set values as follows: - - `projectId`: the same string as `DEFAULT_DATABASE`. + - `projectId`: your GCP project id - `credentials`: the entire content of your GCP service account key JSON file as a single string (you can generate it with `jq -Rsa < path/to/key.json`). - - `location`: the same string as `DEFAULT_LOCATION`. + - `location`: location to use in your project Example: @@ -70,10 +68,10 @@ To run the CLI integration test against your own GCP project: } ``` -3. Run the test: +2. You can run all integration tests by using this script: ```bash - bazel test //cli:index_test + ./scripts/run_integration_tests ``` ### Lint diff --git a/test_credentials/BUILD b/test_credentials/BUILD index bd9edfa97..04c896838 100644 --- a/test_credentials/BUILD +++ b/test_credentials/BUILD @@ -1,11 +1,13 @@ -load("//tools/gcloud:secrets.bzl", "gcloud_secret") +load("//tools/gcloud:secrets.bzl", "gcloud_secret_or_local") package(default_visibility = ["//visibility:public"]) -# You can update the credentials for testing by: +# The credentials from bigquery.json will be used if the file exists; otherwise the default will +# be loaded from bigquery.json.enc. +# You can update the default credentials for testing by: # * Ensuring you have the required permissions (at time of writing, you need to request a grant). # * Run the "scripts/update_test_credentials" script. -gcloud_secret( +gcloud_secret_or_local( name = "bigquery.json", testonly = 1, ciphertext_file = ":bigquery.json.enc", diff --git a/tests/integration/BUILD b/tests/integration/BUILD index be96b07f1..f0cf30e13 100644 --- a/tests/integration/BUILD +++ b/tests/integration/BUILD @@ -20,6 +20,7 @@ ts_test_suite( ], tags = ["integration"], deps = [ + "//cli:index_test_base", "//cli/api", "//cli/api/utils", "//common/promises", diff --git a/tests/integration/bigquery.spec.ts b/tests/integration/bigquery.spec.ts index 0e4f278cb..fde9eaab6 100644 --- a/tests/integration/bigquery.spec.ts +++ b/tests/integration/bigquery.spec.ts @@ -7,6 +7,7 @@ import * as dfapi from "df/cli/api"; import * as dbadapters from "df/cli/api/dbadapters"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; import { ExecutionSql } from "df/cli/api/dbadapters/execution_sql"; +import { INTEGRATION_TEST_LOCATION, INTEGRATION_TEST_PROJECT } from "df/cli/index_test_base"; import { targetAsReadableString } from "df/core/targets"; import { dataform } from "df/protos/ts"; import { suite, test } from "df/testing"; @@ -29,7 +30,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) // Drop schemas to make sure schema creation works. await dbadapter.execute( - "drop schema if exists `dataform-open-source.df_integration_test_project_e2e` cascade" + `drop schema if exists \`${INTEGRATION_TEST_PROJECT}.df_integration_test_project_e2e\` cascade` ); // Run the project. @@ -41,8 +42,8 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) // Check the status of action execution. const expectedFailedActions = [ - "dataform-open-source.df_integration_test_assertions_project_e2e.example_assertion_fail", - "dataform-open-source.df_integration_test_project_e2e.example_operation_partial_fail" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_assertions_project_e2e.example_assertion_fail`, + `${INTEGRATION_TEST_PROJECT}.df_integration_test_project_e2e.example_operation_partial_fail` ]; for (const actionName of Object.keys(actionMap)) { const expectedResult = expectedFailedActions.includes(actionName) @@ -56,13 +57,13 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) expect( actionMap[ - "dataform-open-source.df_integration_test_assertions_project_e2e.example_assertion_fail" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_assertions_project_e2e.example_assertion_fail` ].tasks[1].errorMessage ).to.eql("bigquery error: Assertion failed: query returned 1 row(s)."); expect( actionMap[ - "dataform-open-source.df_integration_test_project_e2e.example_operation_partial_fail" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_project_e2e.example_operation_partial_fail` ].tasks[0].errorMessage ).to.eql("bigquery error: Query error: Unrecognized name: invalid_column at [3:8]"); }); @@ -106,7 +107,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) const [incrementalRows, incrementalMergeRows] = await Promise.all([ getTableRows( { - database: "dataform-open-source", + database: INTEGRATION_TEST_PROJECT, schema: "df_integration_test_incremental_tables", name: "example_incremental" }, @@ -115,7 +116,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ), getTableRows( { - database: "dataform-open-source", + database: INTEGRATION_TEST_PROJECT, schema: "df_integration_test_incremental_tables", name: "example_incremental_merge" }, @@ -152,7 +153,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) for (const expectedMetadata of [ { target: { - database: "dataform-open-source", + database: INTEGRATION_TEST_PROJECT, schema: "df_integration_test_dataset_metadata", name: "example_incremental" }, @@ -191,7 +192,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) }, { target: { - database: "dataform-open-source", + database: INTEGRATION_TEST_PROJECT, schema: "df_integration_test_dataset_metadata", name: "example_view" }, @@ -260,7 +261,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) await dfapi.run(dbadapter, executionGraph).result(); const view = keyBy(compiledGraph.tables, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_evaluate.example_view" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_evaluate.example_view` ]; let evaluations = await dbadapter.evaluate(dataform.Table.create(view)); expect(evaluations.length).to.equal(1); @@ -269,7 +270,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ); const materializedView = keyBy(compiledGraph.tables, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_evaluate.example_materialized_view" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_evaluate.example_materialized_view` ]; evaluations = await dbadapter.evaluate(dataform.Table.create(materializedView)); expect(evaluations.length).to.equal(1); @@ -278,7 +279,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ); const table = keyBy(compiledGraph.tables, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_evaluate.example_table" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_evaluate.example_table` ]; evaluations = await dbadapter.evaluate(dataform.Table.create(table)); expect(evaluations.length).to.equal(1); @@ -287,7 +288,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ); const operation = keyBy(compiledGraph.operations, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_evaluate.example_operation" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_evaluate.example_operation` ]; evaluations = await dbadapter.evaluate(dataform.Operation.create(operation)); expect(evaluations.length).to.equal(1); @@ -296,7 +297,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ); const assertion = keyBy(compiledGraph.assertions, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_assertions_evaluate.example_assertion_pass" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_assertions_evaluate.example_assertion_pass` ]; evaluations = await dbadapter.evaluate(dataform.Assertion.create(assertion)); expect(evaluations.length).to.equal(1); @@ -305,7 +306,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) ); const incremental = keyBy(compiledGraph.tables, t => targetAsReadableString(t.target))[ - "dataform-open-source.df_integration_test_evaluate.example_incremental" + `${INTEGRATION_TEST_PROJECT}.df_integration_test_evaluate.example_incremental` ]; evaluations = await dbadapter.evaluate(dataform.Table.create(incremental)); expect(evaluations.length).to.equal(2); @@ -321,7 +322,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) const target = (name: string) => ({ schema: "df_integration_test", name, - database: "dataform-open-source" + database: INTEGRATION_TEST_PROJECT }); let evaluations = await dbadapter.evaluate( @@ -503,8 +504,8 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after }) fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), ` -defaultProject: dataform-open-source -defaultLocation: US +defaultProject: ${INTEGRATION_TEST_PROJECT} +defaultLocation: ${INTEGRATION_TEST_LOCATION} defaultDataset: df_integration_test_jit ` ); @@ -531,7 +532,7 @@ defaultDataset: df_integration_test_jit // Drop dataset to start fresh await dbadapter.execute( - "drop schema if exists `dataform-open-source.df_integration_test_jit` cascade" + `drop schema if exists \`${INTEGRATION_TEST_PROJECT}.df_integration_test_jit\` cascade` ); const executionGraph = await dfapi.build(compiledGraph, {}, dbadapter); @@ -541,7 +542,7 @@ defaultDataset: df_integration_test_jit dataform.RunResult.ExecutionStatus[dataform.RunResult.ExecutionStatus.SUCCESSFUL] ); - const rows = await dbadapter.execute("SELECT * FROM `dataform-open-source.df_integration_test_jit.jit_table`").then(res => res.rows); + const rows = await dbadapter.execute(`SELECT * FROM \`${INTEGRATION_TEST_PROJECT}.df_integration_test_jit.jit_table\``).then(res => res.rows); expect(rows).to.eql([{ id: 1 }]); } finally { if (fs.existsSync(projectDir)) { @@ -560,7 +561,7 @@ defaultDataset: df_integration_test_jit fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), ` -defaultProject: dataform-open-source +defaultProject: ${INTEGRATION_TEST_PROJECT} defaultLocation: US defaultDataset: df_integration_test_jit_dry_run ` @@ -587,7 +588,7 @@ defaultDataset: df_integration_test_jit_dry_run // Drop dataset to start fresh await dbadapter.execute( - "drop schema if exists `dataform-open-source.df_integration_test_jit_dry_run` cascade" + `drop schema if exists \`${INTEGRATION_TEST_PROJECT}.df_integration_test_jit_dry_run\` cascade` ); const executionGraph = await dfapi.build(compiledGraph, {}, dbadapter); @@ -602,11 +603,11 @@ defaultDataset: df_integration_test_jit_dry_run ); // Verify that the table was NOT created - const tables = await dbadapter.schemas("dataform-open-source").then(schemas => { + const tables = await dbadapter.schemas(INTEGRATION_TEST_PROJECT).then(schemas => { if (!schemas.includes("df_integration_test_jit_dry_run")) { return []; } - return dbadapter.tables("dataform-open-source", "df_integration_test_jit_dry_run"); + return dbadapter.tables(INTEGRATION_TEST_PROJECT, "df_integration_test_jit_dry_run"); }); expect(tables.length).to.equal(0); } finally { diff --git a/tests/integration/property_graph.spec.ts b/tests/integration/property_graph.spec.ts index ab73982ca..e420345e5 100644 --- a/tests/integration/property_graph.spec.ts +++ b/tests/integration/property_graph.spec.ts @@ -3,12 +3,12 @@ import { randomBytes } from "crypto"; import * as dfapi from "df/cli/api"; import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { INTEGRATION_TEST_PROJECT } from "df/cli/index_test_base"; import { targetAsReadableString } from "df/core/targets"; import { dataform } from "df/protos/ts"; import { suite, test } from "df/testing"; import { compile, keyBy } from "df/tests/integration/utils"; -const PROJECT = "dataform-open-source"; const GRAPH_NAME = "LibraryGraph"; function makeSuffix() { @@ -21,7 +21,7 @@ function makeSuffix() { async function dropDataset(dbadapter: BigQueryDbAdapter, dataset: string) { await dbadapter.execute( - `drop schema if exists \`${PROJECT}.${dataset}\` cascade` + `drop schema if exists \`${INTEGRATION_TEST_PROJECT}.${dataset}\` cascade` ); } @@ -29,7 +29,7 @@ suite("@dataform/integration/property_graph", { parallel: true }, ({ before, aft const credentials = dfapi.credentials.read("test_credentials/bigquery.json"); const schemaSuffix = `e2e_${makeSuffix()}`; const dataset = `df_integration_test_pg_${schemaSuffix}`; - const graphTarget = `${dataset}.${GRAPH_NAME}`; + const graphTarget = `${INTEGRATION_TEST_PROJECT}.${dataset}.${GRAPH_NAME}`; let dbadapter: BigQueryDbAdapter; before("create adapter", async () => { @@ -65,11 +65,11 @@ suite("@dataform/integration/property_graph", { parallel: true }, ({ before, aft const rows = (await dbadapter.execute( `select property_graph_catalog, property_graph_schema, property_graph_name, ddl - from \`${PROJECT}.${dataset}\`.INFORMATION_SCHEMA.PROPERTY_GRAPHS` + from \`${INTEGRATION_TEST_PROJECT}.${dataset}\`.INFORMATION_SCHEMA.PROPERTY_GRAPHS` )).rows; expect(rows).to.have.lengthOf(1); const [row] = rows; - expect(row.property_graph_catalog).equals(PROJECT); + expect(row.property_graph_catalog).equals(INTEGRATION_TEST_PROJECT); expect(row.property_graph_schema).equals(dataset); expect(row.property_graph_name).equals(GRAPH_NAME); for (const needle of [ diff --git a/tests/integration/utils.ts b/tests/integration/utils.ts index 4d7a1d6ff..7a834d76c 100644 --- a/tests/integration/utils.ts +++ b/tests/integration/utils.ts @@ -3,6 +3,7 @@ import { expect } from "chai"; import * as dfapi from "df/cli/api"; import * as dbadapters from "df/cli/api/dbadapters"; import { ExecutionSql } from "df/cli/api/dbadapters/execution_sql"; +import { INTEGRATION_TEST_PROJECT } from "df/cli/index_test_base"; import { dataform } from "df/protos/ts"; export function keyBy(values: V[], keyFn: (value: V) => string): { [key: string]: V } { @@ -37,7 +38,10 @@ export async function compile( ) { const compiledGraph = await dfapi.compile({ projectDir, - projectConfigOverride: { schemaSuffix: schemaSuffixOverride } + projectConfigOverride: { + defaultDatabase: INTEGRATION_TEST_PROJECT, + schemaSuffix: schemaSuffixOverride + } }); expect(compiledGraph.graphErrors.compilationErrors).to.eql([]); diff --git a/tools/gcloud/secrets.bzl b/tools/gcloud/secrets.bzl index 862b9fda2..298c739a1 100644 --- a/tools/gcloud/secrets.bzl +++ b/tools/gcloud/secrets.bzl @@ -32,3 +32,9 @@ gcloud_secret = rule( "location": attr.string(default = "global"), }, ) + +def gcloud_secret_or_local(name, **kwargs): + if native.glob([name], allow_empty = True): + native.exports_files([name]) + else: + gcloud_secret(name = name, **kwargs)