From 4e5c8c48de44102ff3a6eeee8f0ac9b223389e54 Mon Sep 17 00:00:00 2001 From: "databricks-ci-ghec-1[bot]" <184311507+databricks-ci-ghec-1[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:14:11 +0000 Subject: [PATCH] Release sdk-go --- .codegen/_last_sha | 2 +- accessmanagement/CHANGELOG.md | 2 + accessmanagement/internal/version.go | 2 +- accessmanagement/v1/model.go | 4 +- agentkit/CHANGELOG.md | 2 + agentkit/internal/version.go | 2 +- agentkit/v1/client.go | 8 +- agentkit/v1/model.go | 34 +- aifunctions/CHANGELOG.md | 2 + aifunctions/internal/version.go | 2 +- aifunctions/v1/model.go | 10 +- aigateway/CHANGELOG.md | 2 + aigateway/internal/version.go | 2 +- aigateway/v1/model.go | 60 +- aisearch/CHANGELOG.md | 2 + aisearch/internal/version.go | 2 +- aisearch/v1/client.go | 9 +- aisearch/v1/model.go | 100 +- aisearch/v1/wire.go | 11 +- budgetpolicy/CHANGELOG.md | 2 + budgetpolicy/internal/version.go | 2 +- budgetpolicy/v1/client.go | 15 +- budgetpolicy/v1/model.go | 34 +- cleanrooms/CHANGELOG.md | 2 + cleanrooms/internal/version.go | 2 +- cleanrooms/v1/client.go | 21 +- cleanrooms/v1/model.go | 42 +- clusterlibraries/CHANGELOG.md | 2 + clusterlibraries/internal/version.go | 2 +- clusterlibraries/v2/client.go | 11 +- clusterlibraries/v2/model.go | 26 +- clusterpolicies/CHANGELOG.md | 2 + clusterpolicies/internal/version.go | 2 +- clusterpolicies/v2/model.go | 18 +- clusters/CHANGELOG.md | 2 + clusters/internal/version.go | 2 +- clusters/v2/model.go | 453 +++++- disasterrecovery/CHANGELOG.md | 2 + disasterrecovery/internal/version.go | 2 +- disasterrecovery/v1/client.go | 34 +- disasterrecovery/v1/model.go | 97 +- environments/CHANGELOG.md | 2 + environments/internal/version.go | 2 +- environments/v1/client.go | 29 +- environments/v1/model.go | 31 +- experiments/CHANGELOG.md | 2 + experiments/internal/version.go | 2 +- experiments/v1/client.go | 11 +- experiments/v1/model.go | 79 +- features/CHANGELOG.md | 2 + features/internal/version.go | 2 +- features/v1/model.go | 14 +- features/v1/wire.go | 57 + files/CHANGELOG.md | 2 + files/internal/version.go | 2 +- files/v2/ext_discovery.go | 201 +++ files/v2/ext_discovery_test.go | 203 +++ files/v2/ext_endpoints.go | 313 ++++ files/v2/ext_endpoints_test.go | 394 +++++ files/v2/ext_limiter.go | 8 +- files/v2/ext_limiter_test.go | 21 - files/v2/ext_multipart.go | 90 +- files/v2/ext_port_contract_test.go | 11 +- files/v2/ext_resumable.go | 187 --- files/v2/ext_resumable_test.go | 21 - files/v2/ext_tunables.go | 71 +- files/v2/ext_upload.go | 320 ++-- files/v2/ext_upload_control.go | 323 ++-- files/v2/ext_upload_control_test.go | 85 +- files/v2/ext_upload_data.go | 209 +++ files/v2/ext_upload_helpers.go | 43 +- files/v2/ext_upload_test.go | 833 ++++++++-- .../v2/internal/cloudstorage/cloudstorage.go | 17 +- .../cloudstorage/cloudstorage_test.go | 34 +- files/v2/internal/cloudstorage/retries.go | 15 +- genie/CHANGELOG.md | 2 + genie/internal/version.go | 2 +- genie/v1/model.go | 9 +- gitcredentials/CHANGELOG.md | 2 + gitcredentials/internal/version.go | 2 +- gitcredentials/v1/client.go | 3 +- gitcredentials/v1/model.go | 47 +- instancepools/CHANGELOG.md | 2 + instancepools/internal/version.go | 2 +- instancepools/v2/model.go | 4 +- jobs/CHANGELOG.md | 2 + jobs/internal/version.go | 2 +- jobs/v2/model.go | 99 +- lakeview/CHANGELOG.md | 2 + lakeview/internal/version.go | 2 +- lakeview/v1/client.go | 42 +- lakeview/v1/model.go | 179 +- lakewatch/CHANGELOG.md | 2 + lakewatch/internal/version.go | 2 +- lakewatch/v1/client.go | 1337 ++++++++++++--- lakewatch/v1/model.go | 315 ++++ lakewatch/v1/wire.go | 1445 +++++++++++++++-- modelregistry/CHANGELOG.md | 2 + modelregistry/internal/version.go | 2 +- modelregistry/v1/client.go | 2 +- modelregistry/v1/model.go | 93 +- pipelines/CHANGELOG.md | 2 + pipelines/internal/version.go | 2 +- pipelines/v2/model.go | 1 + postgres/CHANGELOG.md | 2 + postgres/internal/version.go | 2 +- postgres/v1/client.go | 26 +- postgres/v1/model.go | 57 +- queryhistory/CHANGELOG.md | 2 + queryhistory/internal/version.go | 2 +- queryhistory/v1/model.go | 76 +- scim/CHANGELOG.md | 2 + scim/internal/version.go | 2 +- scim/v1/client.go | 17 +- scim/v1/model.go | 188 ++- secrets/CHANGELOG.md | 2 + secrets/internal/version.go | 2 +- secrets/v1/client.go | 62 +- secrets/v1/model.go | 13 +- sharing/CHANGELOG.md | 2 + sharing/internal/version.go | 2 +- sharing/v1/client.go | 3 + sharing/v1/model.go | 28 + sharing/v1/wire.go | 81 + softwareartifacts/CHANGELOG.md | 2 + softwareartifacts/internal/version.go | 2 +- softwareartifacts/v1/client.go | 9 +- softwareartifacts/v1/model.go | 38 +- statementexecution/CHANGELOG.md | 2 + statementexecution/internal/version.go | 2 +- statementexecution/v1/model.go | 4 +- statementexecution/v2/model.go | 4 +- tagassignments/CHANGELOG.md | 2 + tagassignments/internal/version.go | 2 +- tagassignments/v1/model.go | 14 +- uc/connections/CHANGELOG.md | 2 + uc/connections/internal/version.go | 2 +- uc/connections/v1/model.go | 17 +- uc/schemas/CHANGELOG.md | 2 + uc/schemas/internal/version.go | 2 +- uc/schemas/v1/model.go | 132 +- uc/systemschemas/CHANGELOG.md | 2 + uc/systemschemas/internal/version.go | 2 +- uc/systemschemas/v1/model.go | 25 +- uc/tables/CHANGELOG.md | 2 + uc/tables/internal/version.go | 2 +- uc/tables/v1/model.go | 142 +- vectorsearch/CHANGELOG.md | 2 + vectorsearch/internal/version.go | 2 +- vectorsearch/v1/model.go | 74 +- vectorsearch/v1/wire.go | 107 +- warehouses/CHANGELOG.md | 2 + warehouses/internal/version.go | 2 +- warehouses/v1/model.go | 16 +- 154 files changed, 7578 insertions(+), 1888 deletions(-) create mode 100644 files/v2/ext_discovery.go create mode 100644 files/v2/ext_discovery_test.go create mode 100644 files/v2/ext_endpoints.go create mode 100644 files/v2/ext_endpoints_test.go delete mode 100644 files/v2/ext_resumable.go delete mode 100644 files/v2/ext_resumable_test.go create mode 100644 files/v2/ext_upload_data.go diff --git a/.codegen/_last_sha b/.codegen/_last_sha index 2a64ce6a..187561b9 100644 --- a/.codegen/_last_sha +++ b/.codegen/_last_sha @@ -1 +1 @@ -b4c5fc35beb5ed8fe8790b448b97238daef64520 \ No newline at end of file +d3ffd787e1c4e241854c80101a4fb53ec326c94e \ No newline at end of file diff --git a/accessmanagement/CHANGELOG.md b/accessmanagement/CHANGELOG.md index 79c44032..5589e223 100644 --- a/accessmanagement/CHANGELOG.md +++ b/accessmanagement/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-12) ### New Features and Improvements diff --git a/accessmanagement/internal/version.go b/accessmanagement/internal/version.go index 85585817..e8f62916 100644 --- a/accessmanagement/internal/version.go +++ b/accessmanagement/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-accessmanagement" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/accessmanagement/v1/model.go b/accessmanagement/v1/model.go index babe91d6..37b6991b 100644 --- a/accessmanagement/v1/model.go +++ b/accessmanagement/v1/model.go @@ -409,8 +409,8 @@ func (*PrincipalOutput_PrincipalName_ServicePrincipalName) isPrincipalOutput_Pri type ResourceInfo struct { // Id of the current resource. Id *string - // Parent resource info for the current resource. The parent may have another - // parent. + // Deprecated: use `contributing_resources`. Parent resource info for the + // current resource. The parent may have another parent. ParentResourceInfo *ResourceInfo // The legacy acl path of the current resource. LegacyAclPath *string diff --git a/agentkit/CHANGELOG.md b/agentkit/CHANGELOG.md index 12c269e5..34271e2e 100644 --- a/agentkit/CHANGELOG.md +++ b/agentkit/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-12) ### New Features and Improvements diff --git a/agentkit/internal/version.go b/agentkit/internal/version.go index c81a778a..0f06e7c9 100644 --- a/agentkit/internal/version.go +++ b/agentkit/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-agentkit" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/agentkit/v1/client.go b/agentkit/v1/client.go index 46f1bf8d..840e7b64 100644 --- a/agentkit/v1/client.go +++ b/agentkit/v1/client.go @@ -299,7 +299,8 @@ func (c *internalClient) CreateMemory(ctx context.Context, req CreateManagedMemo return resp, nil } -// Creates a memory store in the caller's workspace. +// Creates a memory store in the caller's workspace. Returns `ALREADY_EXISTS` +// when a store with the requested ID already exists. func (c *internalClient) CreateMemoryStore(ctx context.Context, req CreateManagedMemoryStoreRequest, opts ...call.Option) (*ManagedMemoryStore, error) { wireReq, err := createManagedMemoryStoreRequestToWire(&req) if err != nil { @@ -567,8 +568,9 @@ func (c *internalClient) DeleteMemory(ctx context.Context, req DeleteManagedMemo return nil } -// Deletes a memory store by resource name. Returns `NOT_FOUND` when the store -// does not exist in the caller's workspace. +// Deletes a memory store by resource name. Deleting a store also deletes all of +// its memory entries, and its ID becomes available for reuse. Returns +// `NOT_FOUND` when the store does not exist in the caller's workspace. func (c *internalClient) DeleteMemoryStore(ctx context.Context, req DeleteManagedMemoryStoreRequest, opts ...call.Option) error { headers := http.Header{} diff --git a/agentkit/v1/model.go b/agentkit/v1/model.go index 8bfa5519..69dbd90c 100644 --- a/agentkit/v1/model.go +++ b/agentkit/v1/model.go @@ -105,7 +105,8 @@ type CreateManagedMemoryStoreRequest struct { ManagedMemoryStore *ManagedMemoryStore // Caller-provided, workspace-unique memory store ID. It must be 3-56 // characters, begin with a lowercase letter, contain only lowercase letters, - // digits, and hyphens, and end with a letter or digit. + // digits, and hyphens, and end with a letter or digit. It becomes the last + // segment of the store's `name` and cannot be changed after creation. ManagedMemoryStoreId *string } @@ -260,7 +261,9 @@ type GrantMemoryStorePermissionRequest struct { Name *string // Principal to grant the permission to. Principal *StorePrincipal - // Permission level to grant. + // Permission level to grant. `WRITE` includes `READ`. Grants are additive: + // granting `READ` to a principal that already holds `WRITE` does not lower its + // access. To downgrade a principal, revoke its grants and grant again. Permission PermissionLevel } @@ -321,7 +324,8 @@ type ListManagedMemoryStoresRequest struct { // Maximum number of stores to return. The service may return fewer stores than // requested. Defaults to 10; must be between 1 and 100. PageSize *int - // Opaque pagination token from a previous ListManagedMemoryStores response. + // Opaque pagination token, taken from the `next_page_token` of a previous + // response. PageToken *string } @@ -466,9 +470,7 @@ type ManagedMemoryEntry struct { // Human-readable description of the memory entry. Description *string `fieldmask:"description"` // Which writer created this entry. Caller sets this on Create; immutable after - // creation. While the workspace's default-source-type rollout flag is on, - // omitted or UNSPECIFIED is stored as AGENT. While it is off, omitted is stored - // unset and UNSPECIFIED as UNSPECIFIED. + // creation. The server may store an omitted or UNSPECIFIED value as AGENT. SourceType ManagedMemoryEntrySourceType `fieldmask:"source_type"` // Time when the entry was created. CreateTime *types.Time `fieldmask:"create_time"` @@ -484,7 +486,11 @@ type ManagedMemoryEntrySearchResult struct { Score *float64 } -// A workspace-scoped memory store backed by service-managed storage.. +// A workspace-scoped memory store backed by service-managed storage. A store is +// the container for an agent's memory entries and the boundary at which access +// is granted; creating one provisions its Lakebase storage automatically. See +// https://docs.databricks.com/aws/en/agents/agent-memory/managed-memory for an +// overview.. type ManagedMemoryStore struct { // Resource name in the form `memory-stores/{managed_memory_store_id}`. Name *string `fieldmask:"name"` @@ -682,7 +688,9 @@ type SessionStorePermission struct { type StorageBackend struct { // Type of the storage backend. BackendType StorageBackendType `fieldmask:"backend_type"` - // Backend-specific identifier. For Lakebase, this is the project ID. + // Backend-specific identifier. For Lakebase, this is the resource name of the + // database that holds the store's entries, in the form + // `projects/{project_id}/branches/{branch_id}/databases/{database_id}`. BackendId *string `fieldmask:"backend_id"` } @@ -690,8 +698,11 @@ type StorageBackend struct { // Lakebase identifier: a user's email or a service principal's application // (client) id.. type StorePrincipal struct { + // Identifier of the principal: the email address of a user, or the application + // (client) ID of a service principal. Groups are not supported. Name *string - // The kind of principal. Required unless the deprecated type alias is supplied. + // The kind of principal, which determines how `name` is interpreted. A + // principal without a type is rejected. PrincipalType PrincipalType } @@ -707,8 +718,9 @@ type UpdateManagedMemoryEntryRequest struct { type UpdateManagedMemoryStoreRequest struct { // The memory store to update. `name` is taken from the URL. ManagedMemoryStore *ManagedMemoryStore - // Only `description` and `metadata` may be updated. Selecting `metadata` - // replaces the entire map. + // Only `description` and `metadata` may be updated. Selecting `description` + // while leaving it unset clears it. Selecting `metadata` replaces the entire + // map. UpdateMask *types.FieldMask[ManagedMemoryStore] } diff --git a/aifunctions/CHANGELOG.md b/aifunctions/CHANGELOG.md index d5e83464..6b381228 100644 --- a/aifunctions/CHANGELOG.md +++ b/aifunctions/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.10.0 (2026-11-17) + ## Release v0.9.0 (2026-11-16) ## Release v0.8.0 (2026-11-12) diff --git a/aifunctions/internal/version.go b/aifunctions/internal/version.go index 4d334efe..cc0c5b33 100644 --- a/aifunctions/internal/version.go +++ b/aifunctions/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-aifunctions" -const Version = "0.9.0" +const Version = "0.10.0" diff --git a/aifunctions/v1/model.go b/aifunctions/v1/model.go index ea7cd30f..9c6daf61 100644 --- a/aifunctions/v1/model.go +++ b/aifunctions/v1/model.go @@ -251,10 +251,12 @@ type AiExtractRequest struct { Content json.RawMessage // The extraction schema defining the fields to extract. Either a JSON array of // field names, assumed to be strings (e.g. ["company", "valuation"]), or a JSON - // object mapping each field to its type/description/nullability (e.g. - // {"company": {"type": "string", "description": "the company name"}}). Accepts - // up to 256 fields, 12 levels of nesting, and 500 enum values. Supported field - // types are string, integer, number, boolean, and enum. + // object mapping each field to its type/description (e.g. {"company": {"type": + // "string", "description": "the company name"}}). Enum fields list their values + // in "labels" (e.g. {"currency": {"type": "enum", "labels": ["USD", "EUR"]}}). + // Primitive and enum fields are nullable by default. Accepts up to 256 fields, + // 12 levels of nesting, and 500 enum values. Supported field types are string, + // integer, number, boolean, and enum. Schema json.RawMessage // Function options. Omitted fields fall back to their documented defaults. Options *AiExtractOptions diff --git a/aigateway/CHANGELOG.md b/aigateway/CHANGELOG.md index cda31efd..d556d836 100644 --- a/aigateway/CHANGELOG.md +++ b/aigateway/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.22.0 (2026-11-17) + ## Release v0.21.0 (2026-11-16) ## Release v0.20.0 (2026-11-12) diff --git a/aigateway/internal/version.go b/aigateway/internal/version.go index 1403cf24..f4a8d95d 100644 --- a/aigateway/internal/version.go +++ b/aigateway/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-aigateway" -const Version = "0.21.0" +const Version = "0.22.0" diff --git a/aigateway/v1/model.go b/aigateway/v1/model.go index be39eb60..c9847941 100644 --- a/aigateway/v1/model.go +++ b/aigateway/v1/model.go @@ -146,7 +146,9 @@ type CreateMcpServiceRequest struct { // Name of the parent schema. Format: `schemas/{catalog}.{schema}`. Each `{...}` // component is capped at 255 characters individually. Parent *string - // Name for the MCP service, e.g. "my_mcp_service". + // Name of the MCP service within the parent schema. The name must not contain + // periods, spaces, forward slashes, percent signs, or control characters, and + // it is stored in lowercase. McpServiceId *string // The MCP service to create. Do not set `name`; the server derives it from // `parent` and `mcp_service_id`. `source_connection` is required. @@ -167,7 +169,10 @@ type CreateModelProviderServiceRequest struct { // Name of the parent schema. Format: `schemas/{catalog}.{schema}`. Each `{...}` // component is capped at 255 characters individually. Parent *string - // Name for the model provider service, e.g. "openai_prod". + // Name of the model provider service within the parent schema. The name must + // not contain periods, spaces, forward slashes, percent signs, or control + // characters, must not end in `_DO_NOT_MODIFY`, and must not exceed 241 + // characters. It is stored in lowercase. ModelProviderServiceId *string // The model provider service to create. Do not set `name`; the server derives // it from `parent` and `model_provider_service_id`. @@ -442,20 +447,24 @@ type ListSkillsResponse struct { type McpService struct { // Resource name of the MCP service. Format: // `mcp-services/{catalog}.{schema}.{mcp_service}`. Each `{...}` component is - // capped at 255 characters individually. Server-derived on Create from `parent` - // + `mcp_service_id`; required and immutable on Update/Get/Delete. + // capped at 255 characters individually. The server derives the name on Create + // from `parent` and `mcp_service_id`; it is required and immutable on Update, + // Get, and Delete. Name *string `fieldmask:"name"` - // Owner of the MCP service. + // Username, group name, or service principal application ID of the MCP service + // owner. EffectiveOwner *string `fieldmask:"effective_owner"` - // Metastore hosting the MCP service. + // Unique identifier of the metastore that contains the MCP service. MetastoreId *string `fieldmask:"metastore_id"` // Time the MCP service was created. CreateTime *types.Time `fieldmask:"create_time"` - // Creator identity. + // Username or service principal application ID of the principal that created + // the MCP service. CreatedBy *string `fieldmask:"created_by"` // Time the MCP service was last modified. UpdateTime *types.Time `fieldmask:"update_time"` - // Identity of the last updater. + // Username or service principal application ID of the principal that last + // updated the MCP service. UpdatedBy *string `fieldmask:"updated_by"` // User-provided description. Comment *string `fieldmask:"comment"` @@ -569,21 +578,21 @@ type McpServiceUserMappedCredentialLogin struct { type ModelProviderService struct { // Resource name of the provider service. Format: // `model-provider-services/{catalog}.{schema}.{model_provider_service}`. Each - // `{...}` component is capped at 255 characters individually. Server-derived on - // Create from `parent` + `model_provider_service_id`; required and immutable on - // Update/Get/Delete. + // `{...}` component is capped at 255 characters individually. The server + // derives the name on Create from `parent` and `model_provider_service_id`; it + // is required and immutable on Update, Get, and Delete. Name *string `fieldmask:"name"` - // Owner of the model provider service. + // Username or group name of the model provider service owner. EffectiveOwner *string `fieldmask:"effective_owner"` - // Metastore hosting the provider service. + // Unique identifier of the metastore that contains the model provider service. MetastoreId *string `fieldmask:"metastore_id"` // Time the provider service was created. CreateTime *types.Time `fieldmask:"create_time"` - // Creator identity. + // Username of the user who created the model provider service. CreatedBy *string `fieldmask:"created_by"` // Time the provider service was last modified. UpdateTime *types.Time `fieldmask:"update_time"` - // Identity of the last updater. + // Username of the user who last updated the model provider service. UpdatedBy *string `fieldmask:"updated_by"` // User-provided description. Comment *string `fieldmask:"comment"` @@ -679,6 +688,8 @@ type isModelProviderServiceConfig_Provider interface { } // ModelProviderServiceConfig_Provider_Openai selects Openai for ModelProviderServiceConfig.Provider. +// Configuration for OpenAI. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI`. type ModelProviderServiceConfig_Provider_Openai struct { Openai ModelProviderServiceConfig_OpenAiProviderConfig `fieldmask:"openai"` } @@ -686,6 +697,8 @@ type ModelProviderServiceConfig_Provider_Openai struct { func (*ModelProviderServiceConfig_Provider_Openai) isModelProviderServiceConfig_Provider() {} // ModelProviderServiceConfig_Provider_AzureOpenai selects AzureOpenai for ModelProviderServiceConfig.Provider. +// Configuration for Azure OpenAI. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_AZURE_OPENAI`. type ModelProviderServiceConfig_Provider_AzureOpenai struct { AzureOpenai ModelProviderServiceConfig_AzureOpenAiProviderConfig `fieldmask:"azure_openai"` } @@ -693,6 +706,8 @@ type ModelProviderServiceConfig_Provider_AzureOpenai struct { func (*ModelProviderServiceConfig_Provider_AzureOpenai) isModelProviderServiceConfig_Provider() {} // ModelProviderServiceConfig_Provider_Anthropic selects Anthropic for ModelProviderServiceConfig.Provider. +// Configuration for Anthropic. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC`. type ModelProviderServiceConfig_Provider_Anthropic struct { Anthropic ModelProviderServiceConfig_AnthropicProviderConfig `fieldmask:"anthropic"` } @@ -700,6 +715,8 @@ type ModelProviderServiceConfig_Provider_Anthropic struct { func (*ModelProviderServiceConfig_Provider_Anthropic) isModelProviderServiceConfig_Provider() {} // ModelProviderServiceConfig_Provider_AmazonBedrock selects AmazonBedrock for ModelProviderServiceConfig.Provider. +// Configuration for Amazon Bedrock. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_AMAZON_BEDROCK`. type ModelProviderServiceConfig_Provider_AmazonBedrock struct { AmazonBedrock ModelProviderServiceConfig_AmazonBedrockProviderConfig `fieldmask:"amazon_bedrock"` } @@ -707,6 +724,8 @@ type ModelProviderServiceConfig_Provider_AmazonBedrock struct { func (*ModelProviderServiceConfig_Provider_AmazonBedrock) isModelProviderServiceConfig_Provider() {} // ModelProviderServiceConfig_Provider_Custom selects Custom for ModelProviderServiceConfig.Provider. +// Configuration for a custom OpenAI-compatible provider. Set this field when +// `provider_type` is `EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM`. type ModelProviderServiceConfig_Provider_Custom struct { Custom ModelProviderServiceConfig_CustomProviderConfig `fieldmask:"custom"` } @@ -714,6 +733,8 @@ type ModelProviderServiceConfig_Provider_Custom struct { func (*ModelProviderServiceConfig_Provider_Custom) isModelProviderServiceConfig_Provider() {} // ModelProviderServiceConfig_Provider_MicrosoftFoundry selects MicrosoftFoundry for ModelProviderServiceConfig.Provider. +// Configuration for Microsoft Foundry. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_MICROSOFT_FOUNDRY`. type ModelProviderServiceConfig_Provider_MicrosoftFoundry struct { MicrosoftFoundry ModelProviderServiceConfig_MicrosoftFoundryProviderConfig `fieldmask:"microsoft_foundry"` } @@ -722,6 +743,8 @@ func (*ModelProviderServiceConfig_Provider_MicrosoftFoundry) isModelProviderServ } // ModelProviderServiceConfig_Provider_GeminiEnterprise selects GeminiEnterprise for ModelProviderServiceConfig.Provider. +// Configuration for Google Gemini Enterprise. Set this field when +// `provider_type` is `EXTERNAL_MODEL_PROVIDER_TYPE_GEMINI_ENTERPRISE`. type ModelProviderServiceConfig_Provider_GeminiEnterprise struct { GeminiEnterprise ModelProviderServiceConfig_GeminiEnterpriseProviderConfig `fieldmask:"gemini_enterprise"` } @@ -730,6 +753,9 @@ func (*ModelProviderServiceConfig_Provider_GeminiEnterprise) isModelProviderServ } // ModelProviderServiceConfig_Provider_BedrockMantle selects BedrockMantle for ModelProviderServiceConfig.Provider. +// Configuration for Amazon Bedrock Mantle, with the same fields as +// `amazon_bedrock`. Set this field when `provider_type` is +// `EXTERNAL_MODEL_PROVIDER_TYPE_BEDROCK_MANTLE`. type ModelProviderServiceConfig_Provider_BedrockMantle struct { BedrockMantle ModelProviderServiceConfig_AmazonBedrockProviderConfig `fieldmask:"bedrock_mantle"` } @@ -1009,7 +1035,7 @@ type ModelProviderServiceConfig_CustomProviderApiKeyHeaderAuth struct { // supplied via `api_key_value`. ApiKeyName *string `fieldmask:"api_key_name"` // Secret value forwarded under the `api_key_name` header on outbound requests. - // Supplied as inline plaintext via `ProviderSecret.plaintext`. + // Supply the value in `api_key_value.plaintext`. ApiKeyValue *ModelProviderServiceConfig_ProviderSecret `fieldmask:"api_key_value"` } @@ -1389,7 +1415,7 @@ type isModelProviderServiceConfig_ProviderSecret_Value interface { } // ModelProviderServiceConfig_ProviderSecret_Value_Plaintext selects Plaintext for ModelProviderServiceConfig_ProviderSecret.Value. -// Inline plaintext credential. INPUT_ONLY: the value never round-trips on +// Inline plaintext credential. The value is write-only and never round-trips on // reads. Get and List responses omit `plaintext`; the enclosing secret object // remains present to indicate that a secret is configured. type ModelProviderServiceConfig_ProviderSecret_Value_Plaintext struct { diff --git a/aisearch/CHANGELOG.md b/aisearch/CHANGELOG.md index 4aa2fb8b..6e4ec60d 100644 --- a/aisearch/CHANGELOG.md +++ b/aisearch/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.11.0 (2026-11-17) + ## Release v0.10.0 (2026-11-16) ## Release v0.9.0 (2026-11-12) diff --git a/aisearch/internal/version.go b/aisearch/internal/version.go index d61875a2..f3e440c9 100644 --- a/aisearch/internal/version.go +++ b/aisearch/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-aisearch" -const Version = "0.10.0" +const Version = "0.11.0" diff --git a/aisearch/v1/client.go b/aisearch/v1/client.go index 735d6168..d27b5088 100644 --- a/aisearch/v1/client.go +++ b/aisearch/v1/client.go @@ -1051,11 +1051,10 @@ func (c *internalClient) ListEndpointsIter(ctx context.Context, req ListEndpoint } } -// Update an existing AI Search endpoint. Multi-bucket masks are supported and -// dispatched in deterministic bucket order: budget policy, custom tags, -// throughput, scaling/replicas, then metrics export. Per-bucket dispatch is not -// atomic across buckets — if a later bucket fails, earlier buckets may -// already have been applied. +// Update an existing AI Search endpoint. Only the fields named in `update_mask` +// change. The server applies a mask that spans several settings, such as +// `budget_policy_id` and `custom_tags`, one setting at a time, so if a later +// setting fails, the earlier ones may already have been applied. func (c *internalClient) UpdateEndpoint(ctx context.Context, req UpdateEndpointRequest, opts ...call.Option) (*Endpoint, error) { wireReq, err := updateEndpointRequestToWire(&req) if err != nil { diff --git a/aisearch/v1/model.go b/aisearch/v1/model.go index 36869cf8..da60a4af 100644 --- a/aisearch/v1/model.go +++ b/aisearch/v1/model.go @@ -164,6 +164,44 @@ const ( EndpointStatus_State_Starting EndpointStatus_State = "STARTING" ) +// Detailed lifecycle state of the index. +type IndexStatus_State string + +const ( + IndexStatus_State_Unspecified IndexStatus_State = "" + // Provisioning the serving endpoint. + IndexStatus_State_ProvisioningEndpoint IndexStatus_State = "PROVISIONING_ENDPOINT" + // Setting up a direct access index. + IndexStatus_State_ProvisioningIndex IndexStatus_State = "PROVISIONING_INDEX" + // Setting up the Spark cluster for the sync pipeline. + IndexStatus_State_ProvisioningPipelineResources IndexStatus_State = "PROVISIONING_PIPELINE_RESOURCES" + // Syncing the initial snapshot of a Delta Sync source table. + IndexStatus_State_ProvisioningInitialSnapshot IndexStatus_State = "PROVISIONING_INITIAL_SNAPSHOT" + // Default Online state, if a more detailed state cannot be determined. + IndexStatus_State_Online IndexStatus_State = "ONLINE" + // Index is ready and the sync pipeline is running in continuous mode. + IndexStatus_State_OnlineContinuousUpdate IndexStatus_State = "ONLINE_CONTINUOUS_UPDATE" + // Index is ready and the sync pipeline is running in triggered mode with an + // active update. + IndexStatus_State_OnlineTriggeredUpdate IndexStatus_State = "ONLINE_TRIGGERED_UPDATE" + // Index is ready and the sync pipeline is running in triggered mode with no + // active update. + IndexStatus_State_OnlineNoPendingUpdate IndexStatus_State = "ONLINE_NO_PENDING_UPDATE" + // Index is ready for direct access; no sync pipeline status details are needed. + IndexStatus_State_OnlineDirectAccess IndexStatus_State = "ONLINE_DIRECT_ACCESS" + // Index is offline. + IndexStatus_State_Offline IndexStatus_State = "OFFLINE" + // Index is offline because of a failure. + IndexStatus_State_OfflineFailed IndexStatus_State = "OFFLINE_FAILED" + // The latest pipeline update failed, but the index can still serve previously + // indexed data. + IndexStatus_State_OnlinePipelineFailed IndexStatus_State = "ONLINE_PIPELINE_FAILED" + // Index is online, but the pipeline is restarting and acquiring resources. + IndexStatus_State_OnlineUpdatingPipelineResources IndexStatus_State = "ONLINE_UPDATING_PIPELINE_RESOURCES" + // Downloading index data onto serving compute. + IndexStatus_State_Downloading IndexStatus_State = "DOWNLOADING" +) + // How the `model` field is interpreted. type RerankerConfig_ModelType string @@ -196,11 +234,10 @@ type CreateEndpointRequest struct { // desired configuration; `endpoint.name` is server-assigned from `parent` and // `endpoint_id`. Endpoint *Endpoint - // The user-supplied short name for the Endpoint, per AIP-133. The server - // composes the full `Endpoint.name` as `{parent}/endpoints/{endpoint_id}`. - // AIP-133 does not list `endpoint_id` as a fields-may-be-required entry, so we - // annotate it OPTIONAL on the wire; the server still rejects empty values with - // INVALID_PARAMETER_VALUE. + // Short name of the endpoint, which becomes the last segment of `name`. The + // server rejects a request without it. The name can have at most 50 characters, + // can contain only lowercase letters, digits, hyphens, periods, and + // underscores, and must start and end with a letter or digit. EndpointId *string } @@ -326,12 +363,14 @@ type EmbeddingVectorColumn struct { // and serves queries against them. Customers create, query, and delete // endpoints; the system manages provisioning, scaling, and health status.. type Endpoint struct { - // Name of the AI Search endpoint. Server-assigned full resource path - // (`workspaces/{workspace}/endpoints/{endpoint}`) on output. On create, the - // user-supplied short name is conveyed via `CreateEndpointRequest.endpoint_id`; - // the server composes the full `name` and returns it on the response. + // Full resource name of the endpoint, in the form + // `workspaces/{workspace_id}/endpoints/{endpoint_id}`. On create, the server + // composes it from `parent` and `endpoint_id` and ignores any value supplied in + // the request body. Name *string `fieldmask:"name"` - // Creator of the endpoint + // User name of the principal that created the endpoint. For a user, it is the + // user's email address; for any other principal, it is the principal's unique + // name. Creator *string `fieldmask:"creator"` // Time the endpoint was created. CreateTime *types.Time `fieldmask:"create_time"` @@ -339,31 +378,39 @@ type Endpoint struct { UpdateTime *types.Time `fieldmask:"update_time"` // Type of endpoint. Required on create and immutable thereafter. EndpointType EndpointType `fieldmask:"endpoint_type"` - // User who last updated the endpoint + // User name of the principal that last updated the endpoint, in the same format + // as `creator`. LastUpdatedUser *string `fieldmask:"last_updated_user"` - // Unique identifier of the endpoint + // System-generated UUID that uniquely identifies the endpoint. Id *string `fieldmask:"id"` - // Current status of the endpoint + // Current lifecycle state of the endpoint, with an optional message that + // explains it. EndpointStatus *EndpointStatus `fieldmask:"endpoint_status"` // Number of indexes on the endpoint IndexCount *int `fieldmask:"index_count"` - // The user-selected budget policy id for the endpoint. + // ID of the serverless budget policy to apply to the endpoint. The caller must + // have permission to use the policy. Set either this field or + // `usage_policy_id`, not both. Responses do not echo this field; they report + // the applied policy in `effective_budget_policy_id`. BudgetPolicyId *string `fieldmask:"budget_policy_id"` - // The budget policy id applied to the endpoint + // ID of the budget policy that is currently applied to the endpoint. EffectiveBudgetPolicyId *string `fieldmask:"effective_budget_policy_id"` - // The custom tags assigned to the endpoint + // Custom tags assigned to the endpoint. On update, the list replaces all of the + // endpoint's existing tags. CustomTags []CustomTag `fieldmask:"custom_tags"` // Throughput information for the endpoint ThroughputInfo *EndpointThroughputInfo `fieldmask:"throughput_info"` // Scaling information for the endpoint ScalingInfo *EndpointScalingInfo `fieldmask:"scaling_info"` - // The usage policy id applied to the endpoint. + // ID of the usage policy to apply to the endpoint. The server treats it as the + // same setting as `budget_policy_id`, so set only one of the two. UsagePolicyId *string `fieldmask:"usage_policy_id"` // The client-supplied desired number of replicas for the endpoint, applied at // create/update time. Mutually exclusive with `target_qps`. ReplicaCount *int `fieldmask:"replica_count"` // Target QPS for the endpoint. Mutually exclusive with `replica_count`. // Best-effort; the system does not guarantee this QPS will be achieved. + // Responses report the requested value in `scaling_info.requested_target_qps`. TargetQps *int `fieldmask:"target_qps"` } @@ -513,12 +560,16 @@ func (*Index_IndexSpec_DeltaSyncIndexSpec) isIndex_IndexSpec() {} // Lifecycle and health state of an AI Search index, along with human-readable // detail about that state and basic indexing progress.. type IndexStatus struct { + // Lifecycle and ingestion state. An online index can still have a failed + // update. + DetailedState IndexStatus_State // Human-readable detail about the index's current state. Message *string // Number of rows indexed. IndexedRowCount *int64 - // Whether the index is ready to serve queries. After CreateIndex, poll GetIndex - // until this is true; `message` describes the current state in the meantime. + // Whether the index can serve queries. After CreateIndex, poll GetIndex until + // true; message describes progress. This does not guarantee freshness or update + // success; check detailed_state for update failures. Ready *bool // Index API URL used to perform operations on the index. IndexUrl *string @@ -529,10 +580,9 @@ type ListEndpointsRequest struct { // The Workspace that owns this collection of endpoints. Format: // `workspaces/{workspace_id}` Parent *string - // Best-effort upper bound on the number of results to return. Honored as an - // upper bound by the shim: `page_size` only narrows the legacy backend's - // response, never widens it, so the practical cap is `min(page_size, - // legacy_fixed_page_size)`. + // Maximum number of endpoints to return in one page. The server applies its own + // page size limit, so a page can hold fewer endpoints than `page_size` even + // when more remain. PageSize *int // Page token from a previous response. If not provided, returns the first page. PageToken *string @@ -744,7 +794,9 @@ type UpdateEndpointRequest struct { // The Endpoint resource to update. `endpoint.name` carries the full resource // path. Endpoint *Endpoint - // The list of fields to update. + // Comma-separated list of `Endpoint` fields to update. The updatable fields are + // `budget_policy_id`, `usage_policy_id`, `custom_tags`, `replica_count`, and + // `target_qps`. The server rejects nested paths and unknown fields. UpdateMask *types.FieldMask[Endpoint] } diff --git a/aisearch/v1/wire.go b/aisearch/v1/wire.go index 9d723159..85962865 100644 --- a/aisearch/v1/wire.go +++ b/aisearch/v1/wire.go @@ -1254,10 +1254,11 @@ func (v *Index) UnmarshalJSON(data []byte) error { } type indexStatusWire struct { - Message *string `json:"message,omitempty"` - IndexedRowCount *wireInt64 `json:"indexed_row_count,omitempty"` - Ready *bool `json:"ready,omitempty"` - IndexUrl *string `json:"index_url,omitempty"` + DetailedState IndexStatus_State `json:"detailed_state,omitempty"` + Message *string `json:"message,omitempty"` + IndexedRowCount *wireInt64 `json:"indexed_row_count,omitempty"` + Ready *bool `json:"ready,omitempty"` + IndexUrl *string `json:"index_url,omitempty"` } func indexStatusToWire(v *IndexStatus) (*indexStatusWire, error) { @@ -1269,6 +1270,7 @@ func indexStatusToWire(v *IndexStatus) (*indexStatusWire, error) { return nil, fmt.Errorf("%s: %w", "IndexStatus.IndexedRowCount", err) } return &indexStatusWire{ + DetailedState: v.DetailedState, Message: v.Message, IndexedRowCount: indexedRowCountWireValue, Ready: v.Ready, @@ -1285,6 +1287,7 @@ func indexStatusFromWire(w *indexStatusWire) (*IndexStatus, error) { return nil, fmt.Errorf("%s: %w", "IndexStatus.IndexedRowCount", err) } return &IndexStatus{ + DetailedState: w.DetailedState, Message: w.Message, IndexedRowCount: indexedRowCountPublicValue, Ready: w.Ready, diff --git a/budgetpolicy/CHANGELOG.md b/budgetpolicy/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/budgetpolicy/CHANGELOG.md +++ b/budgetpolicy/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/budgetpolicy/internal/version.go b/budgetpolicy/internal/version.go index 543f0f96..6ac1eab5 100644 --- a/budgetpolicy/internal/version.go +++ b/budgetpolicy/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-budgetpolicy" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/budgetpolicy/v1/client.go b/budgetpolicy/v1/client.go index 828d0be4..6acd53f7 100644 --- a/budgetpolicy/v1/client.go +++ b/budgetpolicy/v1/client.go @@ -78,7 +78,8 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Creates a new policy. +// Creates a new usage policy. The creator is granted the manager role on the +// new policy. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) CreateBudgetPolicy(ctx context.Context, req CreateBudgetPolicyRequest, opts ...call.Option) (*BudgetPolicy, error) { wireReq, err := createBudgetPolicyRequestToWire(&req) @@ -153,7 +154,7 @@ func (c *internalClient) CreateBudgetPolicy(ctx context.Context, req CreateBudge return resp, nil } -// Deletes a policy +// Deletes a usage policy. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) DeleteBudgetPolicy(ctx context.Context, req DeleteBudgetPolicyRequest, opts ...call.Option) error { @@ -210,7 +211,7 @@ func (c *internalClient) DeleteBudgetPolicy(ctx context.Context, req DeleteBudge return nil } -// Retrieves a policy by it's ID. +// Retrieves a usage policy by its ID. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) GetBudgetPolicy(ctx context.Context, req GetBudgetPolicyRequest, opts ...call.Option) (*BudgetPolicy, error) { @@ -276,8 +277,9 @@ func (c *internalClient) GetBudgetPolicy(ctx context.Context, req GetBudgetPolic return resp, nil } -// Lists all policies. Policies are returned in the alphabetically ascending -// order of their names. +// Lists the usage policies that the caller has permission to view. Policies are +// sorted by name in ascending order unless `sort_spec` requests a different +// order. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) ListBudgetPolicies(ctx context.Context, req ListBudgetPoliciesRequest, opts ...call.Option) (*ListBudgetPoliciesResponse, error) { wireReq, err := listBudgetPoliciesRequestToWire(&req) @@ -396,7 +398,8 @@ func (c *internalClient) ListBudgetPoliciesIter(ctx context.Context, req ListBud } } -// Updates a policy +// Updates a usage policy. Changes apply only to serverless usage that starts +// after the update, not to workloads that are already running. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) UpdateBudgetPolicy(ctx context.Context, req UpdateBudgetPolicyRequest, opts ...call.Option) (*BudgetPolicy, error) { wireReq, err := updateBudgetPolicyRequestToWire(&req) diff --git a/budgetpolicy/v1/model.go b/budgetpolicy/v1/model.go index 91fa701a..604e0cdc 100644 --- a/budgetpolicy/v1/model.go +++ b/budgetpolicy/v1/model.go @@ -10,7 +10,10 @@ const ( SortSpec_Field_PolicyName SortSpec_Field = "POLICY_NAME" ) -// Contains the BudgetPolicy details.. +// A serverless usage policy is a set of tags that are applied to the serverless +// usage of the workloads that use the policy. Each usage record in the billable +// usage system table carries the policy ID and these tags, so that serverless +// usage can be attributed to the policy.. type BudgetPolicy struct { // The Id of the policy. This field is generated by and globally // unique. @@ -36,9 +39,9 @@ type CreateBudgetPolicyRequest struct { RequestId *string // The account Id of the customer AccountId *string - // The policy to create. `policy_id` needs to be empty as it will be generated - // `policy_name` must be provided, custom_tags may need to be provided depending - // on the cloud provider. All other fields are optional. + // The policy to create. `policy_name` must be provided, and `custom_tags` may + // be required depending on the cloud provider. Leave `policy_id` empty because + // Databricks generates it. All other fields are optional. Policy *BudgetPolicy } @@ -62,11 +65,11 @@ type DeleteBudgetPolicyRequest struct { // Structured representation of a filter to be applied to a list of policies. // All specified filters will be applied in conjunction.. type Filter struct { - // The partial name of policies to be filtered on. If unspecified, all policies - // will be returned. + // Only policies whose name contains this string are returned. Matching is + // case-insensitive. If unspecified, all policies are returned. PolicyName *string - // The policy creator user id to be filtered on. If unspecified, all policies - // will be returned. + // Deprecated: This field is ignored. Creator filtering will be removed in a + // future version. CreatorUserId *int64 // Deprecated: Do not use this field in new integrations. Creator filtering will // be removed in a future version. The policy creator user name to be filtered @@ -92,17 +95,17 @@ type ListBudgetPoliciesRequest struct { // budget policies will be returned. The maximum value is 1000; values above // 1000 will be coerced to 1000. PageSize *int - // A page token, received from a previous `ListServerlessPolicies` call. Provide + // A page token, received from a previous `ListBudgetPolicies` call. Provide // this to retrieve the subsequent page. If unspecified, the first page will be // returned. // - // When paginating, all other parameters provided to - // `ListServerlessPoliciesRequest` must match the call that provided the page - // token. + // When paginating, all other parameters provided to `ListBudgetPolicies` must + // match the call that provided the page token. PageToken *string // A filter to apply to the list of policies. FilterBy *Filter - // The sort specification. + // How to sort the returned policies. If unspecified, policies are sorted by + // name in ascending order. SortSpec *SortSpec // The account Id of the customer AccountId *string @@ -120,7 +123,7 @@ type ListBudgetPoliciesResponse struct { } type SortSpec struct { - // The filed to sort by + // The field to sort by. If unspecified, policies are sorted by name. Field SortSpec_Field // Whether to sort in descending order. Descending *bool @@ -134,7 +137,6 @@ type UpdateBudgetPolicyRequest struct { Policy *BudgetPolicy // The account Id of the customer AccountId *string - // DEPRECATED. This is redundant field as LimitConfig is part of the - // BudgetPolicy + // Deprecated: This field is ignored. LimitConfig *LimitConfig } diff --git a/cleanrooms/CHANGELOG.md b/cleanrooms/CHANGELOG.md index f9c81d37..07b206a9 100644 --- a/cleanrooms/CHANGELOG.md +++ b/cleanrooms/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/cleanrooms/internal/version.go b/cleanrooms/internal/version.go index 25adf621..e16d771f 100644 --- a/cleanrooms/internal/version.go +++ b/cleanrooms/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-cleanrooms" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/cleanrooms/v1/client.go b/cleanrooms/v1/client.go index a1c0efd7..656378d1 100644 --- a/cleanrooms/v1/client.go +++ b/cleanrooms/v1/client.go @@ -80,10 +80,10 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { } // Create a new clean room with the specified collaborators. This method is -// asynchronous; the returned name field inside the clean_room field can be used -// to poll the clean room status, using the [cleanrooms/get] method. When this -// method returns, the clean room will be in a PROVISIONING state, with only -// name, owner, comment, created_at and status populated. The clean room will be +// asynchronous; the name field of the returned clean room can be used to poll +// the clean room status, using the [cleanrooms/get] method. When this method +// returns, the clean room will be in a PROVISIONING state, with only name, +// owner, comment, created_at and status populated. The clean room will be // usable once it enters an ACTIVE state. // // The caller must be a metastore admin or have the **CREATE_CLEAN_ROOM** @@ -156,10 +156,10 @@ func (c *internalClient) createCleanRoomBase(ctx context.Context, req CreateClea } // Create a new clean room with the specified collaborators. This method is -// asynchronous; the returned name field inside the clean_room field can be used -// to poll the clean room status, using the [cleanrooms/get] method. When this -// method returns, the clean room will be in a PROVISIONING state, with only -// name, owner, comment, created_at and status populated. The clean room will be +// asynchronous; the name field of the returned clean room can be used to poll +// the clean room status, using the [cleanrooms/get] method. When this method +// returns, the clean room will be in a PROVISIONING state, with only name, +// owner, comment, created_at and status populated. The clean room will be // usable once it enters an ACTIVE state. // // The caller must be a metastore admin or have the **CREATE_CLEAN_ROOM** @@ -477,7 +477,10 @@ func (c *internalClient) CreateCleanRoomAutoApprovalRule(ctx context.Context, re return resp, nil } -// Create the output catalog of the clean room. +// Create the output catalog of the clean room. The output catalog is a catalog +// in the caller's metastore where the output tables of the clean room runs that +// the caller triggers are made available. Each collaborator can create one +// output catalog per clean room, and the clean room must be ACTIVE. func (c *internalClient) CreateCleanRoomOutputCatalog(ctx context.Context, req CreateCleanRoomOutputCatalogRequest, opts ...call.Option) (*CreateCleanRoomOutputCatalogResponse, error) { wireReq, err := createCleanRoomOutputCatalogRequestToWire(&req) if err != nil { diff --git a/cleanrooms/v1/model.go b/cleanrooms/v1/model.go index ef5840ab..5e19833f 100644 --- a/cleanrooms/v1/model.go +++ b/cleanrooms/v1/model.go @@ -278,6 +278,12 @@ const ( PartitionSpecification_Partition_PartitionValue_PartitionValueOp_Like PartitionSpecification_Partition_PartitionValue_PartitionValueOp = "LIKE" ) +// A clean room is a secure, Databricks-managed environment where collaborators +// from one or more Unity Catalog metastores share tables, views, volumes and +// notebooks and run approved analyses without seeing each other's data. Each +// collaborator sees the clean room as a securable in their own metastore, while +// the analyses run in a central clean room in the cloud and region chosen at +// creation.. type CleanRoom struct { // The name of the clean room. It should follow [UC securable naming // requirements]. @@ -289,8 +295,11 @@ type CleanRoom struct { // not be filled in the ListCleanRooms call. RemoteDetailedInfo *CleanRoomRemoteDetail // This is the username of the owner of the local clean room - // securable for permission management. - Owner *string + // securable for permission management. When creating a clean room, leave it + // empty or set it to the caller. + Owner *string + // Free-form description of the clean room. Comments are local to the caller's + // metastore and are not shared with the other collaborators. Comment *string // When the clean room was created, in epoch milliseconds. CreatedAt *int64 @@ -673,6 +682,7 @@ type CleanRoomNotebookTaskRun struct { } type CleanRoomOutputCatalog struct { + // Status of the output catalog for the calling collaborator. Status CleanRoomOutputCatalog_OutputCatalogStatus // The name of the output catalog in UC. It should follow [UC securable naming // requirements]. The field will always exist if status is CREATED. @@ -689,7 +699,7 @@ type CleanRoomRemoteDetail struct { CloudVendor *string // Region of the central clean room. Region *string - // Collaborators in the central clean room. There should one and only one + // Collaborators in the central clean room. There should be one and only one // collaborator in the list that satisfies the owner condition: // // 1. It has the creator's global_metastore_id (determined by caller of @@ -699,8 +709,14 @@ type CleanRoomRemoteDetail struct { Collaborators []CleanRoomCollaborator // Collaborator who creates the clean room. Creator *CleanRoomCollaborator - // Egress network policy to apply to the central clean room workspace. - EgressNetworkPolicy *EgressNetworkPolicy + // Egress network policy to apply to the central clean room workspace. Only + // internet_access.restriction_mode (FULL_ACCESS or RESTRICTED_ACCESS) and + // internet_access.allowed_internet_destinations can be set. When omitted, + // internet access is restricted with no allowed destinations. + EgressNetworkPolicy *EgressNetworkPolicy + // Compliance security profile of the central clean room. It is taken from the + // creator's workspace when the clean room is created; a collaborator whose + // workspace profile differs sees access_restricted set to CSP_MISMATCH. ComplianceSecurityProfile *ComplianceSecurityProfile // Whether to enable shared output for the central clean room. When enabled, // clean room task runs can write to the run-scoped shared output schema which @@ -785,6 +801,9 @@ type ColumnInfo struct { Name *string // Full data type specification as SQL/catalogString text. TypeText *string + // The name of the column's data type, for example STRING, DECIMAL or STRUCT. + // The complete type, including parameters such as the precision of a decimal or + // the element type of an array, is in type_text. TypeName ColumnTypeName // Ordinal position of column (starting at position 0). Position *int @@ -802,7 +821,11 @@ type ColumnInfo struct { Nullable *bool // Partition index for column. PartitionIndex *int - Mask *ColumnMask + // The column mask function applied to this column, if any. A column mask is a + // SQL user-defined function that transforms the values a query returns for the + // column. See https://docs.databricks.com/aws/en/tables/row-and-column-filters + // for details. + Mask *ColumnMask } type ColumnMask struct { @@ -904,6 +927,7 @@ type CreateCleanRoomOutputCatalogRequest struct { } type CreateCleanRoomOutputCatalogResponse struct { + // The output catalog that was created. OutputCatalog *CleanRoomOutputCatalog } @@ -1007,6 +1031,7 @@ type GetCleanRoomAutoApprovalRuleRequest struct { } type GetCleanRoomRequest struct { + // Name of the clean room. Name *string } @@ -1209,6 +1234,9 @@ type UpdateCleanRoomAutoApprovalRuleRequest struct { type UpdateCleanRoomRequest struct { // Name of the clean room. - Name *string + Name *string + // Clean room fields to update. Only mutable fields such as owner, comment and + // enable_shared_output can be changed; remote_detailed_info, status, created_at + // and updated_at must be omitted. CleanRoom *CleanRoom } diff --git a/clusterlibraries/CHANGELOG.md b/clusterlibraries/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/clusterlibraries/CHANGELOG.md +++ b/clusterlibraries/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/clusterlibraries/internal/version.go b/clusterlibraries/internal/version.go index f617f7cb..6d09524e 100644 --- a/clusterlibraries/internal/version.go +++ b/clusterlibraries/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-clusterlibraries" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/clusterlibraries/v2/client.go b/clusterlibraries/v2/client.go index e212457b..47cabd27 100644 --- a/clusterlibraries/v2/client.go +++ b/clusterlibraries/v2/client.go @@ -76,8 +76,11 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Get the status of all libraries on all clusters. A status is returned for all -// libraries installed on this cluster via the API or the libraries UI. +// Get the status of all libraries on all clusters in the workspace. A status is +// returned for all libraries installed on each cluster via the API or the +// libraries UI. If the workspace has more library statuses than the response +// limit allows, the request fails; call the per-cluster status method for each +// cluster instead. func (c *internalClient) AllClusterStatuses(ctx context.Context, req ListAllClusterLibraryStatusesRequest, opts ...call.Option) (*ListAllClusterLibraryStatusesResponse, error) { headers := http.Header{} @@ -138,8 +141,8 @@ func (c *internalClient) AllClusterStatuses(ctx context.Context, req ListAllClus // order of returned libraries is as follows: 1. Libraries set to be installed // on this cluster, in the order that the libraries were added to the cluster, // are returned first. 2. Libraries that were previously requested to be -// installed on this cluster or, but are now marked for removal, in no -// particular order, are returned last. +// installed on this cluster but are now marked for removal, in no particular +// order, are returned last. func (c *internalClient) ClusterStatus(ctx context.Context, req ClusterStatusRequest, opts ...call.Option) (*ClusterLibraryStatuses, error) { wireReq, err := clusterStatusRequestToWire(&req) if err != nil { diff --git a/clusterlibraries/v2/model.go b/clusterlibraries/v2/model.go index a6f6f2cc..5558399b 100644 --- a/clusterlibraries/v2/model.go +++ b/clusterlibraries/v2/model.go @@ -53,6 +53,8 @@ type InstallLibrariesRequest struct { type InstallLibrariesResponse struct { } +// A library to install on a cluster. Exactly one of the library-type fields +// must be set.. type Library struct { Lib isLibrary_Lib } @@ -76,8 +78,9 @@ type Library_Lib_Jar struct { func (*Library_Lib_Jar) isLibrary_Lib() {} // Library_Lib_Egg selects Egg for Library.Lib. -// Deprecated. URI of the egg library to install. Installing Python egg files is -// deprecated and is not supported in Databricks Runtime 14.0 and above. +// Deprecated: Installing Python egg files is not supported in Databricks +// Runtime 14.0 and above. Use a wheel (`whl`) or a PyPI package (`pypi`) +// instead. The value is the URI of the egg file to install. type Library_Lib_Egg struct { Egg string } @@ -85,7 +88,7 @@ type Library_Lib_Egg struct { func (*Library_Lib_Egg) isLibrary_Lib() {} // Library_Lib_Pypi selects Pypi for Library.Lib. -// Specification of a PyPi library to be installed. For example: `{ "package": +// Specification of a PyPI library to be installed. For example: `{ "package": // "simplejson" }` type Library_Lib_Pypi struct { Pypi PythonPyPiLibrary @@ -103,7 +106,7 @@ type Library_Lib_Maven struct { func (*Library_Lib_Maven) isLibrary_Lib() {} // Library_Lib_Cran selects Cran for Library.Lib. -// Specification of a CRAN library to be installed as part of the library +// Specification of a CRAN library to be installed. type Library_Lib_Cran struct { Cran RCranLibrary } @@ -128,7 +131,8 @@ func (*Library_Lib_Whl) isLibrary_Lib() {} // URI of the requirements.txt file to install. Only Workspace paths and Unity // Catalog Volumes paths are supported. For example: `{ "requirements": // "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : -// "/Volumes/path/to/requirements.txt" }` +// "/Volumes/path/to/requirements.txt" }`. Installing a requirements file +// requires Databricks Runtime 15.0 or above. type Library_Lib_Requirements struct { Requirements string } @@ -137,9 +141,13 @@ func (*Library_Lib_Requirements) isLibrary_Lib() {} // The status of the library on a specific cluster.. type LibraryFullStatus struct { - // Unique identifier for the library. + // The library whose status is reported. Library *Library - // Status of installing the library on the cluster. + // Status of installing the library on the cluster. The status starts at PENDING + // and ends at INSTALLED, FAILED or SKIPPED. Intermediate states are not always + // reported. A library stays UNINSTALL_ON_RESTART after an uninstall request + // until the cluster restarts. While the cluster is not running, libraries are + // reported as PENDING, and libraries marked for uninstall are omitted. Status LibraryInstallStatus // All the info and warning messages that have occurred so far for this library. Messages []string @@ -162,7 +170,7 @@ type MavenLibrary struct { // Maven repo to install the Maven package from. If omitted, both Maven Central // Repository and Spark Packages are searched. Repo *string - // List of dependences to exclude. For example: `["slf4j:slf4j", + // List of dependencies to exclude. For example: `["slf4j:slf4j", // "*:hadoop-client"]`. // // Maven dependency exclusions: @@ -171,7 +179,7 @@ type MavenLibrary struct { } type PythonPyPiLibrary struct { - // The name of the pypi package to install. An optional exact version + // The name of the PyPI package to install. An optional exact version // specification is also supported. Examples: "simplejson" and // "simplejson==3.8.0". Package *string diff --git a/clusterpolicies/CHANGELOG.md b/clusterpolicies/CHANGELOG.md index 6dd51a29..5ad662cc 100644 --- a/clusterpolicies/CHANGELOG.md +++ b/clusterpolicies/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.9.0 (2026-11-17) + ## Release v0.8.0 (2026-11-16) ## Release v0.7.0 (2026-11-12) diff --git a/clusterpolicies/internal/version.go b/clusterpolicies/internal/version.go index bf2fc1c5..62cad2bc 100644 --- a/clusterpolicies/internal/version.go +++ b/clusterpolicies/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-clusterpolicies" -const Version = "0.8.0" +const Version = "0.9.0" diff --git a/clusterpolicies/v2/model.go b/clusterpolicies/v2/model.go index 3f86db8b..b3ecb4a7 100644 --- a/clusterpolicies/v2/model.go +++ b/clusterpolicies/v2/model.go @@ -190,6 +190,8 @@ type LakebaseConstraints struct { MaxIdleMinutesBeforeScaleToZero *int64 } +// A library to install on a cluster. Exactly one of the library-type fields +// must be set.. type Library struct { Lib isLibrary_Lib } @@ -213,8 +215,9 @@ type Library_Lib_Jar struct { func (*Library_Lib_Jar) isLibrary_Lib() {} // Library_Lib_Egg selects Egg for Library.Lib. -// Deprecated. URI of the egg library to install. Installing Python egg files is -// deprecated and is not supported in Databricks Runtime 14.0 and above. +// Deprecated: Installing Python egg files is not supported in Databricks +// Runtime 14.0 and above. Use a wheel (`whl`) or a PyPI package (`pypi`) +// instead. The value is the URI of the egg file to install. type Library_Lib_Egg struct { Egg string } @@ -222,7 +225,7 @@ type Library_Lib_Egg struct { func (*Library_Lib_Egg) isLibrary_Lib() {} // Library_Lib_Pypi selects Pypi for Library.Lib. -// Specification of a PyPi library to be installed. For example: `{ "package": +// Specification of a PyPI library to be installed. For example: `{ "package": // "simplejson" }` type Library_Lib_Pypi struct { Pypi PythonPyPiLibrary @@ -240,7 +243,7 @@ type Library_Lib_Maven struct { func (*Library_Lib_Maven) isLibrary_Lib() {} // Library_Lib_Cran selects Cran for Library.Lib. -// Specification of a CRAN library to be installed as part of the library +// Specification of a CRAN library to be installed. type Library_Lib_Cran struct { Cran RCranLibrary } @@ -265,7 +268,8 @@ func (*Library_Lib_Whl) isLibrary_Lib() {} // URI of the requirements.txt file to install. Only Workspace paths and Unity // Catalog Volumes paths are supported. For example: `{ "requirements": // "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : -// "/Volumes/path/to/requirements.txt" }` +// "/Volumes/path/to/requirements.txt" }`. Installing a requirements file +// requires Databricks Runtime 15.0 or above. type Library_Lib_Requirements struct { Requirements string } @@ -327,7 +331,7 @@ type MavenLibrary struct { // Maven repo to install the Maven package from. If omitted, both Maven Central // Repository and Spark Packages are searched. Repo *string - // List of dependences to exclude. For example: `["slf4j:slf4j", + // List of dependencies to exclude. For example: `["slf4j:slf4j", // "*:hadoop-client"]`. // // Maven dependency exclusions: @@ -384,7 +388,7 @@ type Policy struct { } type PythonPyPiLibrary struct { - // The name of the pypi package to install. An optional exact version + // The name of the PyPI package to install. An optional exact version // specification is also supported. Examples: "simplejson" and // "simplejson==3.8.0". Package *string diff --git a/clusters/CHANGELOG.md b/clusters/CHANGELOG.md index f3d3d44e..fa0ebd0d 100644 --- a/clusters/CHANGELOG.md +++ b/clusters/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-12) ### New Features and Improvements diff --git a/clusters/internal/version.go b/clusters/internal/version.go index dfc35384..c6f25e18 100644 --- a/clusters/internal/version.go +++ b/clusters/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-clusters" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/clusters/v2/model.go b/clusters/v2/model.go index e2355325..a1c6363c 100644 --- a/clusters/v2/model.go +++ b/clusters/v2/model.go @@ -979,8 +979,13 @@ type AwsAttributes struct { // remainder will be placed on `availability` instances. Note that this value // does not affect cluster size and cannot currently be mutated over the // lifetime of a cluster. - FirstOnDemand *int `fieldmask:"first_on_demand"` - Availability AwsAvailability `fieldmask:"availability"` + FirstOnDemand *int `fieldmask:"first_on_demand"` + // Availability type used for all subsequent nodes past the `first_on_demand` + // ones. + // + // Note: If `first_on_demand` is zero, this availability type will be used for + // the entire cluster. + Availability AwsAvailability `fieldmask:"availability"` // Identifier for the availability zone/datacenter in which the cluster resides. // This string will be of a form like "us-west-2a". The provided availability // zone must be in the same region as the deployment. For example, @@ -1043,7 +1048,9 @@ type AwsAttributes struct { // Attributes set during cluster creation which are related to Microsoft Azure.. type AzureAttributes struct { - // Defines values necessary to configure and run Azure Log Analytics agent + // Defines values necessary to configure and run Azure Log Analytics agent. Both + // `log_analytics_workspace_id` and `log_analytics_primary_key` must be set, or + // neither. LogAnalyticsInfo *LogAnalyticsInfo `fieldmask:"log_analytics_info"` // The first `first_on_demand` nodes of the cluster will be placed on on-demand // instances. This value should be greater than 0, to make sure the cluster @@ -1097,6 +1104,7 @@ type CancelPendingClusterEnforcementResponse struct { } type ChangeClusterOwnerRequest struct { + // The ID of the cluster whose owner is changed. ClusterId *string // New owner of the cluster_id after this RPC. OwnerUsername *string @@ -1206,7 +1214,10 @@ type ClusterAttributes struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -1220,8 +1231,39 @@ type ClusterAttributes struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -1231,7 +1273,22 @@ type ClusterAttributes struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -1268,12 +1325,16 @@ type ClusterCompliance struct { } type ClusterEvent struct { + // The ID of the cluster the event belongs to. ClusterId *string // The timestamp when the event occurred, stored as the number of milliseconds // since the Unix epoch. If not provided, this will be assigned by the Timeline // service. - Timestamp *int64 - Type ClusterEventType_ClusterEventType + Timestamp *int64 + // The type of event, which determines which `details` fields are populated. + Type ClusterEventType_ClusterEventType + // Details about the event. Which fields are populated depends on the event + // `type`. Details *EventDetails DataPlaneEventDetails *DataPlaneEventDetails } @@ -1332,8 +1393,8 @@ type ClusterInfo struct { // driver restarts. The pair `(cluster_id, spark_context_id)` is a globally // unique identifier over all Spark contexts. SparkContextId *int64 - // Port on which Spark JDBC server is listening, in the driver nod. No service - // will be listeningon on this port in executor nodes. + // Port on which Spark JDBC server is listening in the driver node. No service + // will be listening on this port in executor nodes. JdbcPort *int // Cluster name requested by the user. This doesn't have to be unique. If not // specified at creation, the cluster name will be an empty string. For job @@ -1423,7 +1484,10 @@ type ClusterInfo struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -1437,8 +1501,39 @@ type ClusterInfo struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -1448,7 +1543,22 @@ type ClusterInfo struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -1475,7 +1585,7 @@ type ClusterInfo struct { // Time when the cluster driver last lost its state (due to a restart or driver // failure). LastStateLossTime *int64 - // the timestamp that the cluster was started/restarted + // Time (in epoch milliseconds) when the cluster was last started or restarted. LastRestartedTime *int64 Size isClusterInfo_Size } @@ -1605,7 +1715,10 @@ type ClusterInfo_ComputeSpec struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -1619,8 +1732,39 @@ type ClusterInfo_ComputeSpec struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -1630,7 +1774,22 @@ type ClusterInfo_ComputeSpec struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -1862,7 +2021,10 @@ type CreateClusterRequest struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -1876,8 +2038,39 @@ type CreateClusterRequest struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -1887,7 +2080,22 @@ type CreateClusterRequest struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -1964,9 +2172,9 @@ type DeleteClusterResponse struct { } type DockerBasicAuth struct { - // Name of the user + // User name for authenticating to the Docker registry that hosts the image. Username *string `fieldmask:"username"` - // Password of the user + // Password for authenticating to the Docker registry that hosts the image. Password *string `fieldmask:"password"` } @@ -2089,7 +2297,10 @@ type EditClusterRequest struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -2103,8 +2314,39 @@ type EditClusterRequest struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -2114,7 +2356,22 @@ type EditClusterRequest struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -2290,7 +2547,10 @@ type EnforcePolicyComplianceForClusterResponse_ClusterSettings struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -2304,8 +2564,39 @@ type EnforcePolicyComplianceForClusterResponse_ClusterSettings struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -2315,7 +2606,22 @@ type EnforcePolicyComplianceForClusterResponse_ClusterSettings struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -2412,10 +2718,14 @@ type EventDetails struct { // Previous disk size in bytes PreviousDiskSize *int64 // Current disk size in bytes - DiskSize *int64 + DiskSize *int64 + // Free disk space in bytes. Included in disk expansion events such as + // `EXPANDED_DISK` and `DID_NOT_EXPAND_DISK`. FreeSpace *int64 // Instance Id where the event originated from - InstanceId *string + InstanceId *string + // The reason the disk was not expanded. Included in `DID_NOT_EXPAND_DISK` and + // `FAILED_TO_EXPAND_DISK` events. DidNotExpandReason *string // More details about the change in driver's state DriverStateMessage *string @@ -2492,6 +2802,7 @@ type GetClusterRequest struct { } type GetEventsResponse struct { + // The events matching the request, in the requested `order`. Events []ClusterEvent // Deprecated: use next_page_token or prev_page_token instead. // @@ -2830,8 +3141,12 @@ type LocalFileInfo struct { } type LogAnalyticsInfo struct { + // ID of the Azure Log Analytics workspace that the Log Analytics agent on the + // cluster's nodes reports to. LogAnalyticsWorkspaceId *string `fieldmask:"log_analytics_workspace_id"` - LogAnalyticsPrimaryKey *string `fieldmask:"log_analytics_primary_key"` + // Primary key of the Azure Log Analytics workspace. The agent uses it to + // authenticate when reporting to the workspace. + LogAnalyticsPrimaryKey *string `fieldmask:"log_analytics_primary_key"` } // The log delivery status. @@ -3004,7 +3319,10 @@ type ResizeClusterResponse struct { type RestartClusterRequest struct { // The cluster to be started. - ClusterId *string + ClusterId *string + // User name recorded in the `RESTARTING` cluster event as the user who + // restarted the cluster. If omitted, the name of the user making the request is + // recorded. RestartUser *string } @@ -3211,7 +3529,10 @@ type UpdateClusterRequest_UpdateClusterResource struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo `fieldmask:"init_scripts"` - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage `fieldmask:"docker_image"` // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string `fieldmask:"instance_pool_id"` @@ -3224,9 +3545,40 @@ type UpdateClusterRequest_UpdateClusterResource struct { // The optional ID of the instance pool for the driver of the cluster belongs. // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. - DriverInstancePoolId *string `fieldmask:"driver_instance_pool_id"` - WorkloadType *WorkloadType `fieldmask:"workload_type"` - DataSecurityMode DataSecurityMode `fieldmask:"data_security_mode"` + DriverInstancePoolId *string `fieldmask:"driver_instance_pool_id"` + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType `fieldmask:"workload_type"` + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode `fieldmask:"data_security_mode"` // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -3236,7 +3588,22 @@ type UpdateClusterRequest_UpdateClusterResource struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine `fieldmask:"runtime_engine"` - Kind ComputeKind `fieldmask:"kind"` + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind `fieldmask:"kind"` // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), diff --git a/disasterrecovery/CHANGELOG.md b/disasterrecovery/CHANGELOG.md index f3d3d44e..fa0ebd0d 100644 --- a/disasterrecovery/CHANGELOG.md +++ b/disasterrecovery/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-12) ### New Features and Improvements diff --git a/disasterrecovery/internal/version.go b/disasterrecovery/internal/version.go index f745aff8..99f0e81d 100644 --- a/disasterrecovery/internal/version.go +++ b/disasterrecovery/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-disasterrecovery" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/disasterrecovery/v1/client.go b/disasterrecovery/v1/client.go index 1ade4ee2..37bbf5b3 100644 --- a/disasterrecovery/v1/client.go +++ b/disasterrecovery/v1/client.go @@ -78,7 +78,10 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Create a new failover group. +// Create a failover group. The group starts in the CREATING state and moves to +// INITIAL_REPLICATION while the first replication cycle copies all in-scope +// data to the secondary region; failover becomes available once the group is +// ACTIVE. Set validate_only to check the request without creating anything. func (c *internalClient) CreateFailoverGroup(ctx context.Context, req CreateFailoverGroupRequest, opts ...call.Option) (*FailoverGroup, error) { wireReq, err := createFailoverGroupRequestToWire(&req) if err != nil { @@ -236,7 +239,8 @@ func (c *internalClient) CreateStableUrl(ctx context.Context, req CreateStableUr return resp, nil } -// Delete a failover group. +// Delete a failover group. Deleting stops replication for the group; stable +// URLs linked to it are detached but not deleted. func (c *internalClient) DeleteFailoverGroup(ctx context.Context, req DeleteFailoverGroupRequest, opts ...call.Option) error { wireReq, err := deleteFailoverGroupRequestToWire(&req) if err != nil { @@ -349,7 +353,15 @@ func (c *internalClient) DeleteStableUrl(ctx context.Context, req DeleteStableUr return nil } -// Initiate a failover to a new primary region. +// Initiate a failover that makes target_primary_region the primary region. +// Stable URLs linked to the group are pointed at the new primary workspace, +// replication is reversed so that the former primary becomes the secondary, and +// the group moves through FAILING_OVER to INITIAL_REPLICATION until it has a +// replication point in the new direction. The group must be ACTIVE; repeating +// the request while it is FAILING_OVER is accepted. To fail back, repeat the +// request with the former primary region as the target once the group is ACTIVE +// again. In an unplanned failover, data written after the replication point can +// be lost. func (c *internalClient) FailoverFailoverGroup(ctx context.Context, req FailoverFailoverGroupRequest, opts ...call.Option) (*FailoverGroup, error) { wireReq, err := failoverFailoverGroupRequestToWire(&req) if err != nil { @@ -422,7 +434,7 @@ func (c *internalClient) FailoverFailoverGroup(ctx context.Context, req Failover return resp, nil } -// Get a failover group. +// Get a failover group, including its current state and replication point. func (c *internalClient) GetFailoverGroup(ctx context.Context, req GetFailoverGroupRequest, opts ...call.Option) (*FailoverGroup, error) { headers := http.Header{} @@ -546,10 +558,11 @@ func (c *internalClient) GetStableUrl(ctx context.Context, req GetStableUrlReque return resp, nil } -// List failover groups. +// List the failover groups in an account. // -// List entries are abbreviated: `state` and `replication_point` are not -// populated. Call GetFailoverGroup to retrieve the full resource. +// List entries are abbreviated: `state`, `replication_point` and +// `unity_catalog_assets.location_mappings` are not populated. Call +// GetFailoverGroup to retrieve the full resource. func (c *internalClient) ListFailoverGroups(ctx context.Context, req ListFailoverGroupsRequest, opts ...call.Option) (*ListFailoverGroupsResponse, error) { wireReq, err := listFailoverGroupsRequestToWire(&req) if err != nil { @@ -777,7 +790,12 @@ func (c *internalClient) ListStableUrlsIter(ctx context.Context, req ListStableU } } -// Update a failover group. +// Update a failover group. Only the fields named in update_mask change. You can +// turn workspace asset replication on or off, attach or detach a stable URL, +// and change the Unity Catalog catalogs and location mappings; the regions, the +// workspace set name and the workspace IDs cannot be changed. Updates are +// rejected while a create, failover or delete is in progress, and after a +// create or delete has failed. func (c *internalClient) UpdateFailoverGroup(ctx context.Context, req UpdateFailoverGroupRequest, opts ...call.Option) (*FailoverGroup, error) { wireReq, err := updateFailoverGroupRequestToWire(&req) if err != nil { diff --git a/disasterrecovery/v1/model.go b/disasterrecovery/v1/model.go index 16b5e66f..247e9671 100644 --- a/disasterrecovery/v1/model.go +++ b/disasterrecovery/v1/model.go @@ -92,12 +92,17 @@ type FailoverFailoverGroupRequest struct { // Opaque version string for optimistic locking. If provided, must match the // current etag. If omitted, the failover proceeds regardless of current state. Etag *string - // The type of failover to perform. + // The type of failover to perform. FORCED is the only supported type. FailoverType FailoverFailoverGroupRequest_FailoverType } -// A failover group manages disaster recovery across workspace sets, -// coordinating Unity Catalog and workspace assets replication.. +// A failover group manages Databricks Managed Disaster Recovery for a pair of +// workspaces in two regions. It continuously replicates the selected Unity +// Catalog catalogs and workspace assets from the primary region to the +// secondary region and lets you fail over to the secondary region; at least one +// of the two replication types must be enabled. See +// https://docs.databricks.com/aws/en/admin/managed-disaster-recovery for the +// product documentation.. type FailoverGroup struct { // Fully qualified resource name in the format // accounts/{account_id}/failover-groups/{failover_group_id}. @@ -105,13 +110,22 @@ type FailoverGroup struct { // Current effective primary region. Replication flows FROM workspaces in this // region. Changes after a successful failover. EffectivePrimaryRegion *string `fieldmask:"effective_primary_region"` - // List of all regions participating in this failover group. + // The two cloud regions that participate in this failover group, the primary + // and the secondary. Each workspace set contains one workspace in each region. Regions []string `fieldmask:"regions"` - // Workspace sets, each containing workspaces that replicate to each other. + // Workspace sets, each containing workspaces that replicate to each other. A + // failover group currently supports exactly one workspace set. WorkspaceSets []WorkspaceSet `fieldmask:"workspace_sets"` - // Unity Catalog replication configuration. + // Unity Catalog replication configuration: the catalogs to replicate and the + // storage location mappings for their data. Omit it to replicate workspace + // assets only. UnityCatalogAssets *UcReplicationConfig `fieldmask:"unity_catalog_assets"` - // Aggregate state of the failover group. + // Aggregate state of the failover group across Unity Catalog and workspace + // asset replication. The group is INITIAL_REPLICATION until it has a + // replication point, ACTIVE while replication is in steady state and failover + // is available, and FAILING_OVER while a failover is in progress; the + // `*_FAILED` values mean that the last operation did not complete. Not + // populated in ListFailoverGroups responses. State FailoverGroup_State `fieldmask:"state"` // Opaque version string for optimistic locking. Server-generated and returned // in responses. @@ -120,15 +134,24 @@ type FailoverGroup struct { CreateTime *types.Time `fieldmask:"create_time"` // Time at which this failover group was last modified. UpdateTime *types.Time `fieldmask:"update_time"` - // The latest point in time to which data has been replicated. + // The most recent point in time up to which all in-scope assets have been + // replicated to the secondary region. Data written after this point might not + // exist in the secondary yet and can be lost in an unplanned failover. When + // both Unity Catalog and workspace assets are replicated, this is the older of + // the two replication points. Not populated until the first replication point + // exists, and not populated in ListFailoverGroups responses. ReplicationPoint *types.Time `fieldmask:"replication_point"` // Initial primary region. Used only in Create requests to set the starting - // primary region. Not returned in responses. + // primary region, and must be one of the regions in `regions`. Not returned in + // responses. InitialPrimaryRegion *string `fieldmask:"initial_primary_region"` // Initial replication progress of all in-scope assets. Set only when the // FailoverGroup is in INITIAL_REPLICATION state. InitialReplicationProgress *InitialReplicationProgress `fieldmask:"initial_replication_progress"` - // True if errors currently block the replication point. + // True when replication errors are preventing the replication point from + // advancing. Populated only while the failover group is INITIAL_REPLICATION or + // ACTIVE; the blocking errors are listed in the system.replication.states + // system table. HasReplicationPointBlockingErrors *bool `fieldmask:"has_replication_point_blocking_errors"` } @@ -150,7 +173,8 @@ type GetStableUrlRequest struct { // workspace assets) of a FailoverGroup. Populated only while the FailoverGroup // is in INITIAL_REPLICATION.. type InitialReplicationProgress struct { - // Percent complete, 0-100. + // Percentage of in-scope assets replicated so far, from 0 to 100. Stays below + // 100 while errors block the first replication point. Completed *int `fieldmask:"completed"` } @@ -205,18 +229,22 @@ type ListStableUrlsResponse struct { // A location mapping identified by a name, with URIs per region. The system // derives replication direction from effective_primary_region.. type LocationMapping struct { - // Resource name for this location. + // Identifier of this location mapping, unique within the failover group. Name *string - // URI for each region. Each entry maps a region name to a storage URI. + // The storage URI of this location in each region, with exactly one entry per + // region of the failover group. UriByRegion []LocationMappingEntry } // A single entry in a location mapping, mapping a region to a storage URI. Used // instead of map for proto2 compatibility.. type LocationMappingEntry struct { - // The region name. + // Cloud region this URI applies to. Must be one of the failover group's + // regions. Region *string - // The storage URI for this region. + // Cloud storage URI of the location in this region. End the URI with `/*` to + // map every path under it as a prefix; within one mapping, either all URIs or + // none carry the `/*` suffix. Uri *string } @@ -255,18 +283,24 @@ type StableUrl struct { // A Unity Catalog catalog to replicate.. type UcCatalog struct { - // The name of the UC catalog to replicate. + // Name of the Unity Catalog catalog to replicate. Name *string } // Unity Catalog replication configuration (top-level, not per-set).. type UcReplicationConfig struct { - // Location mappings - storage URI per region for each location. + // Storage location mappings. Each mapping pairs a storage path used by the + // replicated catalogs in one region with the corresponding path in the other + // region, so that replicated data is written to the matching storage in the + // secondary region. LocationMappings []LocationMapping `fieldmask:"location_mappings"` - // UC catalogs to replicate. + // Unity Catalog catalogs to replicate, including their metadata and managed + // table data. Adding a catalog to an existing failover group returns the group + // to INITIAL_REPLICATION until the catalog has been replicated. Catalogs []UcCatalog `fieldmask:"catalogs"` - // The workspace set whose workspaces will be used for data replication of all - // UC catalogs' underlying storage. + // Name of the workspace set whose workspaces run the data replication for the + // catalogs' underlying storage. Must match the name of a workspace set in this + // failover group. DataReplicationWorkspaceSet *string `fieldmask:"data_replication_workspace_set"` } @@ -275,7 +309,9 @@ type UpdateFailoverGroupRequest struct { // The failover group with updated fields. The name field identifies the // resource and is populated from the URL path. FailoverGroup *FailoverGroup - // Comma-separated list of fields to update. + // Comma-separated list of the fields to update. The supported paths are + // `workspace_sets`, `unity_catalog_assets`, `unity_catalog_assets.catalogs` and + // `unity_catalog_assets.location_mappings`; any other path is rejected. UpdateMask *types.FieldMask[FailoverGroup] // Optional opaque version string for optimistic locking, obtained from a prior // read of the failover group. If provided, the update is rejected unless it @@ -286,16 +322,23 @@ type UpdateFailoverGroupRequest struct { // A set of workspaces that replicate to each other across regions.. type WorkspaceSet struct { - // Resource name for this workspace set. + // Identifier of the workspace set, unique within the failover group and + // referenced by unity_catalog_assets.data_replication_workspace_set. Cannot be + // changed after creation. Name *string - // Workspace IDs in this set. The system derives and validates regions. All - // workspaces must be in the Mission Critical tier. + // IDs of the two workspaces in this set, one in each of the failover group's + // regions. The system derives and validates each workspace's region. Both + // workspaces must belong to the account and have the Mission Critical add-on + // enabled, and they cannot be changed after creation. WorkspaceIds []string - // Whether to enable control plane DR (notebooks, jobs, clusters, etc.) for this - // set. Defaults to false. + // Whether to replicate workspace assets (notebooks, jobs, clusters, SQL + // warehouses and so on) and their ACLs for this set. Defaults to false. Can be + // turned on or off with UpdateFailoverGroup. ReplicateWorkspaceAssets *bool // Resource names of stable URLs associated with this workspace set. Format: // accounts/{account_id}/stable-urls/{stable_url_id}. The referenced stable URLs - // must already exist (via CreateStableUrl). + // must already exist (via CreateStableUrl), and the list can only be set when + // replicate_workspace_assets is true. While workspace asset replication stays + // enabled, an update can attach or detach one stable URL at a time. StableUrlNames []string } diff --git a/environments/CHANGELOG.md b/environments/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/environments/CHANGELOG.md +++ b/environments/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/environments/internal/version.go b/environments/internal/version.go index 66649fe3..fb5d8afe 100644 --- a/environments/internal/version.go +++ b/environments/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-environments" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/environments/v1/client.go b/environments/v1/client.go index 0070215f..a6ccbd09 100644 --- a/environments/v1/client.go +++ b/environments/v1/client.go @@ -82,7 +82,8 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { // Creates a new WorkspaceBaseEnvironment. This is a long-running operation. The // operation will asynchronously generate a materialized environment to optimize // dependency resolution and is only marked as done when the materialized -// environment has been successfully generated or has failed. +// environment has been successfully generated or has failed. Poll the returned +// operation with GetOperation. func (c *internalClient) createWorkspaceBaseEnvironmentBase(ctx context.Context, req CreateWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*Operation, error) { wireReq, err := createWorkspaceBaseEnvironmentRequestToWire(&req) if err != nil { @@ -160,7 +161,8 @@ func (c *internalClient) createWorkspaceBaseEnvironmentBase(ctx context.Context, // Creates a new WorkspaceBaseEnvironment. This is a long-running operation. The // operation will asynchronously generate a materialized environment to optimize // dependency resolution and is only marked as done when the materialized -// environment has been successfully generated or has failed. +// environment has been successfully generated or has failed. Poll the returned +// operation with GetOperation. func (c *internalClient) CreateWorkspaceBaseEnvironment(ctx context.Context, req CreateWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*CreateWorkspaceBaseEnvironmentOperation, error) { operation, err := c.createWorkspaceBaseEnvironmentBase(ctx, req, opts...) if err != nil { @@ -262,6 +264,7 @@ func (o *CreateWorkspaceBaseEnvironmentOperation) Wait(ctx context.Context, opts // Deletes a WorkspaceBaseEnvironment. Deleting a base environment may impact // linked notebooks and jobs. This operation is irreversible and should be // performed only when you are certain the environment is no longer needed. +// Databricks-provided base environments cannot be deleted. func (c *internalClient) DeleteWorkspaceBaseEnvironment(ctx context.Context, req DeleteWorkspaceBaseEnvironmentRequest, opts ...call.Option) error { headers := http.Header{} @@ -379,7 +382,13 @@ func (c *internalClient) GetDefaultWorkspaceBaseEnvironment(ctx context.Context, } // Gets the status of a long-running operation. Clients can use this method to -// poll the operation result. +// poll the operation result. Create, update and refresh return an operation +// named `operations/{workspace_base_environment_id}`. The operation is done +// when the materialized environment build finishes. On success, `response` +// contains the WorkspaceBaseEnvironment with status `CREATED`. When the status +// is `FAILED`, `EXPIRED` or `INVALID`, `error` is set instead, and +// `error.message` carries the environment's status message when one is +// available. func (c *internalClient) getOperation(ctx context.Context, req GetOperationRequest, opts ...call.Option) (*Operation, error) { headers := http.Header{} @@ -626,7 +635,9 @@ func (c *internalClient) ListWorkspaceBaseEnvironmentsIter(ctx context.Context, // is a long-running operation. The operation will asynchronously regenerate the // materialized environment and is only marked as done when the materialized // environment has been successfully generated or has failed. The existing -// materialized environment remains available until it expires. +// materialized environment remains available until it expires. Poll the +// returned operation with GetOperation. Databricks-provided base environments +// cannot be refreshed. func (c *internalClient) refreshWorkspaceBaseEnvironmentBase(ctx context.Context, req RefreshWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*Operation, error) { wireReq, err := refreshWorkspaceBaseEnvironmentRequestToWire(&req) if err != nil { @@ -703,7 +714,9 @@ func (c *internalClient) refreshWorkspaceBaseEnvironmentBase(ctx context.Context // is a long-running operation. The operation will asynchronously regenerate the // materialized environment and is only marked as done when the materialized // environment has been successfully generated or has failed. The existing -// materialized environment remains available until it expires. +// materialized environment remains available until it expires. Poll the +// returned operation with GetOperation. Databricks-provided base environments +// cannot be refreshed. func (c *internalClient) RefreshWorkspaceBaseEnvironment(ctx context.Context, req RefreshWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*RefreshWorkspaceBaseEnvironmentOperation, error) { operation, err := c.refreshWorkspaceBaseEnvironmentBase(ctx, req, opts...) if err != nil { @@ -884,7 +897,8 @@ func (c *internalClient) UpdateDefaultWorkspaceBaseEnvironment(ctx context.Conte // operation. The operation will asynchronously regenerate the materialized // environment and is only marked as done when the materialized environment has // been successfully generated or has failed. The existing materialized -// environment remains available until it expires. +// environment remains available until it expires. Poll the returned operation +// with GetOperation. Databricks-provided base environments cannot be updated. func (c *internalClient) updateWorkspaceBaseEnvironmentBase(ctx context.Context, req UpdateWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*Operation, error) { wireReq, err := updateWorkspaceBaseEnvironmentRequestToWire(&req) if err != nil { @@ -960,7 +974,8 @@ func (c *internalClient) updateWorkspaceBaseEnvironmentBase(ctx context.Context, // operation. The operation will asynchronously regenerate the materialized // environment and is only marked as done when the materialized environment has // been successfully generated or has failed. The existing materialized -// environment remains available until it expires. +// environment remains available until it expires. Poll the returned operation +// with GetOperation. Databricks-provided base environments cannot be updated. func (c *internalClient) UpdateWorkspaceBaseEnvironment(ctx context.Context, req UpdateWorkspaceBaseEnvironmentRequest, opts ...call.Option) (*UpdateWorkspaceBaseEnvironmentOperation, error) { operation, err := c.updateWorkspaceBaseEnvironmentBase(ctx, req, opts...) if err != nil { diff --git a/environments/v1/model.go b/environments/v1/model.go index 98ffff21..5b9c21e8 100644 --- a/environments/v1/model.go +++ b/environments/v1/model.go @@ -575,7 +575,7 @@ func (*Operation_Result_Response) isOperation_Result() {} // Request message for RefreshWorkspaceBaseEnvironments.. type RefreshWorkspaceBaseEnvironmentRequest struct { - // Required. The resource name of the workspace base environment to delete. + // Required. The resource name of the workspace base environment to refresh. // Format: workspace-base-environments/{workspace_base_environment} Name *string } @@ -597,9 +597,15 @@ type UpdateDefaultWorkspaceBaseEnvironmentRequest struct { // Request message for UpdateWorkspaceBaseEnvironment.. type UpdateWorkspaceBaseEnvironmentRequest struct { + // The resource name of the workspace base environment to update. Format: + // workspace-base-environments/{workspace_base_environment} Name *string - // Required. The workspace base environment with updated fields. The name field - // is used to identify the environment to update. + // The workspace base environment with its new configuration. The `name` path + // parameter identifies the environment to update. Set `display_name` and + // `filepath` on every update, even when they do not change. An update keeps the + // stored `base_environment_type`, but the YAML file is validated against the + // type in the request, which defaults to `CPU`, so also set it for a GPU + // environment. WorkspaceBaseEnvironment *WorkspaceBaseEnvironment } @@ -608,11 +614,16 @@ type UpdateWorkspaceBaseEnvironmentRequest struct { // dependencies.. type WorkspaceBaseEnvironment struct { // The resource name of the workspace base environment. Format: - // workspace-base-environments/{workspace-base-environment} + // workspace-base-environments/{workspace-base-environment} The server ignores + // this field on create and builds the name from + // `workspace_base_environment_id`, or from `display_name` plus a random suffix + // when no ID is given. Name *string // Human-readable display name for the workspace base environment. DisplayName *string - // The WSFS or UC Volumes path to the environment YAML file. + // The workspace file or Unity Catalog volume path to the YAML file that defines + // the environment version and dependencies. The file name must end in `.yaml` + // or `.yml`. Create and update requests must set this field. Filepath *string // User ID of the creator. CreatorUserId *string @@ -622,15 +633,19 @@ type WorkspaceBaseEnvironment struct { LastUpdatedUserId *string // Timestamp when the environment was last updated. UpdateTime *types.Time - // The status of the materialized workspace base environment. + // The status of the materialized workspace base environment, which the server + // builds from the YAML file. The status is `CREATED` once the build has + // succeeded. Status WorkspaceBaseEnvironmentCache_Status // Status message providing additional details about the environment status. + // When the status is `FAILED`, the message contains the build error. Message *string // Whether this is the default environment for the workspace. IsDefault *bool - // The type of base environment (CPU or GPU). + // The type of base environment (CPU or GPU). The default is `CPU`. BaseEnvironmentType BaseEnvironmentType - // The environment specification containing version and dependencies. + // The environment specification containing version and dependencies, read from + // the YAML file at `filepath`. Spec *EnvironmentSpec } diff --git a/experiments/CHANGELOG.md b/experiments/CHANGELOG.md index 6e20e0f8..cd33673a 100644 --- a/experiments/CHANGELOG.md +++ b/experiments/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.9.0 (2026-11-17) + ## Release v0.8.0 (2026-11-16) ## Release v0.7.0 (2026-11-12) diff --git a/experiments/internal/version.go b/experiments/internal/version.go index 79f51338..5f42e26f 100644 --- a/experiments/internal/version.go +++ b/experiments/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-experiments" -const Version = "0.8.0" +const Version = "0.9.0" diff --git a/experiments/v1/client.go b/experiments/v1/client.go index 5b97ecb0..fc1f48d1 100644 --- a/experiments/v1/client.go +++ b/experiments/v1/client.go @@ -616,7 +616,8 @@ func (c *internalClient) CreateRun(ctx context.Context, req CreateRunRequest, op // Marks an experiment and associated metadata, runs, metrics, params, and tags // for deletion. If the experiment uses FileStore, artifacts associated with the -// experiment are also deleted. +// experiment are also deleted. Only workspace experiments can be deleted +// through this method. func (c *internalClient) DeleteExperiment(ctx context.Context, req DeleteExperimentRequest, opts ...call.Option) (*DeleteExperimentResponse, error) { wireReq, err := deleteExperimentRequestToWire(&req) if err != nil { @@ -1439,7 +1440,8 @@ func (c *internalClient) ListArtifactsIter(ctx context.Context, req ListArtifact } } -// Gets a list of all experiments. +// Gets a list of all experiments, ordered by creation time with the newest +// first. func (c *internalClient) ListExperiments(ctx context.Context, req ListExperimentsRequest, opts ...call.Option) (*ListExperimentsResponse, error) { wireReq, err := listExperimentsRequestToWire(&req) if err != nil { @@ -2152,6 +2154,8 @@ func (c *internalClient) LogParam(ctx context.Context, req LogParamRequest, opts // metadata, runs, metrics, params, and tags. If experiment uses FileStore, // underlying artifacts associated with experiment are also restored. // +// Only workspace experiments can be restored through this method. +// // Throws `RESOURCE_DOES_NOT_EXIST` if experiment was never created or was // permanently deleted. func (c *internalClient) RestoreExperiment(ctx context.Context, req RestoreExperimentRequest, opts ...call.Option) (*RestoreExperimentResponse, error) { @@ -2816,7 +2820,8 @@ func (c *internalClient) SetTag(ctx context.Context, req SetTagRequest, opts ... return resp, nil } -// Updates experiment metadata. +// Updates experiment metadata. Only workspace experiments can be updated; a +// notebook experiment takes its name from its notebook. func (c *internalClient) UpdateExperiment(ctx context.Context, req UpdateExperimentRequest, opts ...call.Option) (*UpdateExperimentResponse, error) { wireReq, err := updateExperimentRequestToWire(&req) if err != nil { diff --git a/experiments/v1/model.go b/experiments/v1/model.go index 784a007a..3538ff7b 100644 --- a/experiments/v1/model.go +++ b/experiments/v1/model.go @@ -121,16 +121,20 @@ const ( ) type CreateExperimentRequest struct { - // Experiment name. + // Name of the experiment, as an absolute path in the workspace. Experiments + // cannot be created under `/Repos`. Name *string - // Location where all artifacts for the experiment are stored. If not provided, - // the remote server will select an appropriate default. + // URI of the location where the artifacts of the experiment's runs are stored, + // such as a Unity Catalog volume path of the form + // `dbfs:/Volumes////`. The URI must have a + // scheme other than `file` and must not point under the MLflow-managed prefix + // `dbfs:/databricks/mlflow-tracking`. If omitted, artifacts are stored in + // MLflow-managed storage. See + // https://docs.databricks.com/aws/en/mlflow/experiments for the supported + // locations. ArtifactLocation *string - // A collection of tags to set on the experiment. Maximum tag size and number of - // tags per request depends on the storage backend. All storage backends are - // guaranteed to support tag keys up to 250 bytes in size and tag values up to - // 5000 bytes in size. All storage backends are also guaranteed to support up to - // 20 tags per request. + // Tags to set on the new experiment, as key-value pairs. A request can include + // at most 100 tags. Tags []ExperimentTag // The location where the experiment's traces are stored. When set, the // underlying storage is provisioned and the experiment's traces are routed to @@ -295,18 +299,24 @@ type DeleteTagResponse struct { type Experiment struct { // Unique identifier for the experiment. ExperimentId *string - // Human readable name that identifies the experiment. + // Name of the experiment. A workspace experiment is named by its absolute path + // in the workspace; a notebook experiment has the path of its notebook. Name *string - // Location where artifacts for the experiment are stored. + // URI of the location where the artifacts of the experiment's runs are stored. + // Experiments created without an explicit location use MLflow-managed storage + // under `dbfs:/databricks/mlflow-tracking/`. ArtifactLocation *string - // Current life cycle stage of the experiment: "active" or "deleted". Deleted - // experiments are not returned by APIs. + // Current life cycle stage of the experiment: `active` or `deleted`. Deleted + // experiments are excluded from list and search results unless `view_type` + // requests them, and can be restored until they are permanently deleted. LifecycleStage *string - // Last update time + // Time when the experiment metadata was last updated, as a Unix timestamp in + // milliseconds. LastUpdateTime *int64 - // Creation time + // Time when the experiment was created, as a Unix timestamp in milliseconds. CreationTime *int64 - // Tags: Additional metadata key-value pairs. + // Tags set on the experiment, as key-value pairs, including system tags such as + // `mlflow.ownerId` that MLflow maintains. Tags []ExperimentTag // The location where the experiment's traces are stored. Unset when traces are // stored in the default MLflow backend. This field cannot be updated after the @@ -316,9 +326,9 @@ type Experiment struct { // A tag for an experiment.. type ExperimentTag struct { - // The tag key. + // The tag key. Keys are at most 255 characters long. Key *string - // The tag value. + // The tag value. Values are at most 65535 characters long. Value *string } @@ -537,7 +547,8 @@ type ListExperimentsRequest struct { // capped at 1000. Callers of this endpoint are encouraged to pass max_results // explicitly and leverage page_token to iterate through experiments. MaxResults *int64 - // Token indicating the page of experiments to fetch + // Page token returned as `next_page_token` by a previous call. Omit it to fetch + // the first page. PageToken *string } @@ -874,16 +885,23 @@ type RunTag struct { } type SearchExperimentsRequest struct { - // Maximum number of experiments desired. Max threshold is 3000. + // Maximum number of experiments to return per page. Larger values are capped by + // the server, so use `page_token` to retrieve further pages. MaxResults *int64 - // Token indicating the page of experiments to fetch + // Page token returned as `next_page_token` by a previous call. Omit it to fetch + // the first page. PageToken *string - // String representing a SQL filter condition (e.g. "name ILIKE - // 'my-experiment%'") + // Filter condition on the experiment name and tags. The `name` attribute and + // `tags.` support the operators `=`, `!=`, `LIKE` and `ILIKE` + // (case-insensitive `LIKE`), string values are wrapped in single quotes, and + // several conditions can be combined with `AND`. No wildcard is added + // automatically, so include `%` in a `LIKE` pattern where you need one. Filter *string - // List of columns for ordering search results, which can include experiment - // name and last updated timestamp with an optional "DESC" or "ASC" annotation, - // where "ASC" is the default. Tiebreaks are done by experiment id DESC. + // Columns to order the results by, each given as `name`, `basename` (the last + // segment of the name), `experiment_id`, `creation_time` or `last_update_time` + // with an optional `ASC` or `DESC` suffix, where `ASC` is the default. If + // omitted, results are ordered by last update time with the most recently + // updated experiment first. Tiebreaks are done by experiment id DESC. OrderBy []string // Qualifier for type of experiments to be returned. If unspecified, return only // active experiments. @@ -1035,10 +1053,11 @@ type SearchRunsResponse struct { type SetExperimentTagRequest struct { // ID of the experiment under which to log the tag. Must be provided. ExperimentId *string - // Name of the tag. Keys up to 250 bytes in size are supported. + // Key of the tag. Keys are at most 255 characters long. Setting a key that + // already exists replaces its value. System tags such as `mlflow.ownerId`, + // `mlflow.ownerEmail` and `mlflow.experimentType` cannot be set. Key *string - // String value of the tag being logged. Values up to 64KB in size are - // supported. + // Value of the tag. Values are at most 65535 characters long. Value *string } @@ -1093,8 +1112,8 @@ type UcTraceLocation struct { type UpdateExperimentRequest struct { // ID of the associated experiment. ExperimentId *string - // If provided, the experiment's name is changed to the new name. The new name - // must be unique. + // New name for the experiment, as an absolute path in the workspace. The new + // name must be unique. NewName *string } diff --git a/features/CHANGELOG.md b/features/CHANGELOG.md index be4b3bfc..7ebcb36e 100644 --- a/features/CHANGELOG.md +++ b/features/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.23.0 (2026-11-17) + ## Release v0.22.0 (2026-11-16) ## Release v0.21.0 (2026-11-12) diff --git a/features/internal/version.go b/features/internal/version.go index 131159ae..4357123d 100644 --- a/features/internal/version.go +++ b/features/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-features" -const Version = "0.22.0" +const Version = "0.23.0" diff --git a/features/v1/model.go b/features/v1/model.go index 6964dff2..872e0d9f 100644 --- a/features/v1/model.go +++ b/features/v1/model.go @@ -2158,9 +2158,11 @@ type MaterializedFeature struct { Notifications *MaterializationNotifications `fieldmask:"notifications"` // The ID of the materialization set containing this materialized feature. Unset // for a materialized feature that is not part of a set. - MaterializationSetId *string `fieldmask:"materialization_set_id"` - _ [0]materializedFeatureDestinationFieldMaskMetadata `fieldmask_oneof:"Destination"` - _ [0]materializedFeatureTriggerFieldMaskMetadata `fieldmask_oneof:"Trigger"` + MaterializationSetId *string `fieldmask:"materialization_set_id"` + // Operations associated with this materialized feature. + Operations *MaterializedFeatureOperations `fieldmask:"operations"` + _ [0]materializedFeatureDestinationFieldMaskMetadata `fieldmask_oneof:"Destination"` + _ [0]materializedFeatureTriggerFieldMaskMetadata `fieldmask_oneof:"Trigger"` } type isMaterializedFeature_Destination interface { @@ -2249,6 +2251,12 @@ type MaterializedFeatureChangeFailure struct { Message *string } +// References to operations on a materialized feature.. +type MaterializedFeatureOperations struct { + // Name of the latest entity purge operation. Format: operations/{operation_id}. + LatestEntityPurgeOperation *string `fieldmask:"latest_entity_purge_operation"` +} + // Computes the maximum value.. type MaxFunction struct { // The input column from which the maximum is computed. diff --git a/features/v1/wire.go b/features/v1/wire.go index ccd64a3e..abe81421 100644 --- a/features/v1/wire.go +++ b/features/v1/wire.go @@ -6186,6 +6186,7 @@ type materializedFeatureWire struct { JobId *wireInt64 `json:"job_id,omitempty"` Notifications *materializationNotificationsWire `json:"notifications,omitempty"` MaterializationSetId *string `json:"materialization_set_id,omitempty"` + Operations *materializedFeatureOperationsWire `json:"operations,omitempty"` } func materializedFeatureToWire(v *MaterializedFeature) (*materializedFeatureWire, error) { @@ -6200,6 +6201,10 @@ func materializedFeatureToWire(v *MaterializedFeature) (*materializedFeatureWire if err != nil { return nil, fmt.Errorf("%s: %w", "MaterializedFeature.Notifications", err) } + operationsWireValue, err := materializedFeatureOperationsToWire(v.Operations) + if err != nil { + return nil, fmt.Errorf("%s: %w", "MaterializedFeature.Operations", err) + } var destinationOfflineStoreConfigWire *offlineStoreConfigWire var destinationOnlineStoreConfigWire *onlineStoreConfigWire switch value := v.Destination.(type) { @@ -6285,6 +6290,7 @@ func materializedFeatureToWire(v *MaterializedFeature) (*materializedFeatureWire JobId: jobIdWireValue, Notifications: notificationsWireValue, MaterializationSetId: v.MaterializationSetId, + Operations: operationsWireValue, }, nil } @@ -6326,6 +6332,10 @@ func materializedFeatureFromWire(w *materializedFeatureWire) (*MaterializedFeatu if err != nil { return nil, fmt.Errorf("%s: %w", "MaterializedFeature.Notifications", err) } + operationsPublicValue, err := materializedFeatureOperationsFromWire(w.Operations) + if err != nil { + return nil, fmt.Errorf("%s: %w", "MaterializedFeature.Operations", err) + } var destinationSelection isMaterializedFeature_Destination switch { case w.OfflineStoreConfig != nil: @@ -6383,6 +6393,7 @@ func materializedFeatureFromWire(w *materializedFeatureWire) (*MaterializedFeatu JobId: jobIdPublicValue, Notifications: notificationsPublicValue, MaterializationSetId: w.MaterializationSetId, + Operations: operationsPublicValue, Destination: destinationSelection, Trigger: triggerSelection, }, nil @@ -6464,6 +6475,52 @@ func (v *MaterializedFeatureChangeFailure) UnmarshalJSON(data []byte) error { return nil } +type materializedFeatureOperationsWire struct { + LatestEntityPurgeOperation *string `json:"latest_entity_purge_operation,omitempty"` +} + +func materializedFeatureOperationsToWire(v *MaterializedFeatureOperations) (*materializedFeatureOperationsWire, error) { + if v == nil { + return nil, nil + } + return &materializedFeatureOperationsWire{ + LatestEntityPurgeOperation: v.LatestEntityPurgeOperation, + }, nil +} + +func materializedFeatureOperationsFromWire(w *materializedFeatureOperationsWire) (*MaterializedFeatureOperations, error) { + if w == nil { + return nil, nil + } + return &MaterializedFeatureOperations{ + LatestEntityPurgeOperation: w.LatestEntityPurgeOperation, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v MaterializedFeatureOperations) MarshalJSON() ([]byte, error) { + w, err := materializedFeatureOperationsToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *MaterializedFeatureOperations) UnmarshalJSON(data []byte) error { + var w materializedFeatureOperationsWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := materializedFeatureOperationsFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type maxFunctionWire struct { Input *string `json:"input,omitempty"` } diff --git a/files/CHANGELOG.md b/files/CHANGELOG.md index f4677500..124c8b7d 100644 --- a/files/CHANGELOG.md +++ b/files/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-16) ## Release v0.6.0 (2026-11-12) diff --git a/files/internal/version.go b/files/internal/version.go index ef54e9f9..c5b13c74 100644 --- a/files/internal/version.go +++ b/files/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-files" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/files/v2/ext_discovery.go b/files/v2/ext_discovery.go new file mode 100644 index 00000000..82148ae0 --- /dev/null +++ b/files/v2/ext_discovery.go @@ -0,0 +1,201 @@ +package files + +// Endpoint discovery: the workspace lists the endpoints a transfer should use, +// best first (GET /api/2.0/fs/endpoints). + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/databricks/sdk-go/core/apierr" +) + +const endpointsAPIPath = "/api/2.0/fs/endpoints" + +// endpointsResolveTimeout bounds discovery, which is a single attempt: it +// precedes a transfer, so it must fall back to the built-in order quickly +// rather than retry and stall the upload. +const endpointsResolveTimeout = 10 * time.Second + +// endpointsCacheKey keys the client's discovered endpoints in its extensions. +type endpointsCacheKey struct{} + +// endpointsCache holds the workspace's answer, a failure included, so a client +// asks once. +type endpointsCache struct { + mu sync.Mutex + resolved bool + endpoints []endpoint +} + +// transferEndpoints returns the endpoints a transfer walks, best first: the +// built-in order, or with discovery on, the workspace's list. Anything short of +// a usable list falls back to the built-in order, so discovery never fails a +// transfer. +func (e *engine) transferEndpoints(ctx context.Context) []endpoint { + if !e.tun.endpointsAPI { + return e.builtinEndpoints() + } + v, _ := e.c.extensions.LoadOrStore(endpointsCacheKey{}, &endpointsCache{}) + cache := v.(*endpointsCache) + cache.mu.Lock() + defer cache.mu.Unlock() + if !cache.resolved { + endpoints := e.fetchEndpoints(ctx) + if ctx.Err() != nil { + // The caller gave up; that is not the workspace's answer. + return e.builtinEndpoints() + } + if len(endpoints) == 0 { + endpoints = e.builtinEndpoints() + } + cache.resolved, cache.endpoints = true, endpoints + } + return slices.Clone(cache.endpoints) +} + +// fetchEndpoints asks the workspace for its endpoint list and keeps the +// entries this package can use. It returns nil if there are none. +func (e *engine) fetchEndpoints(ctx context.Context) []endpoint { + ctx, cancel := context.WithTimeout(ctx, endpointsResolveTimeout) + defer cancel() + urlStr, err := endpointURL(e.c.host, endpointsAPIPath, nil) + if err != nil { + e.c.logger.WarnContext(ctx, "failed to resolve file endpoints, using the default endpoints", "error", err) + return nil + } + headers := http.Header{} + headers.Set("Accept", "application/json") + req, err := newHTTPRequest(ctx, httpRequestOptions{Method: http.MethodGet, URL: urlStr, Credentials: e.c.credentials, Headers: headers}) + if err != nil { + e.c.logger.WarnContext(ctx, "failed to resolve file endpoints, using the default endpoints", "error", err) + return nil + } + e.setWorkspaceHeader(req) + body, respHeader, err := executeHTTPCall(httpCallOptions{req: req, client: e.c.httpClient, logger: e.c.logger}) + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + requestID := aerr.HTTPHeader().Get("X-Request-Id") + if aerr.HTTPStatusCode() == http.StatusNotFound { + // Not a warning: some workspaces don't serve the API yet. + e.c.logger.InfoContext(ctx, "the endpoints API is not served for this workspace, using the default endpoints", "request_id", requestID) + return nil + } + e.c.logger.WarnContext(ctx, "unexpected status resolving file endpoints, using the default endpoints", + "status", aerr.HTTPStatusCode(), "request_id", requestID) + return nil + } + if err != nil { + e.c.logger.WarnContext(ctx, "failed to resolve file endpoints, using the default endpoints", "error", err) + return nil + } + requestID := respHeader.Get("X-Request-Id") + var resp struct { + // Proto3 JSON leaves out an empty repeated field. + Endpoints []json.RawMessage `json:"endpoints"` + } + if err := json.Unmarshal(body, &resp); err != nil { + e.c.logger.WarnContext(ctx, "unexpected endpoints response, using the default endpoints", "request_id", requestID, "error", err) + return nil + } + var endpoints []endpoint + for _, raw := range resp.Endpoints { + var entry map[string]any + _ = json.Unmarshal(raw, &entry) + kind, ok := endpointKindOf(entry) + if !ok { + // Not a warning: an older SDK skips the APIs added after it shipped. + e.c.logger.InfoContext(ctx, "skipping an endpoint this SDK cannot use", "request_id", requestID, "entry", string(raw)) + continue + } + ep, ok := e.parseEndpoint(entry, kind) + if !ok { + e.c.logger.WarnContext(ctx, "skipping an endpoint with an unusable address", "request_id", requestID, "entry", string(raw)) + continue + } + endpoints = append(endpoints, ep) + } + if len(endpoints) == 0 { + e.c.logger.WarnContext(ctx, "no usable endpoints in the response, using the default endpoints", "request_id", requestID) + } + return endpoints +} + +// endpointKindOf returns the protocol of a listed endpoint by its api and +// api_version, or false if this package doesn't speak it. Pairs not listed +// here are skipped, which lets the server list newer protocols without +// breaking older SDKs. IO_OFFLOAD is unversioned during the preview, so a +// versioned IO_OFFLOAD is newer than this package. +func endpointKindOf(entry map[string]any) (endpointKind, bool) { + api, ok := entry["api"].(string) + if !ok { + return 0, false + } + version, hasVersion := entry["api_version"] + if version == nil { + hasVersion = false + } + versionStr, isString := version.(string) + if hasVersion && !isString { + return 0, false + } + switch { + case api == "IO_OFFLOAD" && !hasVersion: + return presignedEndpoint, true + case api == "FILES_API" && versionStr == "2.0": + return filesAPIEndpoint, true + case api == "FILES_API" && versionStr == "3.0": + return filesAPIv3Endpoint, true + } + return 0, false +} + +// parseEndpoint returns the endpoint of a listed entry, or false if its address +// is unusable: anything but a bare http or https host and port, which requests +// go to as written. +func (e *engine) parseEndpoint(entry map[string]any, kind endpointKind) (endpoint, bool) { + scheme, _ := entry["scheme"].(string) + hostname, _ := entry["hostname"].(string) + if (scheme != "http" && scheme != "https") || hostname == "" { + return endpoint{}, false + } + if kind == filesAPIv3Endpoint && scheme != "https" { + return endpoint{}, false + } + base := scheme + "://" + hostname + u, err := url.Parse(base) + if err != nil || u.Host != hostname || u.User != nil || strings.HasSuffix(u.Host, ":") { + return endpoint{}, false + } + if p := u.Port(); p != "" { + if n, err := strconv.Atoi(p); err != nil || n < 1 || n > 65535 { + return endpoint{}, false + } + } + // Plain HTTP carries workspace credentials in the clear. The workspace host + // may use it only if the client reaches the workspace that way; another host + // only for Files API 2.0, which is the in-cluster storage proxy. + if scheme == "http" { + workspace, err := originOf(e.c.host) + if err != nil { + return endpoint{}, false + } + if strings.EqualFold(u.Hostname(), workspace.host) { + if workspace.scheme != "http" { + return endpoint{}, false + } + } else if kind != filesAPIEndpoint { + return endpoint{}, false + } + } + // As the proto getter reads it, an unset bool is false. + multipart, _ := entry["multipart_upload_supported"].(bool) + return endpoint{kind: kind, baseURL: base, multipart: multipart}, true +} diff --git a/files/v2/ext_discovery_test.go b/files/v2/ext_discovery_test.go new file mode 100644 index 00000000..989508c1 --- /dev/null +++ b/files/v2/ext_discovery_test.go @@ -0,0 +1,203 @@ +package files + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "slices" + "sync/atomic" + "testing" +) + +// discoveryEngine returns an engine with discovery on whose workspace answers +// GET /api/2.0/fs/endpoints with h, and counts the requests it gets. +func discoveryEngine(t *testing.T, h http.HandlerFunc) (*engine, *atomic.Int32) { + t.Helper() + var calls atomic.Int32 + srv := httptest.NewServer(requireCredentials(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != endpointsAPIPath { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + return + } + calls.Add(1) + h(w, r) + })) + t.Cleanup(srv.Close) + e := newEngine(buildUploadClient(t, srv.URL, srv.Client(), "")) + e.tun.endpointsAPI = true + return e, &calls +} + +func respondJSON(status int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + } +} + +func TestDiscoveryFallsBackToTheBuiltinOrder(t *testing.T) { + cases := map[string]http.HandlerFunc{ + "not served": respondJSON(http.StatusNotFound, `{"error_code":"NOT_FOUND","message":"no"}`), + "a server error": respondJSON(http.StatusInternalServerError, `{"error_code":"INTERNAL_ERROR","message":"boom"}`), + "not JSON": respondJSON(http.StatusOK, ``), + "no endpoints": respondJSON(http.StatusOK, `{}`), + "endpoints not a list": respondJSON(http.StatusOK, `{"endpoints":{"api":"IO_OFFLOAD"}}`), + "only unusable entries": respondJSON(http.StatusOK, `{"endpoints":[ + {"api":"NEW_API","scheme":"https","hostname":"new.example.com"}, + {"api":"FILES_API","api_version":"2.0","scheme":"ftp","hostname":"proxy.example.com"}]}`), + } + for name, h := range cases { + t.Run(name, func(t *testing.T) { + e, calls := discoveryEngine(t, h) + for range 2 { + if got, want := e.transferEndpoints(t.Context()), e.builtinEndpoints(); !slices.Equal(got, want) { + t.Errorf("endpoints = %v, want the built-in %v", got, want) + } + } + // The failure is cached too: the workspace is asked once. + if calls.Load() != 1 { + t.Errorf("endpoints requests = %d, want 1", calls.Load()) + } + }) + } +} + +func TestDiscoveryIsOffByDefault(t *testing.T) { + e, calls := discoveryEngine(t, respondJSON(http.StatusOK, `{"endpoints":[]}`)) + e.tun = defaultTunables() + if got, want := e.transferEndpoints(t.Context()), e.builtinEndpoints(); !slices.Equal(got, want) { + t.Errorf("endpoints = %v, want the built-in %v", got, want) + } + if calls.Load() != 0 { + t.Errorf("endpoints requests = %d, want none", calls.Load()) + } +} + +func TestDiscoveryKeepsTheUsableEntriesInOrder(t *testing.T) { + e, calls := discoveryEngine(t, respondJSON(http.StatusOK, `{"endpoints":[ + {"api":"FILES_API","api_version":"2.0","scheme":"http","hostname":"proxy.example.com","multipart_upload_supported":true}, + {"api":"NEW_API","scheme":"https","hostname":"new.example.com"}, + {"api":"IO_OFFLOAD","scheme":"https","hostname":"ws.example.com","multipart_upload_supported":true}, + {"api":"FILES_API","api_version":"2.0","scheme":"https","hostname":"ws.example.com"}]}`)) + want := []endpoint{ + {kind: filesAPIEndpoint, baseURL: "http://proxy.example.com", multipart: true}, + {kind: presignedEndpoint, baseURL: "https://ws.example.com", multipart: true}, + {kind: filesAPIEndpoint, baseURL: "https://ws.example.com"}, + } + for range 2 { + if got := e.transferEndpoints(t.Context()); !slices.Equal(got, want) { + t.Errorf("endpoints = %v, want %v", got, want) + } + } + if calls.Load() != 1 { + t.Errorf("endpoints requests = %d, want 1", calls.Load()) + } +} + +func TestDiscoveryCancelledByTheCallerIsNotCached(t *testing.T) { + e, calls := discoveryEngine(t, respondJSON(http.StatusOK, `{"endpoints":[ + {"api":"FILES_API","api_version":"2.0","scheme":"http","hostname":"proxy.example.com"}]}`)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if got, want := e.transferEndpoints(ctx), e.builtinEndpoints(); !slices.Equal(got, want) { + t.Errorf("endpoints = %v, want the built-in %v", got, want) + } + want := []endpoint{{kind: filesAPIEndpoint, baseURL: "http://proxy.example.com"}} + if got := e.transferEndpoints(t.Context()); !slices.Equal(got, want) { + t.Errorf("endpoints = %v, want %v", got, want) + } + if calls.Load() != 1 { + t.Errorf("endpoints requests = %d, want 1", calls.Load()) + } +} + +func TestUploadEndpointsAddTheWorkspaceToPresignedOnly(t *testing.T) { + e, _ := discoveryEngine(t, respondJSON(http.StatusOK, `{"endpoints":[ + {"api":"IO_OFFLOAD","scheme":"https","hostname":"ws.example.com","multipart_upload_supported":true}]}`)) + want := []endpoint{ + {kind: presignedEndpoint, baseURL: "https://ws.example.com", multipart: true}, + {kind: filesAPIEndpoint, baseURL: e.c.host}, + } + if got := e.uploadEndpoints(t.Context()); !slices.Equal(got, want) { + t.Errorf("upload endpoints = %v, want %v", got, want) + } +} + +func TestEndpointKindOf(t *testing.T) { + tests := []struct { + name string + entry map[string]any + want endpointKind + wantOK bool + }{ + {"IO_OFFLOAD", map[string]any{"api": "IO_OFFLOAD"}, presignedEndpoint, true}, + {"IO_OFFLOAD with a null version", map[string]any{"api": "IO_OFFLOAD", "api_version": nil}, presignedEndpoint, true}, + {"a versioned IO_OFFLOAD is newer", map[string]any{"api": "IO_OFFLOAD", "api_version": "2.0"}, 0, false}, + {"FILES_API 2.0", map[string]any{"api": "FILES_API", "api_version": "2.0"}, filesAPIEndpoint, true}, + {"FILES_API 3.0", map[string]any{"api": "FILES_API", "api_version": "3.0"}, filesAPIv3Endpoint, true}, + {"FILES_API without a version", map[string]any{"api": "FILES_API"}, 0, false}, + {"a newer FILES_API", map[string]any{"api": "FILES_API", "api_version": "9.0"}, 0, false}, + {"a version that is not a string", map[string]any{"api": "FILES_API", "api_version": 2.0}, 0, false}, + {"an api that is not a string", map[string]any{"api": []any{"FILES_API"}, "api_version": "2.0"}, 0, false}, + {"not an object", nil, 0, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := endpointKindOf(tc.entry) + if got != tc.want || ok != tc.wantOK { + t.Errorf("endpointKindOf = (%v, %v), want (%v, %v)", got, ok, tc.want, tc.wantOK) + } + }) + } +} + +func TestParseEndpoint(t *testing.T) { + entry := func(scheme, hostname string) map[string]any { + return map[string]any{"scheme": scheme, "hostname": hostname, "multipart_upload_supported": true} + } + tests := []struct { + name string + workspaceHost string + kind endpointKind + entry map[string]any + wantBase string // "" when the entry is unusable + }{ + {"https", "https://ws.example.com", presignedEndpoint, entry("https", "ws.example.com"), "https://ws.example.com"}, + {"with a port", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com:8443"), "https://proxy.example.com:8443"}, + {"an IPv6 host", "https://ws.example.com", filesAPIEndpoint, entry("https", "[::1]:8443"), "https://[::1]:8443"}, + {"plain http to the storage proxy", "https://ws.example.com", filesAPIEndpoint, entry("http", "proxy.example.com"), "http://proxy.example.com"}, + {"plain http to a workspace reached that way", "http://localhost", presignedEndpoint, entry("http", "localhost"), "http://localhost"}, + {"plain http to a workspace reached over https", "https://ws.example.com", filesAPIEndpoint, entry("http", "WS.example.com"), ""}, + {"plain http presigned to another host", "https://ws.example.com", presignedEndpoint, entry("http", "other.example.com"), ""}, + {"Files API 3.0 over https", "https://ws.example.com", filesAPIv3Endpoint, entry("https", "rig.example.com"), "https://rig.example.com"}, + {"Files API 3.0 over plain http", "https://ws.example.com", filesAPIv3Endpoint, entry("http", "rig.example.com"), ""}, + {"Files API 3.0 over plain http to a workspace reached that way", "http://localhost", filesAPIv3Endpoint, entry("http", "localhost"), ""}, + {"another scheme", "https://ws.example.com", filesAPIEndpoint, entry("ftp", "proxy.example.com"), ""}, + {"no hostname", "https://ws.example.com", filesAPIEndpoint, entry("https", ""), ""}, + {"a hostname that is not a string", "https://ws.example.com", filesAPIEndpoint, map[string]any{"scheme": "https", "hostname": 1}, ""}, + {"a path", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com/files"), ""}, + {"a query", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com?x=1"), ""}, + {"user info", "https://ws.example.com", filesAPIEndpoint, entry("https", "user@proxy.example.com"), ""}, + {"an empty port", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com:"), ""}, + {"a port out of range", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com:70000"), ""}, + {"a port that is not a number", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy.example.com:https"), ""}, + {"a space", "https://ws.example.com", filesAPIEndpoint, entry("https", "proxy example.com"), ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := newEngine(&Client{internalClient: internalClient{host: tc.workspaceHost}}) + ep, ok := e.parseEndpoint(tc.entry, tc.kind) + if tc.wantBase == "" { + if ok { + t.Errorf("parseEndpoint = %v, want the entry rejected", ep) + } + return + } + if !ok || ep.baseURL != tc.wantBase || ep.kind != tc.kind || !ep.multipart { + t.Errorf("parseEndpoint = (%v, %v), want %s with multipart", ep, ok, tc.wantBase) + } + }) + } +} diff --git a/files/v2/ext_endpoints.go b/files/v2/ext_endpoints.go new file mode 100644 index 00000000..11872c94 --- /dev/null +++ b/files/v2/ext_endpoints.go @@ -0,0 +1,313 @@ +package files + +// The endpoints an upload can go to, and the walk that tries them in order: +// the next endpoint gets the upload only if the previous one failed before any +// part of it landed. + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/databricks/sdk-go/auth" + "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/apierr/codes" +) + +// endpointKind is a transfer protocol this package speaks. +type endpointKind int + +const ( + // presignedEndpoint (IO_OFFLOAD) sends parts to presigned cloud URLs, + // without credentials. The upload session and the URLs come from Files API + // 2.0 on the endpoint's host. It has no single-shot upload. + presignedEndpoint endpointKind = iota + // filesAPIEndpoint (FILES_API 2.0) sends the data to the endpoint's host, + // with credentials: the workspace, or the storage proxy. + filesAPIEndpoint + // filesAPIv3Endpoint (FILES_API 3.0) is the 2.0 protocol under /api/3.0, + // with an upload_id for the session token and parts of at most + // filesAPIv3MaxPartSize. It is reached with credentials over the internet, + // so only over https. + filesAPIv3Endpoint +) + +// filesAPIv3MaxPartSize is the largest part Files API 3.0 takes, below the +// largest part size an upload may pick. +const filesAPIv3MaxPartSize int64 = 4000 << 20 + +func (k endpointKind) String() string { + switch k { + case presignedEndpoint: + return "IO_OFFLOAD" + case filesAPIEndpoint: + return "FILES_API 2.0" + case filesAPIv3Endpoint: + return "FILES_API 3.0" + } + return fmt.Sprintf("endpointKind(%d)", int(k)) +} + +// workspaceMaxSingleShotSize is the largest single-shot upload the workspace's +// Files API takes. +const workspaceMaxSingleShotSize int64 = 5 << 30 + +// endpoint is one entry of the ordered list an upload walks, best first. +type endpoint struct { + kind endpointKind + // baseURL is the scheme and host, such as https://example.cloud.databricks.com. + baseURL string + multipart bool +} + +func (ep endpoint) String() string { return ep.kind.String() + " at " + ep.baseURL } + +// filesPath is the path of a file in the endpoint's Files API. A presigned +// endpoint opens and completes its sessions through Files API 2.0. +func (ep endpoint) filesPath(absPath string) string { + if ep.kind == filesAPIv3Endpoint { + return "/api/3.0/fs/files" + absPath + } + return filesAPIPath(absPath) +} + +// maxPartSize caps the part size of a multipart upload to the endpoint, or is +// -1 for no cap. +func (ep endpoint) maxPartSize() int64 { + if ep.kind == filesAPIv3Endpoint { + return filesAPIv3MaxPartSize + } + return -1 +} + +// sessionQuery returns the query that names a multipart session on the +// endpoint: Files API 3.0 calls it an upload_id. +func (ep endpoint) sessionQuery(token string) url.Values { + if ep.kind == filesAPIv3Endpoint { + return url.Values{"upload_id": {token}} + } + return url.Values{"upload_type": {"multipart"}, "session_token": {token}} +} + +// builtinEndpoints is the order used without endpoint discovery: presigned +// URLs, then the workspace's Files API, which takes uploads only in a single +// shot. +func (e *engine) builtinEndpoints() []endpoint { + return []endpoint{ + {kind: presignedEndpoint, baseURL: e.c.host, multipart: true}, + {kind: filesAPIEndpoint, baseURL: e.c.host}, + } +} + +// uploadEndpoints returns the endpoints an upload walks, in order. A list of +// only presigned URLs gets the workspace's Files API, which every workspace +// serves: small and empty files need a single shot, which presigned URLs +// can't take. +func (e *engine) uploadEndpoints(ctx context.Context) []endpoint { + endpoints := e.transferEndpoints(ctx) + if !slices.ContainsFunc(endpoints, func(ep endpoint) bool { return ep.kind != presignedEndpoint }) { + endpoints = append(endpoints, endpoint{kind: filesAPIEndpoint, baseURL: e.c.host}) + } + return endpoints +} + +// onWorkspace reports whether ep is the workspace host the client is +// configured with. Requests there keep the API client's retries, until the +// retry timeout; other hosts get bounded retries, so that a failing host gives +// way to the next endpoint quickly. +func (e *engine) onWorkspace(ep endpoint) bool { + return sameOrigin(ep.baseURL, e.c.host) +} + +// sameOrigin reports whether two URLs have the same scheme, host and port, with +// the scheme's default port filled in. +func sameOrigin(a, b string) bool { + oa, erra := originOf(a) + ob, errb := originOf(b) + return erra == nil && errb == nil && oa == ob +} + +type origin struct{ scheme, host, port string } + +func originOf(raw string) (origin, error) { + u, err := url.Parse(raw) + if err != nil { + return origin{}, err + } + scheme := strings.ToLower(u.Scheme) + port := u.Port() + if port == "" { + port = map[string]string{"http": "80", "https": "443"}[scheme] + } + return origin{scheme: scheme, host: strings.ToLower(u.Hostname()), port: port}, nil +} + +// endpointPlan is how an upload goes to one endpoint. +type endpointPlan struct { + ep endpoint + // multipart is a multipart upload session; otherwise one single-shot PUT. + multipart bool +} + +// planUpload returns how the data goes to ep, or false if ep can't take it: +// presigned URLs have no single-shot upload, and the workspace takes a single +// shot of at most workspaceMaxSingleShotSize. Data below the multipart +// threshold, or to an endpoint without multipart, goes in a single shot. +func (e *engine) planUpload(ep endpoint, size int64) (endpointPlan, bool) { + small := size >= 0 && size < max(e.tun.minStreamSize, 1) + if !small && ep.multipart { + return endpointPlan{ep: ep, multipart: true}, true + } + if ep.kind == presignedEndpoint { + return endpointPlan{}, false + } + if e.onWorkspace(ep) && size > workspaceMaxSingleShotSize { + return endpointPlan{}, false + } + return endpointPlan{ep: ep}, true +} + +// walkUpload sends the data to each endpoint in turn until one takes it. It +// moves on only while no part has landed, the failure is one the next +// endpoint may not hit (see walksOn), and the data can be sent again from the +// start; otherwise it returns the failure. +func (e *engine) walkUpload(ctx context.Context, uc *uploadContext, data *uploadData) error { + // A one-pass stream learns its size only once it is read: read up to the + // multipart threshold, so a short one is planned as a single shot. + if err := data.readAhead(e.tun.minStreamSize); err != nil { + return err + } + var plans []endpointPlan + var tooLarge []string + for _, ep := range e.uploadEndpoints(ctx) { + if plan, ok := e.planUpload(ep, data.size()); ok { + plans = append(plans, plan) + } else if ep.kind != presignedEndpoint { + tooLarge = append(tooLarge, ep.String()) + } + } + if len(plans) == 0 { + return fmt.Errorf("no endpoint can take this upload of %d bytes: %s takes a single shot of at most %d bytes", + data.size(), strings.Join(tooLarge, ", "), workspaceMaxSingleShotSize) + } + for i, plan := range plans { + uc.startEndpoint(plan.ep, i == len(plans)-1) + err := e.uploadTo(ctx, uc, plan, data) + if err == nil { + return nil + } + if uc.committed() || uc.lastEndpoint || !walksOn(ctx, err) || !data.canSendAgain() { + return err + } + e.c.logger.InfoContext(ctx, "upload failed before any part landed, trying the next endpoint", + "endpoint", plan.ep.String(), "error", err) + if err := data.restart(); err != nil { + return err + } + } + panic("unreachable: the last endpoint returns") +} + +// walksOn reports whether a failure before any part has landed moves the +// upload to the next endpoint. Any failure does once its retries are spent, +// as the endpoint may be unreachable or blocked from here, except one the next +// endpoint would hit too (the file exists, or a precondition failed) and the +// caller's own cancellation. +func walksOn(ctx context.Context, err error) bool { + if ctx.Err() != nil || errors.Is(err, ErrAlreadyExists) { + return false + } + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + return aerr.Code() != codes.AlreadyExists && aerr.HTTPStatusCode() != http.StatusPreconditionFailed + } + return true +} + +// uploadTo sends the data to one endpoint with its plan. A multipart session +// that fails is aborted once. +func (e *engine) uploadTo(ctx context.Context, uc *uploadContext, plan endpointPlan, data *uploadData) error { + if !plan.multipart { + return e.singleShotUpload(ctx, uc, data.whole(), data.size()) + } + token, err := e.initiateMultipart(ctx, uc) + if err != nil { + return err + } + switch { + case uc.parallel && data.ra != nil: + // Parts stream from positioned reads of the source, without buffering. + err = e.parallelMultipartFromReaderAt(ctx, uc, token, data.ra) + case uc.parallel: + err = e.parallelMultipartFromStream(ctx, uc, token, data.r) + default: + err = e.performMultipartUpload(ctx, uc, token, data.r) + } + if err != nil { + e.abortMultipartBestEffort(ctx, uc, token) + } + return err +} + +// filesAPIPartURLs returns the URLs of parts sent straight to a Files API +// host, with credentials: unlike presigned URLs, nothing is minted or expires. +func filesAPIPartURLs(uc *uploadContext, token string, startPart, count int) ([]presignedURL, error) { + urls := make([]presignedURL, 0, count) + for n := startPart; n < startPart+count; n++ { + query := uc.ep.sessionQuery(token) + query.Set("part_number", strconv.Itoa(n)) + u, err := endpointURL(uc.ep.baseURL, uc.ep.filesPath(uc.targetPath), query) + if err != nil { + return nil, err + } + urls = append(urls, presignedURL{URL: u, PartNumber: n}) + } + return urls, nil +} + +// authenticatedHTTPClient returns the HTTP client for parts sent to a Files API +// host, such as the storage proxy: the client's own transport, as for any +// Databricks host, sized like the transfer client for n concurrent parts, with +// the client's credentials on every request. A transport that isn't an +// *http.Transport is used as is. It sets no whole-request timeout, which would +// abort a long transfer. +func (e *engine) authenticatedHTTPClient(n int) *http.Client { + base := http.DefaultTransport + if e.c.httpClient != nil && e.c.httpClient.Transport != nil { + base = e.c.httpClient.Transport + } + if t, ok := base.(*http.Transport); ok { + base = sizedForTransfers(t, n, e.tun) + } + return &http.Client{Transport: &credentialTransport{base: base, creds: e.c.credentials}} +} + +// credentialTransport adds credentials to every request it sends. Presigned +// URLs never go through it. +type credentialTransport struct { + base http.RoundTripper + creds auth.Credentials +} + +func (t *credentialTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.creds == nil { + return t.base.RoundTrip(req) + } + headers, err := t.creds.AuthHeaders(req.Context()) + if err != nil { + if req.Body != nil { + _ = req.Body.Close() + } + return nil, err + } + // A RoundTripper must not modify the request it is given. + req = req.Clone(req.Context()) + for _, h := range headers { + req.Header.Add(h.Key, h.Value) + } + return t.base.RoundTrip(req) +} diff --git a/files/v2/ext_endpoints_test.go b/files/v2/ext_endpoints_test.go new file mode 100644 index 00000000..648b2319 --- /dev/null +++ b/files/v2/ext_endpoints_test.go @@ -0,0 +1,394 @@ +package files + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/databricks/sdk-go/auth" +) + +func TestPlanUpload(t *testing.T) { + e := newEngine(&Client{internalClient: internalClient{host: "https://ws.example.com"}}) + e.tun.minStreamSize = 1024 + presigned := endpoint{kind: presignedEndpoint, baseURL: "https://ws.example.com", multipart: true} + workspace := endpoint{kind: filesAPIEndpoint, baseURL: "https://ws.example.com"} + proxy := endpoint{kind: filesAPIEndpoint, baseURL: "http://proxy.example.com", multipart: true} + tests := []struct { + name string + ep endpoint + size int64 + wantOK bool + wantMultipart bool + }{ + {"presigned takes a large upload in parts", presigned, 4096, true, true}, + {"presigned takes a stream of unknown size in parts", presigned, -1, true, true}, + {"presigned has no single shot for a small upload", presigned, 100, false, false}, + {"presigned has no single shot for an empty upload", presigned, 0, false, false}, + {"the workspace takes a large upload in a single shot", workspace, 4096, true, false}, + {"the workspace takes a stream of unknown size in a single shot", workspace, -1, true, false}, + {"the workspace takes up to 5 GiB in a single shot", workspace, workspaceMaxSingleShotSize, true, false}, + {"the workspace takes no more than 5 GiB in a single shot", workspace, workspaceMaxSingleShotSize + 1, false, false}, + {"another host has no single-shot limit", proxy.withoutMultipart(), workspaceMaxSingleShotSize + 1, true, false}, + {"a small upload goes in a single shot where multipart is offered", proxy, 100, true, false}, + {"an upload at the threshold goes in parts", proxy, 1024, true, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + plan, ok := e.planUpload(tc.ep, tc.size) + if ok != tc.wantOK || plan.multipart != tc.wantMultipart { + t.Errorf("planUpload = (multipart %v, %v), want (multipart %v, %v)", plan.multipart, ok, tc.wantMultipart, tc.wantOK) + } + }) + } +} + +func (ep endpoint) withoutMultipart() endpoint { + ep.multipart = false + return ep +} + +func TestUploadContextRetries(t *testing.T) { + tun := defaultTunables() + tests := []struct { + name string + committed bool + last bool + onWorkspace bool + want int + }{ + {"before commit with an endpoint left", false, false, false, tun.cloudRetriesBeforeCommit}, + {"before commit with an endpoint left, on the workspace", false, false, true, tun.cloudRetriesBeforeCommit}, + {"on the last endpoint", false, true, false, tun.cloudRetries}, + {"on the last endpoint, on the workspace", false, true, true, -1}, + {"once committed", true, false, false, tun.cloudRetries}, + {"once committed, on the workspace", true, false, true, -1}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + uc := &uploadContext{tun: tun} + uc.startEndpoint(endpoint{}, tc.last) + if tc.committed { + uc.commit() + } + if got := uc.retries(tc.onWorkspace); got != tc.want { + t.Errorf("retries = %d, want %d", got, tc.want) + } + }) + } +} + +func TestSameOrigin(t *testing.T) { + tests := []struct { + a, b string + want bool + }{ + {"https://ws.example.com", "https://ws.example.com", true}, + {"https://WS.example.com", "https://ws.example.com/", true}, + {"https://ws.example.com:443", "https://ws.example.com", true}, + {"http://localhost", "http://localhost:80", true}, + {"http://ws.example.com", "https://ws.example.com", false}, + {"https://ws.example.com:8443", "https://ws.example.com", false}, + {"https://proxy.example.com", "https://ws.example.com", false}, + {"://bad", "://bad", false}, + } + for _, tc := range tests { + if got := sameOrigin(tc.a, tc.b); got != tc.want { + t.Errorf("sameOrigin(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + } +} + +func readAllString(t *testing.T, r io.Reader) string { + t.Helper() + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestReplayStreamShortStream(t *testing.T) { + // A stream that ends within the read-ahead learns its size, and its single + // shot is a bytes.Reader that can be sent again. + d := newStreamData(io.MultiReader(bytes.NewReader([]byte("hello")))) + noErr(t, d.readAhead(10)) + if d.size() != 5 { + t.Fatalf("size = %d, want 5", d.size()) + } + body := d.whole() + if _, ok := body.(io.Seeker); !ok { + t.Errorf("whole() of an ended stream is %T, want a seekable reader", body) + } + if got := readAllString(t, body); got != "hello" { + t.Errorf("whole() = %q, want hello", got) + } + if !d.canSendAgain() { + t.Fatal("an ended stream can't be sent again") + } + noErr(t, d.restart()) + if got := readAllString(t, d.whole()); got != "hello" { + t.Errorf("whole() after restart = %q, want hello", got) + } +} + +func TestReplayStreamRestartBeforeCommit(t *testing.T) { + // Until commit, every byte read is kept, so the stream starts again from its + // first byte. + d := newStreamData(io.MultiReader(bytes.NewReader([]byte("abcdefghij")))) + noErr(t, d.readAhead(4)) + if d.size() != -1 { + t.Errorf("size = %d, want -1 before the stream ends", d.size()) + } + first := make([]byte, 6) + if _, err := io.ReadFull(d.r, first); err != nil { + t.Fatal(err) + } + if !d.canSendAgain() { + t.Fatal("can't send again before commit") + } + noErr(t, d.restart()) + if got := readAllString(t, d.r); got != "abcdefghij" { + t.Errorf("after restart = %q, want the whole stream", got) + } +} + +func TestReplayStreamCommit(t *testing.T) { + // Commit drops what was handed out, keeps what was read ahead but not yet + // handed out, and stops keeping: the stream can't start again. + s := &replayStream{src: io.MultiReader(bytes.NewReader([]byte("abcdefghij"))), keeping: true} + d := &uploadData{r: s, stream: s, knownSize: -1} + noErr(t, d.readAhead(6)) + part := make([]byte, 3) + if _, err := io.ReadFull(d.r, part); err != nil { + t.Fatal(err) + } + d.commit() + if d.canSendAgain() { + t.Error("can send again after commit") + } + if got := len(s.kept); got != 3 { + t.Errorf("kept %d bytes after commit, want the 3 read ahead", got) + } + if got := readAllString(t, d.r); got != "defghij" { + t.Errorf("after commit = %q, want the rest", got) + } + if s.kept != nil { + t.Errorf("kept %d bytes once handed out, want none", len(s.kept)) + } +} + +func TestReplayStreamWholeOfALongStream(t *testing.T) { + // A single shot of a stream that did not end within what was kept streams + // the rest, so the stream can't be sent again. + d := newStreamData(io.MultiReader(bytes.NewReader([]byte("abcdefghij")))) + noErr(t, d.readAhead(4)) + if got := readAllString(t, d.whole()); got != "abcdefghij" { + t.Errorf("whole() = %q, want the whole stream", got) + } + if d.canSendAgain() { + t.Error("can send again after streaming the rest") + } +} + +func TestReplayStreamReadAheadError(t *testing.T) { + boom := errors.New("boom") + d := newStreamData(io.MultiReader(bytes.NewReader([]byte("ab")), errReader{boom})) + if err := d.readAhead(10); !errors.Is(err, boom) { + t.Errorf("readAhead err = %v, want %v", err, boom) + } +} + +type errReader struct{ err error } + +func (r errReader) Read([]byte) (int, error) { return 0, r.err } + +func TestStorageProxyPreconditionFailedIsAlreadyExists(t *testing.T) { + // The storage proxy answers an existing file with a 412 where the workspace + // sends 409 ALREADY_EXISTS. Only an upload that doesn't overwrite maps it, + // whichever request gets it. + no, yes := false, true + cases := []struct { + name string + otherHost bool + overwrite *bool + want bool + }{ + {"the storage proxy without overwrite", true, &no, true}, + {"the storage proxy with overwrite", true, &yes, false}, + {"the storage proxy with the server's default", true, nil, false}, + {"the workspace without overwrite", false, &no, false}, + } + requests := map[string]func(*engine, *uploadContext) error{ + "single shot": func(e *engine, uc *uploadContext) error { + return e.uploadSingleShot(t.Context(), uc, strings.NewReader("hello"), 5) + }, + "initiate": func(e *engine, uc *uploadContext) error { + _, err := e.initiateUpload(t.Context(), uc) + return err + }, + "complete": func(e *engine, uc *uploadContext) error { + return e.completeMultipart(t.Context(), uc, "tok", map[int]string{1: "e1"}) + }, + } + for _, tc := range cases { + for name, send := range requests { + t.Run(tc.name+", "+name, func(t *testing.T) { + c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusPreconditionFailed) + }) + uc := controlContext(c, tc.overwrite) + if tc.otherHost { + // The same test server under another name is another origin. + uc.ep = endpoint{kind: filesAPIEndpoint, baseURL: strings.Replace(c.c.host, "127.0.0.1", "localhost", 1), multipart: true} + } + err := send(c, uc) + if err == nil { + t.Fatal("want an error") + } + if got := errors.Is(err, ErrAlreadyExists); got != tc.want { + t.Errorf("errors.Is(%v, ErrAlreadyExists) = %v, want %v", err, got, tc.want) + } + }) + } + } +} + +func TestAuthenticatedHTTPClientIsSizedForTheParts(t *testing.T) { + // Parts to a Files API host go over the client's transport, sized like the + // transfer client so that parallel parts reuse their connections. + own := &http.Transport{MaxIdleConnsPerHost: 2} + e := newEngine(&Client{internalClient: internalClient{httpClient: &http.Client{Transport: own}}}) + ct, ok := e.authenticatedHTTPClient(16).Transport.(*credentialTransport) + if !ok { + t.Fatalf("transport = %T, want a credentialTransport", e.authenticatedHTTPClient(16).Transport) + } + sized, ok := ct.base.(*http.Transport) + if !ok || sized == own || sized.MaxIdleConnsPerHost != 16 || sized.ResponseHeaderTimeout != e.tun.responseHeaderTimeout { + t.Errorf("base = %#v, want a copy of the client's transport sized for 16 parts", ct.base) + } + if own.MaxIdleConnsPerHost != 2 { + t.Error("the client's own transport was changed") + } + + // Any other RoundTripper, such as a test's, is used as is. + custom := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, errors.New("unused") }) + e = newEngine(&Client{internalClient: internalClient{httpClient: &http.Client{Transport: custom}}}) + if ct := e.authenticatedHTTPClient(16).Transport.(*credentialTransport); ct.base == nil { + t.Error("want the client's RoundTripper") + } +} + +type staticCredentials struct { + headers []auth.Header + err error +} + +func (staticCredentials) Name() string { return "static" } + +func (c staticCredentials) AuthHeaders(context.Context) ([]auth.Header, error) { + return c.headers, c.err +} + +func TestCredentialTransport(t *testing.T) { + var seen http.Header + base := roundTripFunc(func(r *http.Request) (*http.Response, error) { + seen = r.Header.Clone() + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: r}, nil + }) + rt := &credentialTransport{base: base, creds: staticCredentials{headers: []auth.Header{{Key: "Authorization", Value: "Bearer t"}}}} + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, "http://proxy.example.com/x", nil) + noErr(t, err) + _, err = rt.RoundTrip(req) + noErr(t, err) + if got := seen.Get("Authorization"); got != "Bearer t" { + t.Errorf("sent Authorization = %q, want the credentials", got) + } + if req.Header.Get("Authorization") != "" { + t.Error("the caller's request was modified") + } + + // A failure to get credentials sends nothing, and closes the body as a + // RoundTripper must. + boom := errors.New("boom") + body := &closeRecorder{Reader: strings.NewReader("x")} + req, err = http.NewRequestWithContext(t.Context(), http.MethodPut, "http://proxy.example.com/x", body) + noErr(t, err) + seen = nil + rt.creds = staticCredentials{err: boom} + if _, err := rt.RoundTrip(req); !errors.Is(err, boom) { + t.Errorf("err = %v, want %v", err, boom) + } + if seen != nil || !body.closed.Load() { + t.Errorf("sent = %v, body closed = %v; want nothing sent and the body closed", seen != nil, body.closed.Load()) + } +} + +func TestFilesAPIv3Protocol(t *testing.T) { + // Files API 3.0 is the 2.0 protocol under /api/3.0, naming the session by + // upload_id, with parts of at most 4000 MiB. + v2 := endpoint{kind: filesAPIEndpoint, baseURL: "http://proxy.example.com", multipart: true} + v3 := endpoint{kind: filesAPIv3Endpoint, baseURL: "https://rig.example.com", multipart: true} + if got := v3.filesPath("/Volumes/c/s/v/f.bin"); got != "/api/3.0/fs/files/Volumes/c/s/v/f.bin" { + t.Errorf("3.0 path = %q", got) + } + if got := v2.filesPath("/Volumes/c/s/v/f.bin"); got != "/api/2.0/fs/files/Volumes/c/s/v/f.bin" { + t.Errorf("2.0 path = %q", got) + } + if got := v3.sessionQuery("u1").Encode(); got != "upload_id=u1" { + t.Errorf("3.0 session query = %q", got) + } + if got := v2.sessionQuery("t1").Encode(); got != "session_token=t1&upload_type=multipart" { + t.Errorf("2.0 session query = %q", got) + } + + uc := &uploadContext{tun: defaultTunables(), targetPath: "/Volumes/c/s/v/f.bin", basePartSize: 4 << 30} + uc.startEndpoint(v3, false) + if uc.partSize != filesAPIv3MaxPartSize { + t.Errorf("3.0 part size = %d, want %d", uc.partSize, filesAPIv3MaxPartSize) + } + urls, err := filesAPIPartURLs(uc, "u1", 2, 2) + noErr(t, err) + if len(urls) != 2 || urls[0].URL != "https://rig.example.com/api/3.0/fs/files/Volumes/c/s/v/f.bin?part_number=2&upload_id=u1" || urls[1].PartNumber != 3 { + t.Errorf("3.0 part URLs = %+v", urls) + } + uc.startEndpoint(v2, false) + if uc.partSize != 4<<30 { + t.Errorf("2.0 part size = %d, want the upload's %d", uc.partSize, int64(4<<30)) + } +} + +func TestFilesAPIv3InitiateNeedsAnUploadID(t *testing.T) { + for name, body := range map[string]string{ + "an upload_id": `{"upload_id":"u1"}`, + "no upload_id": `{}`, + "a 2.0 session": `{"multipart_upload":{"session_token":"t1"}}`, + } { + t.Run(name, func(t *testing.T) { + c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/3.0/fs/files/Volumes/c/s/v/f.bin" || r.URL.Query().Get("action") != "initiate-upload" { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + } + _, _ = io.WriteString(w, body) + }) + uc := controlContext(c, nil) + uc.startEndpoint(endpoint{kind: filesAPIv3Endpoint, baseURL: c.c.host, multipart: true}, true) + token, err := c.initiateMultipart(t.Context(), uc) + if name == "an upload_id" { + noErr(t, err) + if token != "u1" { + t.Errorf("token = %q, want u1", token) + } + return + } + if !errors.Is(err, errUnexpectedServerResponse) { + t.Errorf("err = %v, want errUnexpectedServerResponse", err) + } + }) + } +} diff --git a/files/v2/ext_limiter.go b/files/v2/ext_limiter.go index e485e617..805e8da1 100644 --- a/files/v2/ext_limiter.go +++ b/files/v2/ext_limiter.go @@ -3,10 +3,10 @@ package files import "context" // Limiter bounds the number of concurrent cloud-leg transfers an upload may run. -// The engine acquires one unit before each transfer (every multipart part PUT, -// the single-shot PUT, and every resumable chunk) and releases it when that -// transfer returns. Pass the same Limiter to multiple Upload calls -- for example -// when copying many files at once -- to cap their combined concurrency. +// The engine acquires one unit before each transfer (every multipart part PUT +// and the single-shot PUT) and releases it when that transfer returns. Pass the +// same Limiter to multiple Upload calls -- for example when copying many files +// at once -- to cap their combined concurrency. // Implementations must be safe for concurrent use. type Limiter interface { // Acquire blocks until a slot is free or ctx is cancelled; on cancellation it diff --git a/files/v2/ext_limiter_test.go b/files/v2/ext_limiter_test.go index e695ffe2..54dde15a 100644 --- a/files/v2/ext_limiter_test.go +++ b/files/v2/ext_limiter_test.go @@ -141,27 +141,6 @@ func TestSingleShotAcquiresOneSlot(t *testing.T) { } } -func TestResumableAcquiresPerChunk(t *testing.T) { - f := newFakeServer(t, "resumable") - e := shrunkEngine(t, f) - - lim := newCountingLimiter(4) - in := data(4 * 1024) // several resumable chunks - _, err := e.upload(t.Context(), "/Volumes/c/s/v/r.bin", bytes.NewReader(in), WithLimiter(lim)) - noErr(t, err) - eqBytes(t, f.assembled(), in) - - if lim.acquires < 2 { - t.Errorf("resumable acquires = %d, want >= 2 (one per chunk)", lim.acquires) - } - if lim.acquires != lim.releases { - t.Errorf("unbalanced: %d acquires, %d releases", lim.acquires, lim.releases) - } - if lim.max != 1 { - t.Errorf("resumable is sequential; peak concurrency = %d, want 1", lim.max) - } -} - func TestLimiterCancelNoLeak(t *testing.T) { f := newFakeServer(t, "multipart") slowParts(f, 50*time.Millisecond) diff --git a/files/v2/ext_multipart.go b/files/v2/ext_multipart.go index fb21d8e8..e2671568 100644 --- a/files/v2/ext_multipart.go +++ b/files/v2/ext_multipart.go @@ -8,14 +8,12 @@ package files import ( "context" "errors" - "fmt" "io" "maps" "net/http" "sync" "time" - "github.com/databricks/sdk-go/core/apierr" "github.com/databricks/sdk-go/files/v2/internal/cloudstorage" ) @@ -31,25 +29,15 @@ type uploadPart struct { headers map[string]string // PUT headers accompanying url } -// isCloudStatus reports whether err is a cloud-storage HTTP status error (an -// *apierr.APIError returned by Send) rather than a transport failure. A status -// rejection before any part has landed triggers the single-shot fallback; a -// transport failure does not. -func isCloudStatus(err error) bool { - _, ok := errors.AsType[*apierr.APIError](err) - return ok -} - // performMultipartUpload uploads a stream in parts on a single goroutine using // presigned URLs (AWS/Azure). -func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, token string, r io.Reader, preRead []byte) error { +func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, token string, r io.Reader) error { currentPart := 1 etags := map[int]string{} chunkOffset := int64(0) // AWS and Azure require each part size up front, so we buffer a part before - // uploading it; this also lets us replay a part on retry. The buffer starts - // with the bytes already read while deciding single-shot vs multipart. - buffer := preRead + // uploading it; this also lets us replay a part on retry. + var buffer []byte retryCount := 0 eof := false @@ -62,11 +50,8 @@ func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, break } - urls, err := e.createPartURLs(ctx, uc.targetPath, token, currentPart, uc.batchSize) + urls, err := e.createPartURLs(ctx, uc, token, currentPart, uc.batchSize) if err != nil { - if chunkOffset == 0 { - return &fallbackToFilesAPI{buffer: buffer, reason: fmt.Sprintf("failed to obtain upload URLs: %v", err)} - } return err } @@ -85,10 +70,11 @@ func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, if err := uc.limiter.Acquire(ctx); err != nil { return err } - resp, rerr := uc.cloud.Send(ctx, http.MethodPut, purl.URL, headers, cloudstorage.BytesBody(buffer[:chunkLen])) + resp, rerr := uc.partClient().Send(ctx, http.MethodPut, purl.URL, headers, cloudstorage.BytesBody(buffer[:chunkLen]), uc.partRetries()) uc.limiter.Release() switch { case rerr == nil: + uc.commit() chunkOffset += chunkLen etags[currentPart] = resp.Header.Get("ETag") buffer = buffer[chunkLen:] @@ -104,10 +90,6 @@ func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, retryCount++ // Preserve the buffer: the same bytes are uploaded under the next // part number using a fresh URL. - case chunkOffset == 0 && isCloudStatus(rerr): - // The cloud rejected the first chunk (e.g. a firewall 403); fall back - // to a single-shot upload through the Files API. - return &fallbackToFilesAPI{buffer: buffer, reason: fmt.Sprintf("first chunk upload failed: %v", rerr)} default: return rerr } @@ -115,30 +97,23 @@ func (e *engine) performMultipartUpload(ctx context.Context, uc *uploadContext, } } - return e.completeMultipart(ctx, uc.targetPath, token, etags) + return e.completeMultipart(ctx, uc, token, etags) } // doUploadOnePart uploads a single part. It uses part.url when the caller // pre-minted one (the file path mints in batches); otherwise, and whenever a URL -// expires or a slow attempt is re-issued, it mints a fresh one (count=1). While no -// part has completed yet, it converts a cloud rejection or URL-mint failure into a -// fallback signal so the caller can retry the whole upload through a single-shot -// Files API PUT (the firewall/auth case); once any part has landed, the storage -// endpoint is known good and such a failure is returned as a real error. +// expires or a slow attempt is re-issued, it mints a fresh one (count=1). // // isFirstPart marks the stream path's synchronous, uncontended first part, which -// gets its own tight soft deadline; it does not affect the fallback gating above. +// gets its own tight soft deadline. func (e *engine) doUploadOnePart(ctx context.Context, uc *uploadContext, part uploadPart, token string, isFirstPart bool) (string, error) { start := time.Now() urlRetries, slowRetries := 0, 0 url, headers := part.url, part.headers for { if url == "" { - urls, err := e.createPartURLs(ctx, uc.targetPath, token, part.partNumber, 1) + urls, err := e.createPartURLs(ctx, uc, token, part.partNumber, 1) if err != nil { - if !uc.completed.Load() { - return "", &fallbackToFilesAPI{reason: fmt.Sprintf("failed to obtain upload URL for part %d: %v", part.partNumber, err)} - } return "", err } url, headers = urls[0].URL, octetStreamHeaders(urls[0].Headers) @@ -150,7 +125,7 @@ func (e *engine) doUploadOnePart(ctx context.Context, uc *uploadContext, part up }) switch { case rerr == nil: - uc.completed.Store(true) + uc.commit() uc.slowGuard.record(time.Since(attemptStart)) uc.progress.add(part.body.Size()) e.c.logger.DebugContext(ctx, "part uploaded", @@ -171,11 +146,6 @@ func (e *engine) doUploadOnePart(ctx context.Context, uc *uploadContext, part up } urlRetries++ url = "" // re-mint the expired URL - case isCloudStatus(rerr) && !uc.completed.Load(): - // The cloud rejected this part before any part has landed (e.g. a firewall - // 403 on the storage host); fall back to a single-shot upload through the - // Files API. - return "", &fallbackToFilesAPI{reason: fmt.Sprintf("part %d upload failed before any part completed: %v", part.partNumber, rerr)} default: return "", rerr } @@ -220,7 +190,7 @@ func (e *engine) sendPart(ctx context.Context, uc *uploadContext, url string, he } }() - resp, err = uc.cloud.Send(attemptCtx, http.MethodPut, url, headers, body) + resp, err = uc.partClient().Send(attemptCtx, http.MethodPut, url, headers, body, uc.partRetries()) slow = ctx.Err() == nil && errors.Is(context.Cause(attemptCtx), errSlowAttempt) return resp, slow, err } @@ -261,26 +231,24 @@ func (e *engine) startPartWorkers(pctx context.Context, cancel context.CancelFun } // parallelMultipartFromStream uploads a non-seekable stream in parts using a -// bounded producer/consumer. The first part is uploaded synchronously so an -// auth/firewall rejection surfaces before workers spin up and can fall back. +// bounded producer/consumer. The first part is uploaded synchronously, so a +// rejection surfaces before workers spin up, and a stream keeps no more than the +// read-ahead and that part for the walk to send again. func (e *engine) parallelMultipartFromStream(ctx context.Context, uc *uploadContext, token string, r io.Reader) error { first, err := readUpTo(r, uc.partSize) if err != nil { return err } if len(first) == 0 { - return &fallbackToFilesAPI{buffer: nil, reason: "empty input stream"} + // planUpload sends empty data in a single shot. + return errors.New("a multipart upload was planned for empty data") } etag1, err := e.doUploadOnePart(ctx, uc, uploadPart{partNumber: 1, body: cloudstorage.BytesBody(first)}, token, true) if err != nil { - if fb, ok := errors.AsType[*fallbackToFilesAPI](err); ok { - fb.buffer = first - return fb - } return err } if int64(len(first)) < uc.partSize { - return e.completeMultipart(ctx, uc.targetPath, token, map[int]string{1: etag1}) + return e.completeMultipart(ctx, uc, token, map[int]string{1: etag1}) } pctx, cancel := context.WithCancel(ctx) @@ -333,7 +301,7 @@ func (e *engine) parallelMultipartFromStream(ctx context.Context, uc *uploadCont if err != nil { return err } - return e.completeMultipart(ctx, uc.targetPath, token, etags) + return e.completeMultipart(ctx, uc, token, etags) } // parallelMultipartFromReaderAt uploads a known-size, randomly-readable source in @@ -344,10 +312,9 @@ func (e *engine) parallelMultipartFromStream(ctx context.Context, uc *uploadCont // first). // // Unlike the stream path, the first part is not uploaded synchronously: a -// randomly-readable source lets the single-shot fallback re-read from the start, -// so part 1 is just another worker task rather than a canary that blocks the -// whole upload at 0%. The fallback is instead gated on no part having completed -// (see doUploadOnePart). +// randomly-readable source can be sent again from the start to the next +// endpoint, so part 1 is just another worker task rather than a canary that +// blocks the whole upload at 0%. func (e *engine) parallelMultipartFromReaderAt(ctx context.Context, uc *uploadContext, token string, ra io.ReaderAt) error { fileSize := uc.contentLength partSize := uc.partSize @@ -375,16 +342,12 @@ func (e *engine) parallelMultipartFromReaderAt(ctx context.Context, uc *uploadCo return } count := min(uc.batchSize, numParts-next+1) - urls, err := e.createPartURLs(pctx, uc.targetPath, token, next, count) + urls, err := e.createPartURLs(pctx, uc, token, next, count) if err != nil { if pctx.Err() != nil { return // cancelled by a worker failure; that error wins } - if uc.completed.Load() { - producerErr = err - } else { - producerErr = &fallbackToFilesAPI{reason: fmt.Sprintf("failed to obtain upload URLs at part %d: %v", next, err)} - } + producerErr = err cancel() return } @@ -406,12 +369,9 @@ func (e *engine) parallelMultipartFromReaderAt(ctx context.Context, uc *uploadCo err = producerErr // a producer failure wins over a worker's observed cancellation } if err != nil { - if fb, ok := errors.AsType[*fallbackToFilesAPI](err); ok { - return fb // the fallback re-reads the source, so no buffer is needed - } return err } - return e.completeMultipart(ctx, uc.targetPath, token, etags) + return e.completeMultipart(ctx, uc, token, etags) } // uploadResults collects part ETags from workers and records the first error. diff --git a/files/v2/ext_port_contract_test.go b/files/v2/ext_port_contract_test.go index a84b9642..06f6751e 100644 --- a/files/v2/ext_port_contract_test.go +++ b/files/v2/ext_port_contract_test.go @@ -39,6 +39,7 @@ import ( "github.com/databricks/sdk-go/core/apierr" "github.com/databricks/sdk-go/core/apierr/codes" "github.com/databricks/sdk-go/core/ops" + "github.com/databricks/sdk-go/files/v2/internal/cloudstorage" "github.com/databricks/sdk-go/options/call" ) @@ -181,6 +182,7 @@ func runScenario(t *testing.T, h contractHeader, s contractScenario, wantOutcome srv := newPortFake(cfg) e := newEngine(buildUploadClient(t, "http://"+k.WorkspaceHost, srv.Client(), "")) e.tun = contractTunables(e.tun, k.PartSize) + e.tun.endpointsAPI = h.EndpointLists[s.EndpointList].EndpointsAPI != nil ctx, cancel := context.WithTimeoutCause(t.Context(), 30*time.Second, errScenarioHung) defer cancel() @@ -195,7 +197,7 @@ func runScenario(t *testing.T, h contractHeader, s contractScenario, wantOutcome return hung, hung } - if got := contractOutcome(err); got != wantOutcome { + if got := contractOutcome(err); got != wantOutcome && !retriesRanOut(wantOutcome, err) { mismatches = append(mismatches, fmt.Sprintf("outcome = %s (%v), want %s", got, err, wantOutcome)) } if got := srv.EndpointsTouched(); !slices.Equal(got, s.Expect.EndpointsUsed) { @@ -321,6 +323,13 @@ func contractOutcome(err error) string { return "UNCLASSIFIED" } +// retriesRanOut reports whether err is how Go reports the contract's +// RETRY_ATTEMPTS_EXHAUSTED: with the last failure, as ops.Execute returns it, +// which is one the retrier retries. The request counts show the retries. +func retriesRanOut(wantOutcome string, err error) bool { + return wantOutcome == "RETRY_ATTEMPTS_EXHAUSTED" && cloudstorage.IsRetriable(err) +} + // contractPayload returns size bytes that are the same on every run. func contractPayload(size int) []byte { data := make([]byte, size) diff --git a/files/v2/ext_resumable.go b/files/v2/ext_resumable.go deleted file mode 100644 index 8a5734e9..00000000 --- a/files/v2/ext_resumable.go +++ /dev/null @@ -1,187 +0,0 @@ -package files - -// Resumable upload for GCP: the single-stream, offset-resuming protocol used -// when the server hands back a resumable session instead of multipart. Chunk -// transfers go through uc.cloud.Attempt (raw responses, since 308 is the normal -// "continue" signal); the loop runs its own resume-aware retry. - -import ( - "context" - "fmt" - "io" - "net/http" - "regexp" - "slices" - "strconv" - - "github.com/databricks/sdk-go/core/apierr" - "github.com/databricks/sdk-go/files/v2/internal/cloudstorage" -) - -var resumableRangeRe = regexp.MustCompile(`bytes=0-(\d+)`) - -// performResumableUpload uploads a stream using the GCP resumable protocol, -// aborting the session if any chunk fails. -func (e *engine) performResumableUpload(ctx context.Context, uc *uploadContext, token string, r io.Reader, preRead []byte) error { - purl, err := e.createResumableURL(ctx, uc.targetPath, token) - if err != nil { - return &fallbackToFilesAPI{buffer: preRead, reason: fmt.Sprintf("failed to obtain resumable upload URL: %v", err)} - } - if err := e.resumableUploadLoop(ctx, uc, purl, r, preRead); err != nil { - if aerr := e.abortResumableUpload(ctx, uc, purl); aerr != nil { - e.c.logger.WarnContext(ctx, "failed to abort resumable upload", "error", aerr) - } - return err - } - return nil -} - -func (e *engine) resumableUploadLoop(ctx context.Context, uc *uploadContext, purl presignedURL, r io.Reader, preRead []byte) error { - // Buffer one part plus a read-ahead byte: a resumable chunk cannot be empty, - // so reading one byte past the part tells us whether this is the last chunk - // (which must be sent with the real total size, not "*"). - minBuffer := uc.partSize + uc.tun.readAheadBytes - buffer := slices.Clone(preRead) - chunkOffset := int64(0) - retryCount := 0 - noProgress := 0 - - for { - lastChunk := false - if need := minBuffer - int64(len(buffer)); need > 0 { - tmp := make([]byte, need) - n, rerr := io.ReadFull(r, tmp) - if rerr != nil && rerr != io.EOF && rerr != io.ErrUnexpectedEOF { - return rerr - } - buffer = append(buffer, tmp[:n]...) - if int64(n) < need { - lastChunk = true - } - } - - var chunkLen int64 - var totalSize string - if lastChunk { - chunkLen = int64(len(buffer)) - totalSize = strconv.FormatInt(chunkOffset+chunkLen, 10) - } else { - chunkLen = uc.partSize - totalSize = "*" - } - // An empty stream cannot be sent as a resumable upload: a chunk must carry - // at least one byte, and the range would be the malformed "bytes 0--1/0". - // Fall back to a single-shot PUT, which creates the empty object. Only - // reachable at offset 0, since the read-ahead byte folds a non-empty - // stream's tail into the preceding chunk. - if chunkLen == 0 { - return &fallbackToFilesAPI{reason: "empty input stream"} - } - chunkLastByte := chunkOffset + chunkLen - 1 - - headers := octetStreamHeaders(purl.Headers) - headers["Content-Range"] = fmt.Sprintf("bytes %d-%d/%s", chunkOffset, chunkLastByte, totalSize) - - if err := uc.limiter.Acquire(ctx); err != nil { - return err - } - resp, rerr := uc.cloud.Attempt(ctx, http.MethodPut, purl.URL, headers, cloudstorage.BytesBody(buffer[:chunkLen])) - uc.limiter.Release() - switch { - case rerr != nil: - // On a transient transport error, query the server for the confirmed - // offset and continue from there; otherwise surface the error. - if retryCount >= uc.tun.maxRetries || !cloudstorage.IsRetriable(rerr) { - return rerr - } - retryCount++ - status, qerr := e.resumableStatusQuery(ctx, uc, purl) - if qerr != nil || status == nil { - return rerr - } - resp = status - case cloudstorage.IsRetryableStatus(resp.StatusCode): - if retryCount < uc.tun.maxRetries { - retryCount++ - if status, qerr := e.resumableStatusQuery(ctx, uc, purl); qerr == nil && status != nil { - resp = status - } - } - default: - retryCount = 0 - } - - switch { - case resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated: - if totalSize == "*" { - return fmt.Errorf("resumable upload reported complete with status %d before end of stream", resp.StatusCode) - } - uc.progress.add(chunkLen) - return nil - case resp.StatusCode == http.StatusPermanentRedirect: // 308: chunk accepted, more to come - confirmed, ok, perr := extractRangeOffset(resp.Header.Get("Range")) - if perr != nil { - return perr - } - var nextOffset int64 - if ok { - if confirmed < chunkOffset-1 || confirmed > chunkLastByte { - return fmt.Errorf("resumable upload confirmed offset %d outside chunk [%d, %d]", confirmed, chunkOffset, chunkLastByte) - } - nextOffset = confirmed + 1 - } else { - if chunkOffset > 0 { - return fmt.Errorf("resumable upload returned no confirmed offset at chunk offset %d", chunkOffset) - } - nextOffset = chunkOffset - } - // A 308 confirming no new bytes (nextOffset == chunkOffset) leaves the - // chunk unsent, and the first switch's default arm resets retryCount, so a - // server stuck at a fixed offset would loop forever (fs cp sets no context - // deadline). Bound it with a counter that survives that reset. - if nextOffset == chunkOffset { - noProgress++ - if noProgress > uc.tun.maxRetries { - return fmt.Errorf("resumable upload made no progress at offset %d after %d attempts", chunkOffset, noProgress) - } - } else { - // Forward progress recovered from a transient error via the status - // query: reset both budgets so they bound consecutive stalls, not the - // total blip count over a long, steadily-advancing upload. - noProgress = 0 - retryCount = 0 - } - buffer = buffer[nextOffset-chunkOffset:] - uc.progress.add(nextOffset - chunkOffset) - chunkOffset = nextOffset - case resp.StatusCode == http.StatusPreconditionFailed && (uc.overwrite == nil || !*uc.overwrite): - return ErrAlreadyExists - default: - if apiErr := apierr.FromHTTPError(resp.StatusCode, resp.Header, resp.Body); apiErr != nil { - return apiErr - } - return fmt.Errorf("resumable chunk upload failed with status %d", resp.StatusCode) - } - } -} - -func (e *engine) resumableStatusQuery(ctx context.Context, uc *uploadContext, purl presignedURL) (*cloudstorage.Response, error) { - return uc.cloud.Attempt(ctx, http.MethodPut, purl.URL, map[string]string{"Content-Range": "bytes */*"}, nil) -} - -// extractRangeOffset parses a resumable upload "Range: bytes=0-N" response -// header and returns N. An empty header means the server has confirmed nothing. -func extractRangeOffset(rangeHeader string) (offset int64, ok bool, err error) { - if rangeHeader == "" { - return 0, false, nil - } - m := resumableRangeRe.FindStringSubmatch(rangeHeader) - if m == nil { - return 0, false, fmt.Errorf("cannot parse Range header %q", rangeHeader) - } - n, perr := strconv.ParseInt(m[1], 10, 64) - if perr != nil { - return 0, false, fmt.Errorf("cannot parse Range header %q: %w", rangeHeader, perr) - } - return n, true, nil -} diff --git a/files/v2/ext_resumable_test.go b/files/v2/ext_resumable_test.go deleted file mode 100644 index bdf579ce..00000000 --- a/files/v2/ext_resumable_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package files - -import "testing" - -func TestExtractRangeOffset(t *testing.T) { - n, ok, err := extractRangeOffset("bytes=0-99") - noErr(t, err) - if !ok || n != 99 { - t.Errorf("extractRangeOffset(bytes=0-99) = (%d, %v), want (99, true)", n, ok) - } - - _, ok, err = extractRangeOffset("") - noErr(t, err) - if ok { - t.Error("empty header should report nothing confirmed") - } - - if _, _, err := extractRangeOffset("garbage"); err == nil { - t.Error("malformed header should error") - } -} diff --git a/files/v2/ext_tunables.go b/files/v2/ext_tunables.go index 290e338e..62bbda9f 100644 --- a/files/v2/ext_tunables.go +++ b/files/v2/ext_tunables.go @@ -45,18 +45,22 @@ type tunables struct { // stream) holds in memory, where each worker keeps one partSize buffer for its // in-flight part. The default stream worker count is this budget divided by // the part size, so a larger part size lowers the worker count instead of - // growing memory. At the 16 MiB default part size this is 16 workers. + // growing memory. At the 16 MiB default part size this is 16 workers. A + // stream of unknown size also holds up to minStreamSize of read-ahead until + // its first parts are cut from it. streamMemoryBudget int64 - // readAheadBytes is how far past a chunk the resumable upload reads to detect - // end-of-stream (a resumable chunk cannot be empty, so the last chunk must be - // marked with the real total size). - readAheadBytes int64 - // maxRetries caps how many times a single part is retried after a presigned - // URL expires (multipart) or a chunk must be resumed (resumable). + // URL expires. maxRetries int + // cloudRetries caps how many times a part PUT is retried once a part of the + // upload has landed. cloudRetriesBeforeCommit applies before then: the + // workspace is still left to fall back to, so failing over is cheaper than + // waiting out retries on a storage host that may be blocked. + cloudRetries int + cloudRetriesBeforeCommit int + // maxUploadParts is the provider's maximum number of parts per multipart // upload (S3's 10,000 is the common floor). The default part size is grown for // files large enough to otherwise exceed it. @@ -127,6 +131,12 @@ type tunables struct { // caller's context expires (and forever if it has no deadline). cpRetryTimeout time.Duration + // endpointsAPI is whether a transfer asks the workspace which endpoints to + // use (GET /api/2.0/fs/endpoints). Off by default until the port is done, + // though the Python SDK now turns it on; without it a transfer walks the + // built-in order. + endpointsAPI bool + // urlExpiry is how long requested presigned URLs are valid. urlExpiry time.Duration } @@ -134,28 +144,31 @@ type tunables struct { // defaultTunables returns the production policy values. func defaultTunables() tunables { return tunables{ - minStreamSize: 50 * 1024 * 1024, - defaultPartSize: 16 << 20, // 16 MiB - maxPartSize: 4 << 30, - batchURLCount: 1, - fileParallelism: 128, - streamMemoryBudget: 256 << 20, - readAheadBytes: 1, - maxRetries: 3, - maxUploadParts: 10_000, - responseHeaderTimeout: 60 * time.Second, - cleanupTimeout: 30 * time.Second, - slowFactor: 3, - slowWarmup: 32, - slowWindow: 128, - slowMinDeadline: 5 * time.Second, - slowColdDeadline: 30 * time.Second, - slowFirstPartDeadline: 10 * time.Second, - slowCheckInterval: 1 * time.Second, - slowMaxReissue: 2, - cpBackoff: ops.BackoffPolicy{}, - cpRetryTimeout: 5 * time.Minute, - urlExpiry: time.Hour, + minStreamSize: 50 * 1024 * 1024, + defaultPartSize: 16 << 20, // 16 MiB + maxPartSize: 4 << 30, + batchURLCount: 1, + fileParallelism: 128, + streamMemoryBudget: 256 << 20, + maxRetries: 3, + // Python's experimental_files_ext_cloud_api_max_retries is 3 attempts in + // all, and its _UPLOAD_RETRIES_BEFORE_COMMIT is 1 retry. + cloudRetries: 2, + cloudRetriesBeforeCommit: 1, + maxUploadParts: 10_000, + responseHeaderTimeout: 60 * time.Second, + cleanupTimeout: 30 * time.Second, + slowFactor: 3, + slowWarmup: 32, + slowWindow: 128, + slowMinDeadline: 5 * time.Second, + slowColdDeadline: 30 * time.Second, + slowFirstPartDeadline: 10 * time.Second, + slowCheckInterval: 1 * time.Second, + slowMaxReissue: 2, + cpBackoff: ops.BackoffPolicy{}, + cpRetryTimeout: 5 * time.Minute, + urlExpiry: time.Hour, } } diff --git a/files/v2/ext_upload.go b/files/v2/ext_upload.go index e397ff3b..fee51766 100644 --- a/files/v2/ext_upload.go +++ b/files/v2/ext_upload.go @@ -1,18 +1,16 @@ package files // Large-file upload engine for Databricks Unity Catalog Volumes. It chooses a -// single-shot, multipart (AWS/Azure), or resumable (GCP) protocol based on the -// stream size and the protocol the workspace's Files API selects. +// single-shot or multipart protocol based on the stream size. // // The engine is pure orchestration over two transports: this client's own // authenticated Files API calls (the control plane; see upload_control.go), -// which handle the single-shot upload and the multipart/resumable coordination, +// which handle the single-shot upload and the multipart coordination, // and the unauthenticated cloud-storage client (the cloudstorage subpackage), -// which transfers parts and chunks directly to object storage over the presigned +// which transfers parts directly to object storage over the presigned // URLs the control plane mints. import ( - "bytes" "context" "errors" "fmt" @@ -31,19 +29,6 @@ import ( // [ErrAlreadyExists] sentinel; a malformed server response is an internal error. var errUploadURLExpired = errors.New("upload URL expired after retries") -// fallbackToFilesAPI signals that a multipart or resumable upload failed in a -// way that warrants retrying through a single-shot Files API PUT (for example, -// a cloud firewall rejecting the first part). It carries the bytes already -// buffered but not yet uploaded so they can be replayed. -type fallbackToFilesAPI struct { - buffer []byte - reason string -} - -func (e *fallbackToFilesAPI) Error() string { - return "falling back to single-shot upload: " + e.reason -} - // UploadResult holds the result of an upload. It is currently empty and exists // for forward compatibility. type UploadResult struct{} @@ -163,11 +148,17 @@ func (cfg *uploadConfig) resolvedParallelism(buffered bool, partSize int64, tun // context instead. Callers who need to supply their own client use // WithTransferClient. func newTransferClient(n int, tun tunables) *http.Client { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.ResponseHeaderTimeout = tun.responseHeaderTimeout - transport.MaxIdleConnsPerHost = n - transport.MaxIdleConns = max(transport.MaxIdleConns, n) - return &http.Client{Transport: transport} + return &http.Client{Transport: sizedForTransfers(http.DefaultTransport.(*http.Transport), n, tun)} +} + +// sizedForTransfers returns a copy of t that bounds the response phase of a +// transfer and keeps up to n idle connections to a host. +func sizedForTransfers(t *http.Transport, n int, tun tunables) *http.Transport { + t = t.Clone() + t.ResponseHeaderTimeout = tun.responseHeaderTimeout + t.MaxIdleConnsPerHost = n + t.MaxIdleConns = max(t.MaxIdleConns, n) + return t } // resolveCloudClient returns the caller-supplied transfer client when set, and @@ -186,29 +177,86 @@ type uploadContext struct { tun tunables targetPath string overwrite *bool - partSize int64 + partSize int64 // of the current endpoint: basePartSize, capped by the endpoint + basePartSize int64 batchSize int contentLength int64 // -1 when unknown (non-seekable stream) - sourcePath string parallel bool parallelism int - cloud *cloudstorage.Client - progress *progressReporter - slowGuard *slowAttemptGuard - limiter Limiter // bounds concurrent transfers; never nil (unlimited by default) - - // completed records whether any part has finished uploading. The single-shot - // fallback is taken only while no part has landed; once a part succeeds, a - // later cloud rejection is a real error rather than a blanket-block signal. + cloud *cloudstorage.Client // presigned URLs: no credentials + // files sends parts to a Files API host with the client's credentials. It is + // built only if the walk reaches such an endpoint. + files func() *cloudstorage.Client + progress *progressReporter + slowGuard *slowAttemptGuard + limiter Limiter // bounds concurrent transfers; never nil (unlimited by default) + + // data is the upload's source; the walk sends it to each endpoint it tries. + data *uploadData + + // ep is the endpoint the walk is trying, and lastEndpoint whether no + // endpoint is left after it. They change only between endpoints, while no + // request is in flight. + ep endpoint + lastEndpoint bool + + // completed records whether any part has landed on the current endpoint. + // The walk moves on only while none has; once one has, the upload is + // committed to the endpoint and a later failure is returned. completed atomic.Bool } +// startEndpoint sets the endpoint the walk tries next. +func (uc *uploadContext) startEndpoint(ep endpoint, last bool) { + uc.ep, uc.lastEndpoint = ep, last + uc.partSize = uc.basePartSize + if limit := ep.maxPartSize(); limit >= 0 { + uc.partSize = min(uc.partSize, limit) + } +} + +// commit records that a part landed, the first time on any goroutine. +func (uc *uploadContext) commit() { + if !uc.completed.Swap(true) && uc.data != nil { + uc.data.commit() + } +} + +func (uc *uploadContext) committed() bool { return uc.completed.Load() } + +// retries is how often a request to the current endpoint may be retried, or -1 +// for no cap but the retry timeout. Before any part lands, with an endpoint left +// to move on to, a failing request gets a single retry: moving on is cheaper than +// waiting out a host that may be unreachable from here. Otherwise the workspace +// host keeps the API client's retries, and other hosts get cloudRetries. +func (uc *uploadContext) retries(onWorkspace bool) int { + switch { + case !uc.completed.Load() && !uc.lastEndpoint: + return uc.tun.cloudRetriesBeforeCommit + case onWorkspace: + return -1 + } + return uc.tun.cloudRetries +} + +// partRetries is how often a part PUT is retried. Parts never go to the +// workspace host. +func (uc *uploadContext) partRetries() int { return uc.retries(false) } + +// partClient sends the current endpoint's part PUTs. +func (uc *uploadContext) partClient() *cloudstorage.Client { + if uc.ep.kind == presignedEndpoint { + return uc.cloud + } + return uc.files() +} + // newUploadContext resolves the per-upload state shared by Upload and UploadFrom. -// contentLength is -1 for a non-seekable stream; sourcePath is "" unless the -// source is a local file (UploadFrom). buffered reports whether the upload path -// holds parts in memory (a non-seekable stream) rather than streaming them from -// positioned reads, which sets the default worker count. -func (e *engine) newUploadContext(cfg *uploadConfig, targetPath, sourcePath string, contentLength int64, buffered bool) *uploadContext { +// contentLength is -1 for a non-seekable stream. buffered reports whether the +// upload holds parts in memory (a non-seekable stream) rather than streaming +// them from positioned reads, which sets the default worker count. +func (e *engine) newUploadContext(cfg *uploadConfig, targetPath string, data *uploadData, buffered bool) *uploadContext { + contentLength := data.size() partSize, batchSize := optimizeParams(contentLength, cfg.partSize, e.tun) parallelism := cfg.resolvedParallelism(buffered, partSize, e.tun) limiter := cfg.limiter @@ -220,15 +268,19 @@ func (e *engine) newUploadContext(cfg *uploadConfig, targetPath, sourcePath stri targetPath: targetPath, overwrite: cfg.overwrite, partSize: partSize, + basePartSize: partSize, batchSize: batchSize, contentLength: contentLength, - sourcePath: sourcePath, + data: data, parallel: parallelism > 1, parallelism: parallelism, cloud: cloudstorage.New(cfg.resolveCloudClient(parallelism, e.tun), cloudstorage.WithLogger(e.c.logger), cloudstorage.WithBackoff(e.tun.cloudBackoff)), - progress: newProgressReporter(cfg.progress, contentLength), - slowGuard: &slowAttemptGuard{tun: e.tun}, - limiter: limiter, + files: sync.OnceValue(func() *cloudstorage.Client { + return cloudstorage.New(e.authenticatedHTTPClient(parallelism), cloudstorage.WithLogger(e.c.logger), cloudstorage.WithBackoff(e.tun.cloudBackoff)) + }), + progress: newProgressReporter(cfg.progress, contentLength), + slowGuard: &slowAttemptGuard{tun: e.tun}, + limiter: limiter, } } @@ -306,8 +358,8 @@ func withProgress(r io.Reader, p *progressReporter) io.Reader { // Upload uploads contents to the remote file at filePath, which must be an // absolute path such as "/Volumes/catalog/schema/volume/file". It chooses a -// single-shot, multipart, or resumable upload based on the stream size (when -// the reader is seekable) and the protocol the workspace's Files API selects. +// single-shot or multipart upload based on the stream size (when the reader is +// seekable). // // When contents implements io.ReaderAt (for example an *os.File or a // *bytes.Reader), parts are read from it with concurrent positioned reads rather @@ -360,25 +412,14 @@ func (e *engine) upload(ctx context.Context, filePath string, contents io.Reader // parts from positioned reads; any other source buffers each in-flight part. _, isReaderAt := contents.(io.ReaderAt) buffered := !isReaderAt || contentLength < e.tun.minStreamSize - uc := e.newUploadContext(cfg, filePath, "", contentLength, buffered) - - switch { - case uc.parallel && contentLength < 0: - // Non-seekable stream of unknown size: parts are buffered as they are read. - return &UploadResult{}, e.parallelUploadFromStream(ctx, uc, contents) - case uc.parallel && contentLength >= e.tun.minStreamSize: - // Known-size large upload. A reader supporting concurrent positioned reads - // (io.ReaderAt) is streamed in sections without buffering; anything else - // buffers parts. - if ra, ok := contents.(io.ReaderAt); ok { - return &UploadResult{}, e.parallelUploadFromReaderAt(ctx, uc, ra) - } - return &UploadResult{}, e.parallelUploadFromStream(ctx, uc, contents) - case contentLength >= 0: - return &UploadResult{}, e.uploadSingleThreadKnownSize(ctx, uc, contents) - default: - return &UploadResult{}, e.singleThreadMultipart(ctx, uc, contents) + var data *uploadData + if contentLength >= 0 { + data = newSeekableData(contents.(io.ReadSeeker), contentLength) + } else { + data = newStreamData(contents) } + uc := e.newUploadContext(cfg, filePath, data, buffered) + return &UploadResult{}, e.walkUpload(ctx, uc, data) } func (e *engine) uploadFrom(ctx context.Context, filePath, sourcePath string, opts ...UploadOption) (*UploadResult, error) { @@ -390,21 +431,17 @@ func (e *engine) uploadFrom(ctx context.Context, filePath, sourcePath string, op if err != nil { return nil, err } - size := info.Size() - - // A local file is randomly readable, so the parallel path streams parts from - // positioned reads and never buffers them: buffered is false. - uc := e.newUploadContext(cfg, filePath, sourcePath, size, false) - - if uc.parallel && size >= e.tun.minStreamSize { - return &UploadResult{}, e.parallelUploadFromFile(ctx, uc) - } f, err := os.Open(sourcePath) if err != nil { return nil, err } defer f.Close() - return &UploadResult{}, e.uploadSingleThreadKnownSize(ctx, uc, f) + + // A local file is randomly readable, so the parallel path streams parts from + // positioned reads and never buffers them: buffered is false. + data := newSeekableData(f, info.Size()) + uc := e.newUploadContext(cfg, filePath, data, false) + return &UploadResult{}, e.walkUpload(ctx, uc, data) } // optimizeParams picks a part size and a presigned-URL batch size. contentLength @@ -435,134 +472,25 @@ func optimizeParams(contentLength, override int64, tun tunables) (partSize int64 return partSize, batchSize } -func (e *engine) uploadSingleThreadKnownSize(ctx context.Context, uc *uploadContext, r io.Reader) error { - if uc.contentLength < e.tun.minStreamSize { - return e.singleShotUpload(ctx, uc, r) - } - return e.singleThreadMultipart(ctx, uc, r) -} - -// singleThreadMultipart pre-reads up to the single-shot threshold to handle -// small (or non-seekable but actually small) streams without a multipart -// round trip, then initiates the upload. -func (e *engine) singleThreadMultipart(ctx context.Context, uc *uploadContext, r io.Reader) error { - preRead, err := readUpTo(r, e.tun.minStreamSize) +// initiateMultipart opens a multipart upload session on the current endpoint +// and returns its token. Only multipart sessions are supported; any other +// initiate response is an unexpected one. +func (e *engine) initiateMultipart(ctx context.Context, uc *uploadContext) (string, error) { + resp, err := e.initiateUpload(ctx, uc) if err != nil { - return err - } - if int64(len(preRead)) < e.tun.minStreamSize { - return e.singleShotUpload(ctx, uc, bytes.NewReader(preRead)) + return "", err } - return e.initiateAndUpload(ctx, uc, false, - func(token string) error { return e.performMultipartUpload(ctx, uc, token, r, preRead) }, - func(token string) error { return e.performResumableUpload(ctx, uc, token, r, preRead) }, - func(buffered []byte) error { - return e.singleShotUpload(ctx, uc, io.MultiReader(bytes.NewReader(buffered), r)) - }, - ) -} - -func (e *engine) parallelUploadFromStream(ctx context.Context, uc *uploadContext, r io.Reader) error { - return e.initiateAndUpload(ctx, uc, true, - func(token string) error { return e.parallelMultipartFromStream(ctx, uc, token, r) }, - func(token string) error { return e.performResumableUpload(ctx, uc, token, r, nil) }, - func(buffered []byte) error { - return e.singleShotUpload(ctx, uc, io.MultiReader(bytes.NewReader(buffered), r)) - }, - ) -} - -func (e *engine) parallelUploadFromFile(ctx context.Context, uc *uploadContext) error { - // Each branch opens its own handle: only one runs per upload, and the multipart - // driver keeps its handle open until all its workers finish (wait()). - withFile := func(use func(*os.File) error) error { - f, err := os.Open(uc.sourcePath) - if err != nil { - return err + if uc.ep.kind == filesAPIv3Endpoint { + if resp.UploadID == "" { + return "", fmt.Errorf("%w: missing upload_id", errUnexpectedServerResponse) } - defer f.Close() - return use(f) + return resp.UploadID, nil } - return e.initiateAndUpload(ctx, uc, true, - func(token string) error { - return withFile(func(f *os.File) error { return e.parallelMultipartFromReaderAt(ctx, uc, token, f) }) - }, - func(token string) error { - return withFile(func(f *os.File) error { return e.performResumableUpload(ctx, uc, token, f, nil) }) - }, - func(buffered []byte) error { - return withFile(func(f *os.File) error { return e.singleShotUpload(ctx, uc, f) }) - }, - ) -} - -// parallelUploadFromReaderAt uploads a known-size, randomly-readable source -// (an *os.File or *bytes.Reader passed to Upload) by streaming sections of it -// concurrently, without buffering whole parts. The resumable and fallback paths -// re-read it through fresh section readers. -func (e *engine) parallelUploadFromReaderAt(ctx context.Context, uc *uploadContext, ra io.ReaderAt) error { - return e.initiateAndUpload(ctx, uc, true, - func(token string) error { return e.parallelMultipartFromReaderAt(ctx, uc, token, ra) }, - func(token string) error { - return e.performResumableUpload(ctx, uc, token, io.NewSectionReader(ra, 0, uc.contentLength), nil) - }, - func(buffered []byte) error { - return e.singleShotUpload(ctx, uc, io.NewSectionReader(ra, 0, uc.contentLength)) - }, - ) -} - -// initiateAndUpload initiates the server-coordinated upload, dispatches to the -// multipart or resumable path the server selected (multipart on AWS/Azure, -// resumable on GCP), and applies the shared abort-and-fallback handling. -func (e *engine) initiateAndUpload( - ctx context.Context, - uc *uploadContext, - parallel bool, - performMultipart func(token string) error, - performResumable func(token string) error, - fallbackSingleShot func(buffered []byte) error, -) error { - resp, err := e.initiateUpload(ctx, uc.targetPath, uc.overwrite) - if err != nil { - return err + if resp.MultipartUpload == nil { + return "", fmt.Errorf("%w: no multipart upload session offered", errUnexpectedServerResponse) } - - switch { - case resp.MultipartUpload != nil: - token := resp.MultipartUpload.SessionToken - if token == "" { - return fmt.Errorf("%w: missing multipart session token", errUnexpectedServerResponse) - } - uerr := performMultipart(token) - if uerr == nil { - return nil - } - // Abort is best-effort in both the fallback and the hard-error case. - e.abortMultipartBestEffort(ctx, uc, token) - if fb, ok := errors.AsType[*fallbackToFilesAPI](uerr); ok { - e.c.logger.InfoContext(ctx, "using single-shot fallback", "reason", fb.reason) - return fallbackSingleShot(fb.buffer) - } - return uerr - - case resp.ResumableUpload != nil: - token := resp.ResumableUpload.SessionToken - if token == "" { - return fmt.Errorf("%w: missing resumable session token", errUnexpectedServerResponse) - } - if parallel { - e.c.logger.WarnContext(ctx, "GCP does not support parallel resumable uploads; using single-threaded upload") - } - // The resumable path aborts itself on error, so no abort here. - uerr := performResumable(token) - if fb, ok := errors.AsType[*fallbackToFilesAPI](uerr); ok { - e.c.logger.InfoContext(ctx, "using single-shot fallback", "reason", fb.reason) - return fallbackSingleShot(fb.buffer) - } - return uerr - - default: - return fmt.Errorf("%w: neither multipart nor resumable upload offered", errUnexpectedServerResponse) + if resp.MultipartUpload.SessionToken == "" { + return "", fmt.Errorf("%w: missing multipart session token", errUnexpectedServerResponse) } + return resp.MultipartUpload.SessionToken, nil } diff --git a/files/v2/ext_upload_control.go b/files/v2/ext_upload_control.go index e62f71b9..c4ac0d8a 100644 --- a/files/v2/ext_upload_control.go +++ b/files/v2/ext_upload_control.go @@ -1,10 +1,9 @@ package files -// Files API control-plane calls for large uploads: initiate a server-coordinated -// session, mint presigned URLs for parts (multipart) and the resumable session -// (GCP), complete a multipart upload, and mint abort URLs for cleanup. It also -// owns the single-shot octet-stream PUT used for small files and as the -// multipart/resumable fallback. +// Files API control-plane calls for large uploads: initiate a multipart upload +// session, mint presigned URLs for its parts, complete it, and abort it for +// cleanup. It also owns the single-shot octet-stream PUT used for small +// files and as the multipart fallback. // // These calls attach the client's credentials per request (via newHTTPRequest, // the same helper the generated methods use); the parts/chunks then transfer @@ -23,6 +22,7 @@ import ( "net/url" "slices" "strconv" + "sync" "time" "github.com/databricks/sdk-go/core/apierr" @@ -57,12 +57,12 @@ var controlPlaneRetryStatusCodes = []int{ // --- Result types --- // initiateResult reports which large-file protocol the server selected. Exactly -// one of multipartUpload (AWS/Azure) and resumableUpload (GCP) is non-nil on a -// well-formed response; the pointers distinguish "not offered" from "offered -// with an empty token". +// multipartUpload is non-nil on a well-formed response; the pointer +// distinguishes "not offered" from "offered with an empty token". type initiateResult struct { MultipartUpload *uploadSession `json:"multipart_upload"` - ResumableUpload *uploadSession `json:"resumable_upload"` + // UploadID names the session on Files API 3.0. + UploadID string `json:"upload_id"` } type uploadSession struct { @@ -77,7 +77,7 @@ type presignedURL struct { Headers map[string]string } -// --- Wire types for the Files API multipart/resumable coordination --- +// --- Wire types for the Files API multipart coordination --- type nameValue struct { Name string `json:"name"` @@ -100,32 +100,6 @@ type createPartURLsResponse struct { } `json:"upload_part_urls"` } -type resumableURLRequest struct { - Path string `json:"path"` - SessionToken string `json:"session_token"` -} - -// urlWithHeaders is a presigned cloud-storage URL and its required request -// headers, as returned by the resumable and abort URL endpoints. -type urlWithHeaders struct { - URL string `json:"url"` - Headers []nameValue `json:"headers"` -} - -type resumableURLResponse struct { - ResumableUploadURL *urlWithHeaders `json:"resumable_upload_url"` -} - -type abortURLRequest struct { - Path string `json:"path"` - SessionToken string `json:"session_token"` - ExpireTime string `json:"expire_time"` -} - -type abortURLResponse struct { - AbortUploadURL *urlWithHeaders `json:"abort_upload_url"` -} - type completePart struct { PartNumber int `json:"part_number"` ETag string `json:"etag"` @@ -135,38 +109,39 @@ type completeRequest struct { Parts []completePart `json:"parts"` } -// uploadSingleShot sends the whole body in one PUT. Used for small files and as -// the multipart/resumable fallback. overwrite is tri-state: a non-nil value is -// sent explicitly; nil lets the server apply its default. A 409 ALREADY_EXISTS -// is mapped to ErrAlreadyExists. +// uploadSingleShot sends the whole body in one PUT to the current endpoint's +// Files API. size is the body's length, or -1 if unknown. The upload's overwrite +// is tri-state: a non-nil value is sent explicitly; nil lets the server apply +// its default. A 409 ALREADY_EXISTS is mapped to ErrAlreadyExists. // // The generated UploadFile is unsuitable here: it JSON-marshals the reader // rather than streaming the raw octet stream the Files API expects. A seekable -// body is rewound before each retry; a non-seekable body cannot be replayed, so -// it is attempted once. -func (e *engine) uploadSingleShot(ctx context.Context, path string, overwrite *bool, body io.Reader) error { +// body is sent again on each retry (see retryBodies); a non-seekable body cannot +// be replayed, so it is attempted once. +func (e *engine) uploadSingleShot(ctx context.Context, uc *uploadContext, body io.Reader, size int64) error { query := url.Values{} - if overwrite != nil { - query.Set("overwrite", strconv.FormatBool(*overwrite)) + if uc.overwrite != nil { + query.Set("overwrite", strconv.FormatBool(*uc.overwrite)) } - urlStr, err := e.controlPlaneURL(filesAPIPath(path), query) + urlStr, err := endpointURL(uc.ep.baseURL, uc.ep.filesPath(uc.targetPath), query) if err != nil { return err } - seeker, seekable := body.(io.Seeker) - var start int64 - if seekable { - if start, err = seeker.Seek(0, io.SeekCurrent); err != nil { - seekable = false - } - } + bodies, seekable := newRetryBodies(body) call := func(ctx context.Context) error { + attempt := body if seekable { - if _, err := seeker.Seek(start, io.SeekStart); err != nil { - return fmt.Errorf("rewinding upload body: %w", err) + next, err := bodies.next(ctx) + if err != nil { + return err } + attempt = next + } else if _, ok := body.(io.Closer); ok { + // net/http closes a request body that is an io.Closer once the + // attempt ends, and the body is the caller's. + attempt = struct{ io.Reader }{body} } headers := http.Header{} headers.Set("Content-Type", "application/octet-stream") @@ -175,11 +150,25 @@ func (e *engine) uploadSingleShot(ctx context.Context, path string, overwrite *b URL: urlStr, Credentials: e.c.credentials, Headers: headers, - Body: body, + Body: attempt, }) if err != nil { return err } + // Hosts require a Content-Length, which net/http knows only for a few + // in-memory body types, and never for an attempt's body; an empty body + // must be no body to send one. + length := size + if seekable { + length, req.GetBody = bodies.size, bodies.getBody() + } + if length == 0 { + _ = req.Body.Close() + req.Body, req.GetBody = http.NoBody, nil + } + if length >= 0 { + req.ContentLength = length + } e.setWorkspaceHeader(req) _, _, err = executeHTTPCall(httpCallOptions{req: req, client: e.c.httpClient, logger: e.c.logger}) return err @@ -189,38 +178,114 @@ func (e *engine) uploadSingleShot(ctx context.Context, path string, overwrite *b // cannot be replayed, so it is attempted exactly once. opts := []ops.Option{ops.WithTimeout(e.tun.cpRetryTimeout)} if seekable { - opts = append(opts, ops.WithRetrier(e.newControlPlaneRetrier)) + opts = append(opts, ops.WithRetrier(e.controlPlaneRetrier(uc))) + } + return e.asAlreadyExistsOn(uc, ops.Execute(ctx, call, opts...)) +} + +// retryBodies gives each attempt of a request its own body over the same +// seekable data. net/http may still read an attempt's body after the attempt +// returns, until it closes the body, so a retry that rewound a shared body +// sooner could lose bytes to that late read. +type retryBodies struct { + body io.Reader + start, size int64 + // last is the latest attempt's body, for data without random access. + last *closeSignal +} + +// newRetryBodies returns the bodies over body from its current position to its +// end, or false if body cannot be sent again. +func newRetryBodies(body io.Reader) (*retryBodies, bool) { + seeker, ok := body.(io.Seeker) + if !ok { + return nil, false + } + start, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return nil, false + } + end, err := seeker.Seek(0, io.SeekEnd) + if err != nil { + return nil, false } - return asAlreadyExists(ops.Execute(ctx, call, opts...)) + return &retryBodies{body: body, start: start, size: end - start}, true } -// initiateUpload opens a server-coordinated upload session for path. The result -// reports which protocol the workspace selected (multipart on AWS/Azure, -// resumable on GCP). -func (e *engine) initiateUpload(ctx context.Context, path string, overwrite *bool) (*initiateResult, error) { +// next returns the body of the next attempt. Data with random access gives each +// attempt its own section; other data is rewound only once the last attempt's +// body is closed. +func (b *retryBodies) next(ctx context.Context) (io.Reader, error) { + if ra, ok := b.body.(io.ReaderAt); ok { + return io.NewSectionReader(ra, b.start, b.size), nil + } + if b.last != nil { + select { + case <-b.last.closed: + case <-ctx.Done(): + return nil, context.Cause(ctx) + } + } + if _, err := b.body.(io.Seeker).Seek(b.start, io.SeekStart); err != nil { + return nil, fmt.Errorf("rewinding upload body: %w", err) + } + b.last = &closeSignal{Reader: b.body, closed: make(chan struct{})} + return b.last, nil +} + +// getBody returns a function for http.Request.GetBody, which net/http calls to +// send the body again itself, or nil for data without random access. +func (b *retryBodies) getBody() func() (io.ReadCloser, error) { + ra, ok := b.body.(io.ReaderAt) + if !ok { + return nil + } + return func() (io.ReadCloser, error) { + return io.NopCloser(io.NewSectionReader(ra, b.start, b.size)), nil + } +} + +// closeSignal reports when net/http closes a request body, without closing +// the data under it. +type closeSignal struct { + io.Reader + once sync.Once + closed chan struct{} +} + +func (c *closeSignal) Close() error { + c.once.Do(func() { close(c.closed) }) + return nil +} + +// initiateUpload opens a multipart upload session on the current endpoint. +func (e *engine) initiateUpload(ctx context.Context, uc *uploadContext) (*initiateResult, error) { query := url.Values{"action": {"initiate-upload"}} - if overwrite != nil { - query.Set("overwrite", strconv.FormatBool(*overwrite)) + if uc.overwrite != nil { + query.Set("overwrite", strconv.FormatBool(*uc.overwrite)) } var out initiateResult - if err := e.controlPlaneJSON(ctx, http.MethodPost, filesAPIPath(path), query, nil, &out); err != nil { - return nil, asAlreadyExists(err) + if err := e.controlPlaneJSON(ctx, uc, http.MethodPost, uc.ep.filesPath(uc.targetPath), query, nil, &out); err != nil { + return nil, e.asAlreadyExistsOn(uc, err) } return &out, nil } // createPartURLs mints count presigned URLs for multipart parts starting at // startPart. -func (e *engine) createPartURLs(ctx context.Context, path, token string, startPart, count int) ([]presignedURL, error) { +func (e *engine) createPartURLs(ctx context.Context, uc *uploadContext, token string, startPart, count int) ([]presignedURL, error) { + if uc.ep.kind != presignedEndpoint { + return filesAPIPartURLs(uc, token, startPart, count) + } body := createPartURLsRequest{ - Path: path, + Path: uc.targetPath, SessionToken: token, StartPartNumber: startPart, Count: count, ExpireTime: e.expireTime(), } var out createPartURLsResponse - if err := e.controlPlaneJSON(ctx, http.MethodPost, "/api/2.0/fs/create-upload-part-urls", nil, body, &out); err != nil { + if err := e.controlPlaneJSON(ctx, uc, http.MethodPost, "/api/2.0/fs/create-upload-part-urls", nil, body, &out); err != nil { return nil, err } if len(out.UploadPartURLs) == 0 { @@ -233,59 +298,56 @@ func (e *engine) createPartURLs(ctx context.Context, path, token string, startPa return result, nil } -// createResumableURL mints the single presigned URL for a GCP resumable session. -func (e *engine) createResumableURL(ctx context.Context, path, token string) (presignedURL, error) { - body := resumableURLRequest{Path: path, SessionToken: token} - var out resumableURLResponse - if err := e.controlPlaneJSON(ctx, http.MethodPost, "/api/2.0/fs/create-resumable-upload-url", nil, body, &out); err != nil { - return presignedURL{}, err - } - if out.ResumableUploadURL == nil || out.ResumableUploadURL.URL == "" { - return presignedURL{}, fmt.Errorf("%w: no resumable upload URL returned", errUnexpectedServerResponse) - } - return presignedURL{URL: out.ResumableUploadURL.URL, Headers: headerMap(out.ResumableUploadURL.Headers)}, nil -} - // completeMultipart finalizes a multipart upload with the part ETags, which it // sorts by part number before sending. -func (e *engine) completeMultipart(ctx context.Context, path, token string, etags map[int]string) error { +func (e *engine) completeMultipart(ctx context.Context, uc *uploadContext, token string, etags map[int]string) error { nums := slices.Sorted(maps.Keys(etags)) parts := make([]completePart, 0, len(nums)) for _, n := range nums { parts = append(parts, completePart{PartNumber: n, ETag: etags[n]}) } - query := url.Values{"action": {"complete-upload"}, "upload_type": {"multipart"}, "session_token": {token}} - return asAlreadyExists(e.controlPlaneJSON(ctx, http.MethodPost, filesAPIPath(path), query, completeRequest{Parts: parts}, nil)) + query := uc.ep.sessionQuery(token) + query.Set("action", "complete-upload") + return e.asAlreadyExistsOn(uc, e.controlPlaneJSON(ctx, uc, http.MethodPost, uc.ep.filesPath(uc.targetPath), query, completeRequest{Parts: parts}, nil)) } -// createAbortURL mints a presigned URL for aborting an upload session. The -// caller issues the (unauthenticated) cloud DELETE against it. -func (e *engine) createAbortURL(ctx context.Context, path, token string) (presignedURL, error) { - body := abortURLRequest{Path: path, SessionToken: token, ExpireTime: e.expireTime()} - var out abortURLResponse - if err := e.controlPlaneJSON(ctx, http.MethodPost, "/api/2.0/fs/create-abort-upload-url", nil, body, &out); err != nil { - return presignedURL{}, err - } - if out.AbortUploadURL == nil || out.AbortUploadURL.URL == "" { - return presignedURL{}, fmt.Errorf("%w: no abort upload URL returned", errUnexpectedServerResponse) +// abortMultipart cancels a multipart upload session with the Files API +// abort-upload action. It is best-effort cleanup, so it is sent once. +func (e *engine) abortMultipart(ctx context.Context, uc *uploadContext, token string) error { + query := uc.ep.sessionQuery(token) + query.Set("action", "abort-upload") + call, err := e.controlPlaneCall(uc.ep.baseURL, http.MethodPost, uc.ep.filesPath(uc.targetPath), query, nil, nil) + if err != nil { + return err } - return presignedURL{URL: out.AbortUploadURL.URL, Headers: headerMap(out.AbortUploadURL.Headers)}, nil + return call(ctx) } // controlPlaneJSON performs an authenticated JSON request against the Files API -// control plane, retrying transient failures via core/ops. reqBody and out may -// be nil. A fresh request is built on each attempt so retries re-apply -// credentials (newHTTPRequest re-reads them, picking up any token refresh) and -// rewind the body. -func (e *engine) controlPlaneJSON(ctx context.Context, method, path string, query url.Values, reqBody, out any) error { - urlStr, err := e.controlPlaneURL(path, query) +// on the current endpoint's host, retrying transient failures via core/ops. +// reqBody and out may be nil. A fresh request is built on each attempt so +// retries re-apply credentials (newHTTPRequest re-reads them, picking up any +// token refresh) and rewind the body. +func (e *engine) controlPlaneJSON(ctx context.Context, uc *uploadContext, method, path string, query url.Values, reqBody, out any) error { + call, err := e.controlPlaneCall(uc.ep.baseURL, method, path, query, reqBody, out) if err != nil { return err } + return ops.Execute(ctx, call, ops.WithRetrier(e.controlPlaneRetrier(uc)), ops.WithTimeout(e.tun.cpRetryTimeout)) +} + +// controlPlaneCall returns one attempt of an authenticated JSON request against +// the Files API on baseURL. Each call builds a fresh request, so a retry +// re-applies credentials and rewinds the body. +func (e *engine) controlPlaneCall(baseURL, method, path string, query url.Values, reqBody, out any) (func(context.Context) error, error) { + urlStr, err := endpointURL(baseURL, path, query) + if err != nil { + return nil, err + } var bodyBytes []byte if reqBody != nil { if bodyBytes, err = json.Marshal(reqBody); err != nil { - return err + return nil, err } } @@ -316,13 +378,42 @@ func (e *engine) controlPlaneJSON(ctx context.Context, method, path string, quer } return nil } - return ops.Execute(ctx, call, ops.WithRetrier(e.newControlPlaneRetrier), ops.WithTimeout(e.tun.cpRetryTimeout)) + return call, nil } func (e *engine) newControlPlaneRetrier() ops.Retrier { return apiretry.NewRetrier(e.tun.cpBackoff, apiretry.RetrierConfig{StatusCodes: controlPlaneRetryStatusCodes}) } +// controlPlaneRetrier returns retriers for a request to the current endpoint, +// capped at the retries the walk allows it now (see uploadContext.retries). +func (e *engine) controlPlaneRetrier(uc *uploadContext) func() ops.Retrier { + n := uc.retries(e.onWorkspace(uc.ep)) + return func() ops.Retrier { + if n < 0 { + return e.newControlPlaneRetrier() + } + return &cappedRetrier{Retrier: e.newControlPlaneRetrier(), left: n} + } +} + +// cappedRetrier retries what its Retrier does, at most left more times. +type cappedRetrier struct { + ops.Retrier + left int +} + +func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) { + if r.left <= 0 { + return 0, false + } + d, ok := r.Retrier.IsRetriable(err) + if ok { + r.left-- + } + return d, ok +} + // setWorkspaceHeader applies the workspace routing header when the client is // workspace-scoped. The Files API control plane routes a request to the right // workspace by this header; the credentials are attached separately by @@ -333,18 +424,30 @@ func (e *engine) setWorkspaceHeader(req *http.Request) { } } -// controlPlaneURL joins the client host with a path and optional query, +// endpointURL joins an endpoint's base URL with a path and optional query, // mirroring the generated client's URL construction. -func (e *engine) controlPlaneURL(path string, query url.Values) (string, error) { - baseURL, err := url.Parse(e.c.host) +func endpointURL(base, path string, query url.Values) (string, error) { + u, err := url.Parse(base) if err != nil { return "", err } - baseURL.Path = path + u.Path = path if len(query) > 0 { - baseURL.RawQuery = query.Encode() + u.RawQuery = query.Encode() + } + return u.String(), nil +} + +// asAlreadyExistsOn is asAlreadyExists for a request to the current endpoint. +// The storage proxy answers an existing file with a 412 rather than the +// workspace's 409 ALREADY_EXISTS, which an upload that doesn't overwrite maps +// to ErrAlreadyExists too. +func (e *engine) asAlreadyExistsOn(uc *uploadContext, err error) error { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.HTTPStatusCode() == http.StatusPreconditionFailed && + !e.onWorkspace(uc.ep) && uc.overwrite != nil && !*uc.overwrite { + return ErrAlreadyExists } - return baseURL.String(), nil + return asAlreadyExists(err) } // asAlreadyExists maps an "already exists" API error to the ErrAlreadyExists diff --git a/files/v2/ext_upload_control_test.go b/files/v2/ext_upload_control_test.go index 78eaf4c6..66639947 100644 --- a/files/v2/ext_upload_control_test.go +++ b/files/v2/ext_upload_control_test.go @@ -7,7 +7,11 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" + "time" + + "github.com/databricks/sdk-go/core/ops" ) // newControlEngine builds an engine whose control plane points at an httptest @@ -21,6 +25,15 @@ func newControlEngine(t *testing.T, h http.HandlerFunc) *engine { return newEngine(buildUploadClient(t, srv.URL, srv.Client(), "")) } +// controlContext returns the context of an upload of /Volumes/c/s/v/f.bin to +// e's workspace host, as the walk's last endpoint, so that its requests keep +// the API client's retries. +func controlContext(e *engine, overwrite *bool) *uploadContext { + uc := &uploadContext{tun: e.tun, targetPath: "/Volumes/c/s/v/f.bin", overwrite: overwrite} + uc.startEndpoint(endpoint{kind: presignedEndpoint, baseURL: e.c.host, multipart: true}, true) + return uc +} + // requireCredentials rejects a control-plane request that does not carry // testToken, the way the Files API rejects one with "Credential was not sent". func requireCredentials(t *testing.T, h http.HandlerFunc) http.HandlerFunc { @@ -51,16 +64,13 @@ func TestInitiate(t *testing.T) { }) ow := true - res, err := c.initiateUpload(t.Context(), "/Volumes/c/s/v/f.bin", &ow) + res, err := c.initiateUpload(t.Context(), controlContext(c, &ow)) if err != nil { t.Fatal(err) } if res.MultipartUpload == nil || res.MultipartUpload.SessionToken != "tok" { t.Errorf("multipart = %+v", res.MultipartUpload) } - if res.ResumableUpload != nil { - t.Error("resumable should be nil") - } } func TestCreatePartURLs(t *testing.T) { @@ -68,7 +78,7 @@ func TestCreatePartURLs(t *testing.T) { _, _ = io.WriteString(w, `{"upload_part_urls":[{"url":"https://cloud.test/part/1","part_number":1,"headers":[{"name":"x-h","value":"v"}]}]}`) }) - urls, err := c.createPartURLs(t.Context(), "/Volumes/c/s/v/f.bin", "tok", 1, 1) + urls, err := c.createPartURLs(t.Context(), controlContext(c, nil), "tok", 1, 1) if err != nil { t.Fatal(err) } @@ -84,45 +94,36 @@ func TestCreatePartURLsEmpty(t *testing.T) { c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, `{"upload_part_urls":[]}`) }) - if _, err := c.createPartURLs(t.Context(), "/Volumes/c/s/v/f.bin", "tok", 1, 1); !errors.Is(err, errUnexpectedServerResponse) { + if _, err := c.createPartURLs(t.Context(), controlContext(c, nil), "tok", 1, 1); !errors.Is(err, errUnexpectedServerResponse) { t.Fatalf("err = %v, want errUnexpectedServerResponse", err) } } -func TestCreateResumableURL(t *testing.T) { +func TestAbortMultipart(t *testing.T) { + // The abort is an authenticated POST with the abort-upload action, sent once + // even when it fails with a retryable status. + var requests atomic.Int32 c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, `{"resumable_upload_url":{"url":"https://cloud.test/resumable","headers":[{"name":"x-h","value":"v"}]}}`) - }) - - purl, err := c.createResumableURL(t.Context(), "/Volumes/c/s/v/f.bin", "tok") - if err != nil { - t.Fatal(err) - } - if purl.URL != "https://cloud.test/resumable" || purl.Headers["x-h"] != "v" { - t.Errorf("url = %+v", purl) - } -} - -func TestCreateResumableURLMissing(t *testing.T) { - c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, `{}`) - }) - if _, err := c.createResumableURL(t.Context(), "/Volumes/c/s/v/f.bin", "tok"); !errors.Is(err, errUnexpectedServerResponse) { - t.Fatalf("err = %v, want errUnexpectedServerResponse", err) - } -} - -func TestCreateAbortURL(t *testing.T) { - c := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, `{"abort_upload_url":{"url":"https://cloud.test/abort"}}`) + requests.Add(1) + q := r.URL.Query() + if r.Method != http.MethodPost || r.URL.Path != "/api/2.0/fs/files/Volumes/c/s/v/f.bin" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + } + if q.Get("action") != "abort-upload" || q.Get("upload_type") != "multipart" || q.Get("session_token") != "tok" { + t.Errorf("query = %v", q) + } + w.WriteHeader(http.StatusServiceUnavailable) }) + // Fast retries, so that a retried abort shows up as a second request rather + // than as a test that runs out its retry budget. + c.tun.cpBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} + c.tun.cpRetryTimeout = time.Second - purl, err := c.createAbortURL(t.Context(), "/Volumes/c/s/v/f.bin", "tok") - if err != nil { - t.Fatal(err) + if err := c.abortMultipart(t.Context(), controlContext(c, nil), "tok"); err == nil { + t.Fatal("expected the 503 to be returned") } - if purl.URL != "https://cloud.test/abort" { - t.Errorf("url = %q", purl.URL) + if got := requests.Load(); got != 1 { + t.Errorf("requests = %d, want 1", got) } } @@ -145,7 +146,7 @@ func TestCompleteMultipartSortsParts(t *testing.T) { w.WriteHeader(http.StatusOK) }) - err := c.completeMultipart(t.Context(), "/Volumes/c/s/v/f.bin", "tok", map[int]string{3: "e3", 1: "e1", 2: "e2"}) + err := c.completeMultipart(t.Context(), controlContext(c, nil), "tok", map[int]string{3: "e3", 1: "e1", 2: "e2"}) if err != nil { t.Fatal(err) } @@ -163,21 +164,21 @@ func TestControlPlaneAlreadyExists(t *testing.T) { } // complete-upload (the multipart finish) returning 409 ALREADY_EXISTS maps to - // the sentinel so callers can skip-if-exists, matching single-shot/resumable. + // the sentinel so callers can skip-if-exists, matching single-shot. c := newControlEngine(t, conflict("ALREADY_EXISTS")) - if err := c.completeMultipart(t.Context(), "/Volumes/c/s/v/f.bin", "tok", map[int]string{1: "e1"}); !errors.Is(err, ErrAlreadyExists) { + if err := c.completeMultipart(t.Context(), controlContext(c, nil), "tok", map[int]string{1: "e1"}); !errors.Is(err, ErrAlreadyExists) { t.Errorf("completeMultipart err = %v, want ErrAlreadyExists", err) } // initiate-upload maps the same way. c2 := newControlEngine(t, conflict("ALREADY_EXISTS")) - if _, err := c2.initiateUpload(t.Context(), "/Volumes/c/s/v/f.bin", nil); !errors.Is(err, ErrAlreadyExists) { + if _, err := c2.initiateUpload(t.Context(), controlContext(c2, nil)); !errors.Is(err, ErrAlreadyExists) { t.Errorf("initiateUpload err = %v, want ErrAlreadyExists", err) } // A different 409 is not mistaken for the already-exists sentinel. c3 := newControlEngine(t, conflict("RESOURCE_CONFLICT")) - if _, err := c3.initiateUpload(t.Context(), "/Volumes/c/s/v/f.bin", nil); errors.Is(err, ErrAlreadyExists) { + if _, err := c3.initiateUpload(t.Context(), controlContext(c3, nil)); errors.Is(err, ErrAlreadyExists) { t.Errorf("a non-ALREADY_EXISTS 409 mapped to ErrAlreadyExists: %v", err) } } @@ -205,7 +206,7 @@ func TestSingleShotUploadOverwrite(t *testing.T) { gotContentType = r.Header.Get("Content-Type") w.WriteHeader(http.StatusOK) }) - err := c.uploadSingleShot(t.Context(), "/Volumes/c/s/v/f.bin", tc.overwrite, strings.NewReader("hello")) + err := c.uploadSingleShot(t.Context(), controlContext(c, tc.overwrite), strings.NewReader("hello"), 5) if err != nil { t.Fatal(err) } diff --git a/files/v2/ext_upload_data.go b/files/v2/ext_upload_data.go new file mode 100644 index 00000000..9b2f3b9a --- /dev/null +++ b/files/v2/ext_upload_data.go @@ -0,0 +1,209 @@ +package files + +import ( + "bytes" + "io" + "sync" +) + +// uploadData is the source of one upload. The walk sends it from the start to +// each endpoint it tries. +type uploadData struct { + // r reads the data in order, from the start of the current endpoint's + // attempt. + r io.Reader + // seeker rewinds r for the next endpoint. It is set when the size was + // learned by seeking. + seeker io.Seeker + // ra reads parts with positioned reads, without buffering them. It is set + // for a source of known size that has random access. + ra io.ReaderAt + // stream is set for a one-pass stream, which r reads through. + stream *replayStream + // knownSize is the size of a seekable source. + knownSize int64 +} + +// newSeekableData returns the data of r, of the given size, which r was sized +// by seeking and is positioned at the start of. +func newSeekableData(r io.ReadSeeker, size int64) *uploadData { + d := &uploadData{r: r, seeker: r, knownSize: size} + if ra, ok := r.(io.ReaderAt); ok { + d.ra = ra + } + return d +} + +// newStreamData returns the data of a one-pass stream. +func newStreamData(r io.Reader) *uploadData { + s := &replayStream{src: r, keeping: true} + return &uploadData{r: s, stream: s, knownSize: -1} +} + +// size is the size of the data, or -1 while a stream has not ended. +func (d *uploadData) size() int64 { + if d.stream != nil { + return d.stream.size() + } + return d.knownSize +} + +// readAhead reads up to n bytes of a stream ahead, so that a shorter one +// learns its size. A seekable source knows it already. +func (d *uploadData) readAhead(n int64) error { + if d.stream == nil { + return nil + } + return d.stream.readAhead(n) +} + +// canSendAgain reports whether the data can be sent from the start to another +// endpoint. +func (d *uploadData) canSendAgain() bool { + return d.stream == nil || d.stream.canRestart() +} + +// restart positions the data at the start for the next endpoint. +func (d *uploadData) restart() error { + if d.stream != nil { + d.stream.restart() + return nil + } + _, err := d.seeker.Seek(0, io.SeekStart) + return err +} + +// whole returns the data for a single-shot PUT. A seekable source is the body +// itself, so the PUT can rewind it to retry; a stream is a bytes.Reader only +// if it ended within the bytes it kept. +func (d *uploadData) whole() io.Reader { + if d.stream != nil { + return d.stream.whole() + } + return d.r +} + +// commit is called when the first part of the upload lands: it will not be +// sent again from the start, so a stream stops keeping what it reads. +func (d *uploadData) commit() { + if d.stream != nil { + d.stream.commit() + } +} + +// replayStream reads a one-pass stream and keeps every byte it reads, from the +// start, until the upload commits, so that the walk can send the stream again +// to the next endpoint. A multipart upload commits on its first part, so what +// it keeps is bounded by the read-ahead and one part. +type replayStream struct { + src io.Reader + + mu sync.Mutex + // kept holds the bytes read from src since the start. After commit it holds + // only bytes read ahead but not yet handed out. + kept []byte + // pos is how many bytes of kept Read has handed out. + pos int + total int64 // bytes read from src + keeping bool + eof bool + // lost records that bytes were handed out without being kept, so the + // stream can't start again. + lost bool +} + +func (s *replayStream) Read(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.pos < len(s.kept) { + n := copy(p, s.kept[s.pos:]) + s.pos += n + if !s.keeping && s.pos == len(s.kept) { + s.kept, s.pos = nil, 0 + } + return n, nil + } + if s.eof { + return 0, io.EOF + } + n, err := s.src.Read(p) + s.total += int64(n) + if n > 0 { + if s.keeping { + s.kept = append(s.kept, p[:n]...) + s.pos += n + } else { + s.lost = true + } + } + if err == io.EOF { + s.eof = true + } + return n, err +} + +func (s *replayStream) readAhead(n int64) error { + s.mu.Lock() + defer s.mu.Unlock() + if !s.keeping || s.eof || int64(len(s.kept)) >= n { + return nil + } + buf := make([]byte, n-int64(len(s.kept))) + m, err := io.ReadFull(s.src, buf) + s.total += int64(m) + s.kept = append(s.kept, buf[:m]...) + switch err { + case nil: + return nil + case io.EOF, io.ErrUnexpectedEOF: + s.eof = true + return nil + } + return err +} + +func (s *replayStream) size() int64 { + s.mu.Lock() + defer s.mu.Unlock() + if s.eof { + return s.total + } + return -1 +} + +func (s *replayStream) canRestart() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.keeping && !s.lost +} + +func (s *replayStream) restart() { + s.mu.Lock() + defer s.mu.Unlock() + s.pos = 0 +} + +func (s *replayStream) whole() io.Reader { + s.mu.Lock() + defer s.mu.Unlock() + if s.eof && s.keeping { + return bytes.NewReader(s.kept) + } + // The rest is streamed and can't be read again; keeping it would hold the + // whole stream in memory. + s.keeping, s.lost = false, true + rest := io.MultiReader(bytes.NewReader(s.kept[s.pos:]), s.src) + s.kept, s.pos = nil, 0 + return rest +} + +func (s *replayStream) commit() { + s.mu.Lock() + defer s.mu.Unlock() + if !s.keeping { + return + } + s.keeping = false + s.kept = s.kept[s.pos:] + s.pos = 0 +} diff --git a/files/v2/ext_upload_helpers.go b/files/v2/ext_upload_helpers.go index 99592a3b..0f7f31de 100644 --- a/files/v2/ext_upload_helpers.go +++ b/files/v2/ext_upload_helpers.go @@ -9,7 +9,6 @@ import ( "context" "io" "maps" - "net/http" ) // --- Buffer helpers --- @@ -59,10 +58,10 @@ func octetStreamHeaders(presigned map[string]string) map[string]string { // --- Single-shot delegation and best-effort abort glue --- -// singleShotUpload sends the whole body in one PUT through the control-plane -// single-shot path. Used for small files and as the multipart/resumable -// fallback. Progress is reported as the body streams. -func (e *engine) singleShotUpload(ctx context.Context, uc *uploadContext, body io.Reader) error { +// singleShotUpload sends the whole body, of size bytes or -1 if unknown, in one +// PUT to the current endpoint's Files API. Progress is reported as the body +// streams. +func (e *engine) singleShotUpload(ctx context.Context, uc *uploadContext, body io.Reader, size int64) error { if uc.progress != nil { body = withProgress(body, uc.progress) } @@ -72,30 +71,19 @@ func (e *engine) singleShotUpload(ctx context.Context, uc *uploadContext, body i return err } defer uc.limiter.Release() - return e.uploadSingleShot(ctx, uc.targetPath, uc.overwrite, body) -} - -// abortMultipartUpload mints a presigned abort URL on the control plane and -// issues the unauthenticated cloud DELETE against it. Both halves run on the -// caller's context, which abortMultipartBestEffort detaches and bounds. -func (e *engine) abortMultipartUpload(ctx context.Context, uc *uploadContext, token string) error { - purl, err := e.createAbortURL(ctx, uc.targetPath, token) - if err != nil { - return err - } - headers := octetStreamHeaders(purl.Headers) - _, err = uc.cloud.Send(ctx, http.MethodDelete, purl.URL, headers, nil) - return err + return e.uploadSingleShot(ctx, uc, body, size) } +// abortMultipartBestEffort aborts the multipart session on a context detached +// from the caller's (see cleanupContext), and logs rather than returns a +// failure. func (e *engine) abortMultipartBestEffort(ctx context.Context, uc *uploadContext, token string) { ctx, cancel := e.cleanupContext(ctx) defer cancel() - if err := e.abortMultipartUpload(ctx, uc, token); err != nil { - // Best-effort cleanup; its failure is not user-actionable (e.g. some clouds - // do not support abort presigned URLs), so it stays at debug rather than - // surfacing as a warning on an otherwise-normal outcome like a skip. - e.c.logger.DebugContext(ctx, "failed to abort multipart upload", "error", err) + if err := e.abortMultipart(ctx, uc, token); err != nil { + // Best-effort cleanup: the server expires abandoned sessions, so a failed + // abort is logged but doesn't change the upload's outcome. + e.c.logger.WarnContext(ctx, "failed to abort multipart upload", "error", err) } } @@ -107,10 +95,3 @@ func (e *engine) abortMultipartBestEffort(ctx context.Context, uc *uploadContext func (e *engine) cleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(context.WithoutCancel(ctx), e.tun.cleanupTimeout) } - -func (e *engine) abortResumableUpload(ctx context.Context, uc *uploadContext, purl presignedURL) error { - ctx, cancel := e.cleanupContext(ctx) - defer cancel() - _, err := uc.cloud.Send(ctx, http.MethodDelete, purl.URL, purl.Headers, nil) - return err -} diff --git a/files/v2/ext_upload_test.go b/files/v2/ext_upload_test.go index b98d25cc..bbbf046e 100644 --- a/files/v2/ext_upload_test.go +++ b/files/v2/ext_upload_test.go @@ -5,11 +5,14 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" "os" + "path/filepath" + "slices" "strconv" "strings" "sync" @@ -18,6 +21,8 @@ import ( "time" "github.com/databricks/sdk-go/auth" + "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/apierr/codes" "github.com/databricks/sdk-go/core/ops" "github.com/databricks/sdk-go/options/client" ) @@ -70,14 +75,27 @@ type fakeServer struct { srv *httptest.Server mu sync.Mutex - mode string // "multipart" or "resumable" + mode string // "multipart", or "resumable" to offer a resumable session parts map[int][]byte completed []completePart singleBody []byte - resumable []byte overwriteQ []string // overwrite query values seen on initiate + single-shot partHeaders []http.Header // headers seen on cloud part PUTs partURLReqs []int // Count of each create-upload-part-urls request + aborts []string // session_token of each abort-upload request + + // abortStatus, when non-zero, is returned by abort-upload. + abortStatus int + + // initiateStatus and initiateBody, when the status is non-zero, answer + // initiate-upload. + initiateStatus int + initiateBody string + + // partURLsStatus and partURLsBody, when the status is non-zero, answer + // create-upload-part-urls. + partURLsStatus int + partURLsBody string // partHook injects faults. It returns (statusCode, etag, body); a zero // statusCode means "use the default 200 + store". @@ -93,18 +111,6 @@ type fakeServer struct { // the default 200 + store". singleTry counts single-shot PUTs seen. singleShotHook func(attempt int) int singleTry int - - // stuckResumable makes a resumable chunk PUT confirm no new bytes, modeling a - // server/proxy wedged at a fixed offset. - stuckResumable bool - - // resumableChunkHook injects per-chunk faults on the resumable data PUT. For a - // non-zero returned status the chunk's bytes are still committed (so the - // client's status query reveals forward progress), but the response is that - // status -- modeling a spurious transient failure on a chunk that did land. - // resumableTry counts data-chunk PUTs (not status queries). - resumableChunkHook func(attempt int) int - resumableTry int } func newFakeServer(t *testing.T, mode string) *fakeServer { @@ -144,16 +150,8 @@ func (f *fakeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPost && strings.HasSuffix(path, "/create-upload-part-urls"): f.handleCreatePartURLs(w, r) - case r.Method == http.MethodPost && strings.HasSuffix(path, "/create-resumable-upload-url"): - writeJSON(w, resumableURLResponse{ResumableUploadURL: &urlWithHeaders{URL: f.base() + "/cloud/resumable"}}) - case r.Method == http.MethodPost && strings.HasSuffix(path, "/create-abort-upload-url"): - writeJSON(w, abortURLResponse{AbortUploadURL: &urlWithHeaders{URL: f.base() + "/cloud/abort"}}) case r.Method == http.MethodPut && strings.HasPrefix(path, "/cloud/part/"): f.handlePartPut(w, r) - case r.Method == http.MethodPut && path == "/cloud/resumable": - f.handleResumablePut(w, r) - case r.Method == http.MethodDelete && (path == "/cloud/abort" || path == "/cloud/resumable"): - w.WriteHeader(http.StatusOK) case strings.Contains(path, "/api/2.0/fs/files/"): f.handleFiles(w, r) default: @@ -168,11 +166,31 @@ func (f *fakeServer) handleFiles(w http.ResponseWriter, r *http.Request) { f.mu.Lock() f.overwriteQ = append(f.overwriteQ, r.URL.Query().Get("overwrite")) f.mu.Unlock() + if f.initiateStatus != 0 { + w.WriteHeader(f.initiateStatus) + _, _ = io.WriteString(w, f.initiateBody) + return + } if f.mode == "resumable" { - writeJSON(w, initiateResult{ResumableUpload: &uploadSession{SessionToken: "tok"}}) + // A resumable session, which this package does not support. + writeJSON(w, map[string]any{"resumable_upload": map[string]any{"session_token": "tok"}}) } else { writeJSON(w, initiateResult{MultipartUpload: &uploadSession{SessionToken: "tok"}}) } + case r.Method == http.MethodPost && action == "abort-upload": + if q := r.URL.Query(); q.Get("upload_type") != "multipart" { + http.Error(w, "abort-upload without upload_type=multipart", http.StatusBadRequest) + return + } + f.mu.Lock() + f.aborts = append(f.aborts, r.URL.Query().Get("session_token")) + status := f.abortStatus + f.mu.Unlock() + if status != 0 { + w.WriteHeader(status) + return + } + w.WriteHeader(http.StatusOK) case r.Method == http.MethodPost && action == "complete-upload": var req completeRequest _ = json.NewDecoder(r.Body).Decode(&req) @@ -213,6 +231,11 @@ func (f *fakeServer) handleCreatePartURLs(w http.ResponseWriter, r *http.Request f.mu.Lock() f.partURLReqs = append(f.partURLReqs, req.Count) f.mu.Unlock() + if f.partURLsStatus != 0 { + w.WriteHeader(f.partURLsStatus) + _, _ = io.WriteString(w, f.partURLsBody) + return + } var out createPartURLsResponse for i := range req.Count { n := req.StartPartNumber + i @@ -254,55 +277,11 @@ func (f *fakeServer) handlePartPut(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (f *fakeServer) handleResumablePut(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - cr := r.Header.Get("Content-Range") - // Status query: "bytes */*" -> report current confirmed offset. - if cr == "bytes */*" { - f.mu.Lock() - n := len(f.resumable) - f.mu.Unlock() - if n == 0 { - w.WriteHeader(http.StatusPermanentRedirect) - return - } - w.Header().Set("Range", "bytes=0-"+strconv.Itoa(n-1)) - w.WriteHeader(http.StatusPermanentRedirect) - return - } - - // A wedged server: accept a non-final chunk but confirm no new bytes (no Range - // header), so the client sees a zero-progress 308 and must bound its retries. - if f.stuckResumable && strings.HasSuffix(cr, "/*") { - w.WriteHeader(http.StatusPermanentRedirect) - return - } - +// abortsSeen returns the session_token of each abort-upload request. +func (f *fakeServer) abortsSeen() []string { f.mu.Lock() - f.resumable = append(f.resumable, body...) - total := len(f.resumable) - f.resumableTry++ - hook, attempt := f.resumableChunkHook, f.resumableTry - f.mu.Unlock() - - // The chunk's bytes are committed above; a hooked transient status models a - // blip on a chunk that actually landed, so the client's follow-up status - // query sees forward progress. - if hook != nil { - if s := hook(attempt); s != 0 { - w.WriteHeader(s) - return - } - } - - // Content-Range: bytes {start}-{end}/{total|*} - final := !strings.HasSuffix(cr, "/*") - if final { - w.WriteHeader(http.StatusOK) - return - } - w.Header().Set("Range", "bytes=0-"+strconv.Itoa(total-1)) - w.WriteHeader(http.StatusPermanentRedirect) + defer f.mu.Unlock() + return slices.Clone(f.aborts) } // assembled returns the bytes the server reconstructed for the upload. @@ -312,9 +291,6 @@ func (f *fakeServer) assembled() []byte { if f.singleBody != nil { return f.singleBody } - if f.mode == "resumable" { - return f.resumable - } var out []byte for _, p := range f.completed { out = append(out, f.parts[p.PartNumber]...) @@ -564,6 +540,10 @@ func TestCloudRejectsAllPartsFallsBackToSingleShot(t *testing.T) { noErr(t, err) // The fallback single-shot PUT carries the whole object. eqBytes(t, f.assembled(), in) + // The abandoned multipart session is aborted before falling back. + if got, want := f.abortsSeen(), []string{"tok"}; !slices.Equal(got, want) { + t.Errorf("aborts = %v, want %v", got, want) + } } func TestUploadAlreadyExists(t *testing.T) { @@ -633,6 +613,251 @@ func TestSingleShotRetriesWithProgress(t *testing.T) { } } +func TestSingleShotRetriesAFileBody(t *testing.T) { + // An *os.File body is an io.Closer, which net/http closes after each attempt. + // The retry must still rewind it, and the caller's file must stay open. + in := data(500) // below the 1024 single-shot threshold + src := filepath.Join(t.TempDir(), "src.bin") + if err := os.WriteFile(src, in, 0o600); err != nil { + t.Fatal(err) + } + upload := map[string]func(*engine) error{ + "UploadFrom": func(e *engine) error { + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/p.bin", src) + return err + }, + "Upload with the caller's file": func(e *engine) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + if _, err := e.upload(t.Context(), "/Volumes/c/s/v/p.bin", f); err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("the caller's file was closed: %w", err) + } + return nil + }, + } + for name, run := range upload { + t.Run(name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.singleShotHook = func(attempt int) int { + if attempt == 1 { + return http.StatusServiceUnavailable + } + return 0 + } + e := shrunkEngine(t, f) + e.tun.cpBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} + + noErr(t, run(e)) + eqBytes(t, f.assembled(), in) + if f.singleTry != 2 { + t.Errorf("single-shot attempts = %d, want 2 (one retry after the 503)", f.singleTry) + } + }) + } +} + +func TestSingleShotRetryIsNotCutShortByALateRead(t *testing.T) { + // net/http may still read a request body after the attempt returns, until + // it closes it. This transport answers the first attempt with a 503 at once + // and keeps reading that attempt's body while the retry runs, as a write + // loop still sending it would. The retry must still send every byte. + in := data(500) // below the 1024 single-shot threshold + src := filepath.Join(t.TempDir(), "src.bin") + noErr(t, os.WriteFile(src, in, 0o600)) + uploads := map[string]struct { + upload func(*engine) error + // randomAccess is whether net/http can also send the body again itself. + randomAccess bool + }{ + "a file": {func(e *engine) error { + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/p.bin", src) + return err + }, true}, + "a reader without random access": {func(e *engine) error { + _, err := e.upload(t.Context(), "/Volumes/c/s/v/p.bin", struct{ io.ReadSeeker }{bytes.NewReader(in)}) + return err + }, false}, + } + for name, tc := range uploads { + t.Run(name, func(t *testing.T) { + retrying, lateReadDone := make(chan struct{}), make(chan struct{}) + var attempts int + var sent, resent []byte + var sentLength int64 + rt := roundTripFunc(func(r *http.Request) (*http.Response, error) { + attempts++ + if attempts == 1 { + go func() { + // Wait for the retry, but not forever: a retry that + // waits for this attempt's body to close never starts. + select { + case <-retrying: + case <-time.After(100 * time.Millisecond): + } + _, _ = io.CopyN(io.Discard, r.Body, 100) + _ = r.Body.Close() + close(lateReadDone) + }() + return cannedResponse(r, http.StatusServiceUnavailable, `{"error_code":"TEMPORARILY_UNAVAILABLE","message":"busy"}`), nil + } + if attempts == 2 { + close(retrying) + } + <-lateReadDone + var err error + sent, err = io.ReadAll(r.Body) + _ = r.Body.Close() + sentLength = r.ContentLength + if err != nil { + return nil, err + } + if r.GetBody != nil { + again, err := r.GetBody() + if err != nil { + return nil, err + } + resent, _ = io.ReadAll(again) + } + return cannedResponse(r, http.StatusNoContent, ""), nil + }) + e := newEngine(buildUploadClient(t, "https://ws.example.com", &http.Client{Transport: rt}, "")) + e.tun.minStreamSize = 1024 + e.tun.cpBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} + + noErr(t, tc.upload(e)) + if attempts != 2 { + t.Errorf("attempts = %d, want 2 (one retry after the 503)", attempts) + } + eqBytes(t, sent, in) + if sentLength != int64(len(in)) { + t.Errorf("Content-Length = %d, want %d", sentLength, len(in)) + } + if tc.randomAccess { + eqBytes(t, resent, in) + } else if resent != nil { + t.Errorf("GetBody sent %d bytes, want no GetBody: the body cannot be read twice at once", len(resent)) + } + }) + } +} + +func TestSingleShotRetryWaitingForTheLastBodyStopsWhenCancelled(t *testing.T) { + // A retry of data without random access waits for net/http to close the + // last attempt's body. This transport never closes it, so only the + // caller's cancellation can end the wait. + var attempts atomic.Int32 + rt := roundTripFunc(func(r *http.Request) (*http.Response, error) { + attempts.Add(1) + return cannedResponse(r, http.StatusServiceUnavailable, `{"error_code":"TEMPORARILY_UNAVAILABLE","message":"busy"}`), nil + }) + e := newEngine(buildUploadClient(t, "https://ws.example.com", &http.Client{Transport: rt}, "")) + e.tun.minStreamSize = 1024 + e.tun.cpBackoff = ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} + + stop := errors.New("the caller gave up") + ctx, cancel := context.WithCancelCause(t.Context()) + time.AfterFunc(50*time.Millisecond, func() { cancel(stop) }) + done := make(chan error, 1) + go func() { + _, err := e.upload(ctx, "/Volumes/c/s/v/p.bin", struct{ io.ReadSeeker }{bytes.NewReader(data(500))}) + done <- err + }() + select { + case err := <-done: + if !errors.Is(err, stop) { + t.Errorf("err = %v, want the cancellation cause %v", err, stop) + } + case <-time.After(5 * time.Second): + t.Fatal("the upload still waits for a body that is never closed after its context was cancelled") + } + if got := attempts.Load(); got != 1 { + t.Errorf("attempts = %d, want 1: the retry waits for the first attempt's body", got) + } +} + +// closeRecorder is a one-pass body that records whether it was closed. +type closeRecorder struct { + io.Reader + closed atomic.Bool +} + +func (r *closeRecorder) Close() error { + r.closed.Store(true) + return nil +} + +func TestSingleShotDoesNotCloseANonSeekableBody(t *testing.T) { + // net/http closes a request body that is an io.Closer. A body without Seek + // is sent once and stays the caller's, so the PUT must not close it. + var got []byte + e := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { + got, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + }) + body := &closeRecorder{Reader: strings.NewReader("hello")} + noErr(t, e.uploadSingleShot(t.Context(), controlContext(e, nil), body, -1)) + if string(got) != "hello" { + t.Errorf("sent %q, want %q", got, "hello") + } + if body.closed.Load() { + t.Error("the PUT closed the caller's body") + } +} + +func TestSingleShotSendsContentLength(t *testing.T) { + // Hosts require a Content-Length, which net/http can't tell for a file or + // for the body of an attempt; the single shot sets it from the body's size. + src := filepath.Join(t.TempDir(), "f.bin") + noErr(t, os.WriteFile(src, []byte("hello"), 0o600)) + cases := map[string]struct { + upload func(*engine) error + size int64 + }{ + "a file": {func(e *engine) error { + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/f.bin", src) + return err + }, 5}, + "an empty body": {func(e *engine) error { + _, err := e.upload(t.Context(), "/Volumes/c/s/v/f.bin", bytes.NewReader(nil)) + return err + }, 0}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + var gotLength int64 + var gotEncoding []string + e := newControlEngine(t, func(w http.ResponseWriter, r *http.Request) { + gotLength, gotEncoding = r.ContentLength, r.TransferEncoding + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(http.StatusOK) + }) + noErr(t, tc.upload(e)) + if gotLength != tc.size || len(gotEncoding) != 0 { + t.Errorf("Content-Length = %d, Transfer-Encoding = %v, want %d and none", gotLength, gotEncoding, tc.size) + } + }) + } +} + +func cannedResponse(r *http.Request, status int, body string) *http.Response { + return &http.Response{ + Status: http.StatusText(status), + StatusCode: status, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(body)), + Request: r, + } +} + type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } @@ -657,73 +882,322 @@ func TestUploadWithTransferClient(t *testing.T) { } } -func TestUploadResumableHappyPath(t *testing.T) { - f := newFakeServer(t, "resumable") +func TestUploadPresignedURLExpiryRetried(t *testing.T) { + f := newFakeServer(t, "multipart") + // Every part's first presigned URL is reported expired; the second succeeds. + azure := `AuthenticationFailedSignature not valid in the specified time frame` + f.partHook = func(n, attempt int) (int, string, []byte) { + if attempt == 1 { + return http.StatusForbidden, "", []byte(azure) + } + return 0, "", nil + } e := shrunkEngine(t, f) in := data(5000) - // Parallelism > 1 still uses single-threaded resumable; bytes must match. - _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/gcp.bin", writeTemp(t, in), WithParallelism(4)) + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/exp.bin", writeTemp(t, in), WithParallelism(4)) noErr(t, err) eqBytes(t, f.assembled(), in) } -func TestUploadResumableNoProgressBounded(t *testing.T) { - f := newFakeServer(t, "resumable") - f.stuckResumable = true +// Retry-timing coverage (retryable-status retried/exhausted, with shrunk backoff) +// lives in the cloudstorage package, where Send's retry policy is defined. + +// walkEngine is a shrunkEngine whose retries back off in milliseconds. +func walkEngine(t *testing.T, f *fakeServer) *engine { + t.Helper() e := shrunkEngine(t, f) + fast := ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} + e.tun.cpBackoff, e.tun.cloudBackoff = fast, fast + return e +} - // The server confirms no new bytes on every chunk; the resumable loop must - // error rather than spin forever (fs cp sets no context deadline). - _, err := e.upload(t.Context(), "/Volumes/c/s/v/gcp.bin", bytes.NewReader(data(5000))) - if err == nil { - t.Fatal("expected an error when the resumable server makes no progress") +func TestFailureBeforeAnyPartLandsFallsBackToSingleShot(t *testing.T) { + // Before any part lands, a part that keeps failing gets one retry, and then + // the whole upload is sent in a single shot; a transient failure is no + // different from a cloud rejection. + for _, parallelism := range []int{1, 4} { + t.Run(strconv.Itoa(parallelism), func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.partHook = func(n, attempt int) (int, string, []byte) { + return http.StatusServiceUnavailable, "", nil + } + e := walkEngine(t, f) + + in := data(5000) + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/fb.bin", writeTemp(t, in), WithParallelism(parallelism)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + f.mu.Lock() + defer f.mu.Unlock() + for n, tries := range f.partTry { + if tries > 2 { + t.Errorf("part %d was attempted %d times, want at most 2", n, tries) + } + } + if f.singleTry != 1 { + t.Errorf("single-shot PUTs = %d, want 1", f.singleTry) + } + }) } } -// TestUploadResumableTransientRetryBudgetIsPerChunk guards against the retry -// budget being a global cap rather than per-progress: every chunk PUT returns a -// transient 503 (but commits its bytes, so the status query shows forward -// progress). With more chunks than maxRetries, a global budget would fail the -// upload after maxRetries total blips even though it keeps advancing; the budget -// must reset on each confirmed advance so a steadily-progressing flaky upload -// completes. -func TestUploadResumableTransientRetryBudgetIsPerChunk(t *testing.T) { - f := newFakeServer(t, "resumable") - e := shrunkEngine(t, f) // 1024-byte parts - // Every data-chunk PUT reports a retryable 503 after committing; with a - // 5000-byte input at 1024-byte chunks that is ~5 chunks, well over maxRetries. - f.resumableChunkHook = func(attempt int) int { return http.StatusServiceUnavailable } +func TestFailureAfterAPartLandsIsRetriedThenReturned(t *testing.T) { + f := newFakeServer(t, "multipart") + f.partHook = func(n, attempt int) (int, string, []byte) { + if n == 1 { + return 0, "", nil + } + return http.StatusServiceUnavailable, "", nil + } + e := walkEngine(t, f) + + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/c.bin", writeTemp(t, data(5000)), WithParallelism(1)) + if aerr, ok := errors.AsType[*apierr.APIError](err); !ok || aerr.HTTPStatusCode() != http.StatusServiceUnavailable { + t.Fatalf("err = %v, want the last failure, a 503", err) + } + f.mu.Lock() + defer f.mu.Unlock() + if got, want := f.partTry[2], e.tun.cloudRetries+1; got != want { + t.Errorf("part 2 attempts = %d, want %d", got, want) + } + if f.singleTry != 0 { + t.Errorf("single-shot PUTs = %d, want none once a part has landed", f.singleTry) + } +} + +func TestFailureThatTheSingleShotWouldHitTooIsReturned(t *testing.T) { + cases := []struct { + name string + setup func(*fakeServer) + check func(*testing.T, error) + }{ + { + name: "a part fails its precondition", + setup: func(f *fakeServer) { + f.partHook = func(n, attempt int) (int, string, []byte) { + return http.StatusPreconditionFailed, "", nil + } + }, + check: func(t *testing.T, err error) { + if aerr, ok := errors.AsType[*apierr.APIError](err); !ok || aerr.HTTPStatusCode() != http.StatusPreconditionFailed { + t.Errorf("err = %v, want a 412", err) + } + }, + }, + { + name: "initiate finds the file exists", + setup: func(f *fakeServer) { + f.initiateStatus = http.StatusConflict + f.initiateBody = `{"error_code":"ALREADY_EXISTS","message":"exists"}` + }, + check: func(t *testing.T, err error) { + if !errors.Is(err, ErrAlreadyExists) { + t.Errorf("err = %v, want ErrAlreadyExists", err) + } + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + tc.setup(f) + e := walkEngine(t, f) + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/x.bin", writeTemp(t, data(5000)), WithParallelism(1)) + tc.check(t, err) + if f.singleTry != 0 { + t.Errorf("single-shot PUTs = %d, want none", f.singleTry) + } + }) + } +} + +func TestFailedInitiateFallsBackToSingleShot(t *testing.T) { + sources := map[string]func([]byte) io.Reader{ + "seekable": func(b []byte) io.Reader { return bytes.NewReader(b) }, + "non-seekable": func(b []byte) io.Reader { return io.MultiReader(bytes.NewReader(b)) }, + } + for name, source := range sources { + t.Run(name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.initiateStatus = http.StatusInternalServerError + f.initiateBody = `{"error_code":"INTERNAL_ERROR","message":"boom"}` + e := walkEngine(t, f) + + // Larger than the single-shot threshold, so the read-ahead of the + // non-seekable stream is followed by the rest of it. + in := data(5000) + _, err := e.upload(t.Context(), "/Volumes/c/s/v/i.bin", source(in), WithParallelism(1)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + }) + } +} + +func TestFailedPartURLMintFallsBackToSingleShot(t *testing.T) { + // Each driver mints URLs its own way: the sequential loop per batch, the file + // producer ahead of its workers, and the stream's workers on demand. + uploads := map[string]func(*engine, []byte) error{ + "sequential": func(e *engine, in []byte) error { + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/m.bin", writeTemp(t, in), WithParallelism(1)) + return err + }, + "parallel file": func(e *engine, in []byte) error { + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/m.bin", writeTemp(t, in), WithParallelism(4)) + return err + }, + "parallel stream": func(e *engine, in []byte) error { + _, err := e.upload(t.Context(), "/Volumes/c/s/v/m.bin", io.MultiReader(bytes.NewReader(in)), WithParallelism(4)) + return err + }, + } + for name, upload := range uploads { + t.Run(name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.partURLsStatus = http.StatusInternalServerError + f.partURLsBody = `{"error_code":"INTERNAL_ERROR","message":"boom"}` + e := walkEngine(t, f) + + in := data(5000) + noErr(t, upload(e, in)) + eqBytes(t, f.assembled(), in) + if f.singleTry != 1 { + t.Errorf("single-shot PUTs = %d, want 1", f.singleTry) + } + }) + } + + t.Run("the file exists", func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.partURLsStatus = http.StatusConflict + f.partURLsBody = `{"error_code":"ALREADY_EXISTS","message":"exists"}` + e := walkEngine(t, f) + + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/m.bin", writeTemp(t, data(5000)), WithParallelism(1)) + if apierr.Code(err) != codes.AlreadyExists { + t.Errorf("err = %v, want ALREADY_EXISTS", err) + } + if f.singleTry != 0 { + t.Errorf("single-shot PUTs = %d, want none", f.singleTry) + } + }) +} + +func TestControlPlaneFailureMovesOnAfterOneRetry(t *testing.T) { + // Before any part lands, a control-plane call that keeps failing gets one + // retry, and then the upload moves on to the single-shot PUT: as on any host, + // moving on is cheaper than waiting out the retry timeout. + cases := map[string]struct { + fault func(*fakeServer) + requests func(*fakeServer) int + }{ + "initiate": { + fault: func(f *fakeServer) { f.initiateStatus = http.StatusServiceUnavailable }, + requests: func(f *fakeServer) int { return len(f.overwriteQ) - f.singleTry }, + }, + "part URLs": { + fault: func(f *fakeServer) { f.partURLsStatus = http.StatusServiceUnavailable }, + requests: func(f *fakeServer) int { return len(f.partURLReqs) }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + tc.fault(f) + e := walkEngine(t, f) + + in := data(5000) + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/cp.bin", writeTemp(t, in), WithParallelism(1)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + f.mu.Lock() + defer f.mu.Unlock() + if got := tc.requests(f); got != 2 { + t.Errorf("%s requests = %d, want 2", name, got) + } + }) + } +} + +func TestStorageTimeoutFallsBackToSingleShot(t *testing.T) { + // A blackholed storage host times out rather than refusing the connection; + // that is the case the fallback exists for, so a deadline error from a part + // PUT falls back like any other failure. + f := newFakeServer(t, "multipart") + e := walkEngine(t, f) + blackholed := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("blackholed: %w", context.DeadlineExceeded) + })} in := data(5000) - _, err := e.upload(t.Context(), "/Volumes/c/s/v/flaky.bin", bytes.NewReader(in)) + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/t.bin", writeTemp(t, in), WithParallelism(1), WithTransferClient(blackholed)) noErr(t, err) eqBytes(t, f.assembled(), in) - if f.resumableTry <= e.tun.maxRetries { - t.Errorf("test did not exceed the retry budget: %d chunk PUTs, want > maxRetries (%d)", f.resumableTry, e.tun.maxRetries) - } } -func TestUploadPresignedURLExpiryRetried(t *testing.T) { +func TestSingleShotFallbackOfAPipe(t *testing.T) { + // A pipe is an *os.File, so it has Seek, but Seek fails: the fallback sends + // the bytes read so far and then the rest of the pipe. f := newFakeServer(t, "multipart") - // Every part's first presigned URL is reported expired; the second succeeds. - azure := `AuthenticationFailedSignature not valid in the specified time frame` f.partHook = func(n, attempt int) (int, string, []byte) { - if attempt == 1 { - return http.StatusForbidden, "", []byte(azure) - } - return 0, "", nil + return http.StatusForbidden, "", []byte("blocked") } - e := shrunkEngine(t, f) + e := walkEngine(t, f) in := data(5000) - _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/exp.bin", writeTemp(t, in), WithParallelism(4)) + pr, pw, err := os.Pipe() + noErr(t, err) + t.Cleanup(func() { _ = pr.Close() }) + go func() { + _, _ = pw.Write(in) + _ = pw.Close() + }() + _, err = e.upload(t.Context(), "/Volumes/c/s/v/p.bin", pr, WithParallelism(1)) noErr(t, err) eqBytes(t, f.assembled(), in) } -// Retry-timing coverage (retryable-status retried/exhausted, with shrunk backoff) -// lives in the cloudstorage package, where Send's retry policy is defined. +func TestSingleShotFallbackRetriesOnlyASeekableSource(t *testing.T) { + // The fallback rewinds a seekable source and sends it itself, so the PUT is + // retried; a one-pass stream has been read past the bytes it holds, so the + // PUT is sent once. + cases := []struct { + name string + source func([]byte) io.Reader + wantTries int + wantFailed bool + }{ + {"seekable", func(b []byte) io.Reader { return bytes.NewReader(b) }, 2, false}, + {"non-seekable", func(b []byte) io.Reader { return io.MultiReader(bytes.NewReader(b)) }, 1, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newFakeServer(t, "multipart") + f.partHook = func(n, attempt int) (int, string, []byte) { + return http.StatusForbidden, "", []byte("blocked") + } + f.singleShotHook = func(attempt int) int { + if attempt == 1 { + return http.StatusServiceUnavailable + } + return 0 + } + e := walkEngine(t, f) + + in := data(5000) + _, err := e.upload(t.Context(), "/Volumes/c/s/v/s.bin", tc.source(in), WithParallelism(1)) + if tc.wantFailed != (err != nil) { + t.Fatalf("err = %v, want failed = %v", err, tc.wantFailed) + } + if !tc.wantFailed { + eqBytes(t, f.assembled(), in) + } + if f.singleTry != tc.wantTries { + t.Errorf("single-shot PUTs = %d, want %d", f.singleTry, tc.wantTries) + } + }) + } +} func TestResolvedParallelism(t *testing.T) { n := func(v int) *int { return &v } @@ -752,3 +1226,134 @@ func TestResolvedParallelism(t *testing.T) { }) } } + +func TestUploadRejectsResumableSession(t *testing.T) { + // Only multipart sessions are supported. An initiate response with a + // resumable session and no multipart one is unexpected: no data goes to the + // session, and the upload moves on to the single-shot PUT, as for any other + // failure before a part lands. + f := newFakeServer(t, "resumable") + e := shrunkEngine(t, f) + if _, err := e.initiateMultipart(t.Context(), controlContext(e, nil)); !errors.Is(err, errUnexpectedServerResponse) { + t.Fatalf("initiateMultipart err = %v, want errUnexpectedServerResponse", err) + } + + in := data(4 * 1024) + _, err := e.upload(t.Context(), "/Volumes/c/s/v/r.bin", bytes.NewReader(in)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + f.mu.Lock() + defer f.mu.Unlock() + if len(f.partTry) != 0 { + t.Errorf("parts sent = %v, want none", f.partTry) + } +} + +func TestFailedMultipartUploadIsAbortedOnce(t *testing.T) { + // A part fails with a status that isn't retried after another part has + // landed, so the upload fails. Its session is aborted once, and a failing + // abort is not retried and does not replace the upload's own error. + f := newFakeServer(t, "multipart") + f.partHook = func(n, attempt int) (int, string, []byte) { + if n == 1 { + return 0, "", nil + } + return http.StatusInternalServerError, "", []byte("boom") + } + f.abortStatus = http.StatusServiceUnavailable + e := shrunkEngine(t, f) + + _, err := e.uploadFrom(t.Context(), "/Volumes/c/s/v/fail.bin", writeTemp(t, data(5000)), WithParallelism(1)) + if err == nil { + t.Fatal("expected the upload to fail") + } + if apierr.Code(err) != codes.Internal { + t.Errorf("err = %v, want the part's 500", err) + } + if got, want := f.abortsSeen(), []string{"tok"}; !slices.Equal(got, want) { + t.Errorf("aborts = %v, want %v", got, want) + } +} + +func TestParallelShortStreamIsSentInASingleShot(t *testing.T) { + // A stream of unknown size that turns out shorter than the single-shot + // threshold is sent in one PUT, without opening a multipart session, even + // when the upload is parallel. + for _, size := range []int{0, 100} { + f := newFakeServer(t, "multipart") + e := shrunkEngine(t, f) + in := data(size) + _, err := e.upload(t.Context(), "/Volumes/c/s/v/short.bin", struct{ io.Reader }{bytes.NewReader(in)}, WithParallelism(4)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + f.mu.Lock() + singleShots, initiates := f.singleTry, len(f.overwriteQ)-f.singleTry + f.mu.Unlock() + if singleShots != 1 || initiates != 0 { + t.Errorf("size %d: %d single shots and %d initiates, want 1 and 0", size, singleShots, initiates) + } + } +} + +func TestParallelStreamAroundTheSingleShotThreshold(t *testing.T) { + // The read-ahead decides between one PUT and a multipart session at exactly + // the threshold (1024 in the shrunk engine). With a part size that doesn't + // divide it, a part also straddles the end of the read-ahead, so the stream + // must be reassembled without losing or repeating its bytes. + for _, tc := range []struct { + size int + multipart bool + }{ + {1023, false}, + {1024, true}, + {1025, true}, + {5000, true}, + } { + f := newFakeServer(t, "multipart") + e := shrunkEngine(t, f) + in := data(tc.size) + _, err := e.upload(t.Context(), "/Volumes/c/s/v/edge.bin", struct{ io.Reader }{bytes.NewReader(in)}, + WithParallelism(4), WithPartSize(700)) + noErr(t, err) + eqBytes(t, f.assembled(), in) + f.mu.Lock() + singleShots, initiates := f.singleTry, len(f.overwriteQ)-f.singleTry + f.mu.Unlock() + if gotMultipart := initiates == 1 && singleShots == 0; gotMultipart != tc.multipart { + t.Errorf("size %d: %d initiates and %d single shots, want multipart=%v", tc.size, initiates, singleShots, tc.multipart) + } + } +} + +// failingAfterReader returns n bytes and then err. +type failingAfterReader struct { + n int + err error +} + +func (r *failingAfterReader) Read(p []byte) (int, error) { + if r.n == 0 { + return 0, r.err + } + k := min(len(p), r.n) + r.n -= k + return k, nil +} + +func TestParallelStreamReadErrorDuringReadAhead(t *testing.T) { + // A read error before the threshold is reached fails the upload before any + // request, so there is no session to abort. + f := newFakeServer(t, "multipart") + e := shrunkEngine(t, f) + readErr := errors.New("disk on fire") + _, err := e.upload(t.Context(), "/Volumes/c/s/v/err.bin", &failingAfterReader{n: 500, err: readErr}, WithParallelism(4)) + if !errors.Is(err, readErr) { + t.Fatalf("err = %v, want the read error", err) + } + f.mu.Lock() + requests := len(f.overwriteQ) + len(f.aborts) + f.mu.Unlock() + if requests != 0 { + t.Errorf("%d initiate, single-shot or abort requests, want none", requests) + } +} diff --git a/files/v2/internal/cloudstorage/cloudstorage.go b/files/v2/internal/cloudstorage/cloudstorage.go index 429c0882..45a51bff 100644 --- a/files/v2/internal/cloudstorage/cloudstorage.go +++ b/files/v2/internal/cloudstorage/cloudstorage.go @@ -44,7 +44,7 @@ type Response struct { // only an *http.Client: no Databricks credentials, no config. type Client struct { http *http.Client - retrier func() ops.Retrier + backoff ops.BackoffPolicy idleTimeout time.Duration logger *slog.Logger } @@ -66,7 +66,7 @@ func WithLogger(logger *slog.Logger) Option { // policy uses the [ops.BackoffPolicy] defaults, as when it is not set. func WithBackoff(bp ops.BackoffPolicy) Option { return func(c *Client) { - c.retrier = func() ops.Retrier { return &retrier{maxRetries: maxRetries, backoff: bp} } + c.backoff = bp } } @@ -78,7 +78,6 @@ func WithBackoff(bp ops.BackoffPolicy) Option { func New(httpClient *http.Client, opts ...Option) *Client { c := &Client{ http: httpClient, - retrier: newRetrier, idleTimeout: defaultIdleTimeout, logger: slog.New(slog.DiscardHandler), } @@ -166,12 +165,13 @@ func (c *Client) Attempt(ctx context.Context, method, url string, headers map[st return &Response{StatusCode: resp.StatusCode, Header: resp.Header, Body: respBody}, nil } -// Send performs Attempt with capped retries: transient network errors, the -// retryable statuses in retryStatusCodes (honoring Retry-After), and stalled -// attempts are retried with exponential backoff. A fresh request and body reader +// Send performs Attempt and retries it up to retries times: transient network +// errors, the retryable statuses in retryStatusCodes (honoring Retry-After), and +// stalled attempts are retried with exponential backoff. The caller sets the +// count, as it depends on where the transfer stands. A fresh request and body reader // are built per attempt, so no explicit rewind is needed. Exhausting retries on // a retryable status returns that status as an error. -func (c *Client) Send(ctx context.Context, method, url string, headers map[string]string, body Body) (*Response, error) { +func (c *Client) Send(ctx context.Context, method, url string, headers map[string]string, body Body, retries int) (*Response, error) { var result *Response op := func(ctx context.Context) error { resp, err := c.Attempt(ctx, method, url, headers, body) @@ -191,7 +191,8 @@ func (c *Client) Send(ctx context.Context, method, url string, headers map[strin result = resp return nil } - if err := ops.Execute(ctx, op, ops.WithRetrier(c.retrier)); err != nil { + newRetrier := func() ops.Retrier { return &retrier{maxRetries: retries, backoff: c.backoff} } + if err := ops.Execute(ctx, op, ops.WithRetrier(newRetrier)); err != nil { return nil, err } return result, nil diff --git a/files/v2/internal/cloudstorage/cloudstorage_test.go b/files/v2/internal/cloudstorage/cloudstorage_test.go index d92ae2c2..146ae87e 100644 --- a/files/v2/internal/cloudstorage/cloudstorage_test.go +++ b/files/v2/internal/cloudstorage/cloudstorage_test.go @@ -18,11 +18,7 @@ import ( // tests do not sleep. func fastClient(srv *httptest.Server) *Client { fast := ops.BackoffPolicy{Initial: time.Millisecond, Maximum: time.Millisecond, Factor: 1} - c := New(srv.Client()) - c.retrier = func() ops.Retrier { - return &retrier{backoff: fast, maxRetries: 3} - } - return c + return New(srv.Client(), WithBackoff(fast)) } func TestSendRetriesRetryableStatus(t *testing.T) { @@ -36,7 +32,7 @@ func TestSendRetriesRetryableStatus(t *testing.T) { })) defer srv.Close() - resp, err := fastClient(srv).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x"))) + resp, err := fastClient(srv).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x")), 3) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -54,13 +50,31 @@ func TestSendExhaustsRetryableStatus(t *testing.T) { })) defer srv.Close() - _, err := fastClient(srv).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x"))) + _, err := fastClient(srv).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x")), 3) aerr, ok := errors.AsType[*apierr.APIError](err) if !ok || aerr.HTTPStatusCode() != http.StatusServiceUnavailable { t.Fatalf("err = %v, want a 503 APIError after exhausting retries", err) } } +func TestSendRetriesAtMostTheGivenCount(t *testing.T) { + for _, retries := range []int{0, 1, 2} { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + _, err := fastClient(srv).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x")), retries) + srv.Close() + if err == nil { + t.Fatalf("retries=%d: want an error", retries) + } + if got := int(calls.Load()); got != retries+1 { + t.Errorf("retries=%d: calls = %d, want %d", retries, got, retries+1) + } + } +} + func TestSendReturnsNonRetryableStatusAsError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusForbidden) @@ -68,7 +82,7 @@ func TestSendReturnsNonRetryableStatusAsError(t *testing.T) { })) defer srv.Close() - resp, err := New(srv.Client()).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x"))) + resp, err := New(srv.Client()).Send(t.Context(), http.MethodPut, srv.URL, nil, BytesBody([]byte("x")), 3) if resp != nil { t.Errorf("expected nil response on error, got %v", resp) } @@ -82,8 +96,8 @@ func TestSendReturnsNonRetryableStatusAsError(t *testing.T) { } func TestAttemptReturnsRawResponse(t *testing.T) { - // Attempt never maps a status to an error: a 308 (resumable continue) and its - // Range header come back as a Response. + // Attempt never maps a status to an error: a 308 and its Range header come + // back as a Response. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Range", "bytes=0-41") w.WriteHeader(http.StatusPermanentRedirect) diff --git a/files/v2/internal/cloudstorage/retries.go b/files/v2/internal/cloudstorage/retries.go index 1f6ce8c2..abfd423f 100644 --- a/files/v2/internal/cloudstorage/retries.go +++ b/files/v2/internal/cloudstorage/retries.go @@ -11,9 +11,6 @@ import ( "github.com/databricks/sdk-go/core/ops" ) -// maxRetries caps how many times a single request is retried before giving up. -var maxRetries = 4 - // retryStatusCodes are the HTTP statuses retried for an idempotent request. var retryStatusCodes = []int{ http.StatusRequestTimeout, // 408 @@ -23,17 +20,11 @@ var retryStatusCodes = []int{ http.StatusGatewayTimeout, // 504 } -// IsRetryableStatus reports whether an HTTP status code should be retried. It is -// exported for callers (such as a resumable upload) that run their own -// resume-aware retry loop over Attempt rather than using Send. +// IsRetryableStatus reports whether an HTTP status code should be retried. func IsRetryableStatus(code int) bool { return slices.Contains(retryStatusCodes, code) } -func newRetrier() ops.Retrier { - return &retrier{maxRetries: maxRetries} -} - type retrier struct { backoff ops.BackoffPolicy maxRetries int @@ -56,9 +47,7 @@ func (r *retrier) IsRetriable(err error) (time.Duration, bool) { // IsRetriable reports whether an error from Attempt is worth retrying: a stalled // attempt is, a short read (deterministic) is not, a retryable HTTP status (as // an APIError) is, and any other failure is retried only if it is a transient -// network error (which excludes context cancellation and deadline). It is -// exported for callers (such as a resumable upload) that run their own -// resume-aware retry loop over Attempt. +// network error (which excludes context cancellation and deadline). func IsRetriable(err error) bool { if err == nil { return false diff --git a/genie/CHANGELOG.md b/genie/CHANGELOG.md index c094efab..06989ee3 100644 --- a/genie/CHANGELOG.md +++ b/genie/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.10.0 (2026-11-17) + ## Release v0.9.0 (2026-11-16) ## Release v0.8.0 (2026-11-12) diff --git a/genie/internal/version.go b/genie/internal/version.go index ce00d1d4..b1804230 100644 --- a/genie/internal/version.go +++ b/genie/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-genie" -const Version = "0.9.0" +const Version = "0.10.0" diff --git a/genie/v1/model.go b/genie/v1/model.go index 49dd3e74..c50c8fb7 100644 --- a/genie/v1/model.go +++ b/genie/v1/model.go @@ -709,6 +709,9 @@ type ColumnInfo struct { Name *string // Full data type specification as SQL/catalogString text. TypeText *string + // The name of the column's data type, for example STRING, DECIMAL or STRUCT. + // The complete type, including parameters such as the precision of a decimal or + // the element type of an array, is in type_text. TypeName ColumnTypeName // Ordinal position of column (starting at position 0). Position *int @@ -726,7 +729,11 @@ type ColumnInfo struct { Nullable *bool // Partition index for column. PartitionIndex *int - Mask *ColumnMask + // The column mask function applied to this column, if any. A column mask is a + // SQL user-defined function that transforms the values a query returns for the + // column. See https://docs.databricks.com/aws/en/tables/row-and-column-filters + // for details. + Mask *ColumnMask } type ColumnMask struct { diff --git a/gitcredentials/CHANGELOG.md b/gitcredentials/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/gitcredentials/CHANGELOG.md +++ b/gitcredentials/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/gitcredentials/internal/version.go b/gitcredentials/internal/version.go index 7643befe..a7fe6b87 100644 --- a/gitcredentials/internal/version.go +++ b/gitcredentials/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-gitcredentials" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/gitcredentials/v1/client.go b/gitcredentials/v1/client.go index cf32e015..13347ad5 100644 --- a/gitcredentials/v1/client.go +++ b/gitcredentials/v1/client.go @@ -339,7 +339,8 @@ func (c *internalClient) ListCredentials(ctx context.Context, req ListCredential return resp, nil } -// Updates the specified Git credential. +// Updates the specified Git credential. Fields that are omitted keep their +// current values. func (c *internalClient) UpdateCredentials(ctx context.Context, req UpdateCredentialsRequest, opts ...call.Option) (*UpdateCredentialsResponse, error) { wireReq, err := updateCredentialsRequestToWire(&req) if err != nil { diff --git a/gitcredentials/v1/model.go b/gitcredentials/v1/model.go index fe977faa..e6bc32c6 100644 --- a/gitcredentials/v1/model.go +++ b/gitcredentials/v1/model.go @@ -21,12 +21,15 @@ type CreateCredentialsRequest struct { // // [Learn more]: https://docs.databricks.com/repos/get-access-tokens-from-git-provider.html PersonalAccessToken *string - // The ID of the service principal whose credentials will be modified. Only - // service principal managers can perform this action. + // The ID of the service principal for which the credential will be created. + // Only service principal managers can perform this action. PrincipalId *int64 - // the name of the git credential, used for identification and ease of lookup + // The name of the Git credential, used for identification and ease of lookup. + // It must not exceed 100 characters. Name *string - // if the credential is the default for the given provider + // Set to true to make the new credential the default for its Git provider, + // replacing the current default. The first credential created for a provider + // becomes the default regardless of this value. IsDefaultForProvider *bool // The authenticating email associated with your Git provider user account. Used // for authentication with the remote repository and also sets the author & @@ -47,9 +50,11 @@ type CreateCredentialsResponse struct { // depending on your Git provider / token requirements. Required for AWS // CodeCommit. GitUsername *string - // the name of the git credential, used for identification and ease of lookup + // The name of the Git credential, used for identification and ease of lookup. Name *string - // if the credential is the default for the given provider + // Whether this credential is the default for its Git provider. The default + // credential is the fallback that is used when no specific credential is + // selected. IsDefaultForProvider *bool // The authenticating email associated with your Git provider user account. Used // for authentication with the remote repository and also sets the author & @@ -59,6 +64,9 @@ type CreateCredentialsResponse struct { GitEmail *string } +// A Git credential that Databricks uses to authenticate to a Git provider on +// behalf of a user or service principal. Each user has one default credential +// per Git provider, which is used when no specific credential is selected.. type Credential struct { // ID of the credential object in the workspace. CredentialId *int64 @@ -74,9 +82,12 @@ type Credential struct { // depending on your Git provider / token requirements. Required for AWS // CodeCommit. GitUsername *string - // the name of the git credential, used for identification and ease of lookup + // The name of the Git credential, used for identification and ease of lookup. Name *string - // if the credential is the default for the given provider + // Whether this credential is the default for its Git provider. The default + // credential is the fallback that is used when no specific credential is + // selected. See + // https://docs.databricks.com/aws/en/repos/repos-setup#how-default-credentials-work. IsDefaultForProvider *bool // The authenticating email associated with your Git provider user account. Used // for authentication with the remote repository and also sets the author & @@ -89,7 +100,7 @@ type Credential struct { type DeleteCredentialsRequest struct { // The ID for the corresponding credential to access. Id *int64 - // The ID of the service principal whose credentials will be modified. Only + // The ID of the service principal whose credential will be deleted. Only // service principal managers can perform this action. PrincipalId *int64 } @@ -100,7 +111,7 @@ type DeleteCredentialsResponse struct { type GetCredentialsRequest struct { // The ID for the corresponding credential to access. Id *int64 - // The ID of the service principal whose credentials will be modified. Only + // The ID of the service principal whose credential will be returned. Only // service principal managers can perform this action. PrincipalId *int64 } @@ -116,9 +127,11 @@ type GetCredentialsResponse struct { // depending on your Git provider / token requirements. Required for AWS // CodeCommit. GitUsername *string - // the name of the git credential, used for identification and ease of lookup + // The name of the Git credential, used for identification and ease of lookup. Name *string - // if the credential is the default for the given provider + // Whether this credential is the default for its Git provider. The default + // credential is the fallback that is used when no specific credential is + // selected. IsDefaultForProvider *bool // The authenticating email associated with your Git provider user account. Used // for authentication with the remote repository and also sets the author & @@ -152,7 +165,8 @@ type UpdateCredentialsRequest struct { // `gitHub`, `bitbucketCloud`, `gitLab`, `azureDevOpsServices` (Azure DevOps // Services, including Microsoft Entra ID authentication), `gitHubEnterprise`, // `bitbucketServer` (Bitbucket Data Center), `gitLabEnterpriseEdition` (GitLab - // Self-Managed), and `awsCodeCommit`. + // Self-Managed), and `awsCodeCommit`. Must match the Git provider of the + // existing credential; the provider cannot be changed. GitProvider *string // The username provided with your Git provider account and associated with the // credential. For most Git providers it is only used to set the Git committer & @@ -163,9 +177,12 @@ type UpdateCredentialsRequest struct { // The ID of the service principal whose credentials will be modified. Only // service principal managers can perform this action. PrincipalId *int64 - // the name of the git credential, used for identification and ease of lookup + // The name of the Git credential, used for identification and ease of lookup. + // It must not exceed 100 characters. Name *string - // if the credential is the default for the given provider + // Set to true to make this credential the default for its Git provider, + // replacing the current default. Cannot be set to false; make another + // credential the default instead. IsDefaultForProvider *bool // The authenticating email associated with your Git provider user account. Used // for authentication with the remote repository and also sets the author & diff --git a/instancepools/CHANGELOG.md b/instancepools/CHANGELOG.md index da57c0ac..e274db5d 100644 --- a/instancepools/CHANGELOG.md +++ b/instancepools/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-12) ### New Features and Improvements diff --git a/instancepools/internal/version.go b/instancepools/internal/version.go index ed11d353..880c4c11 100644 --- a/instancepools/internal/version.go +++ b/instancepools/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-instancepools" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/instancepools/v2/model.go b/instancepools/v2/model.go index 79f5a512..c13d538b 100644 --- a/instancepools/v2/model.go +++ b/instancepools/v2/model.go @@ -231,9 +231,9 @@ type DiskType_RemoteVolumeType_AzureDiskVolumeType struct { func (*DiskType_RemoteVolumeType_AzureDiskVolumeType) isDiskType_RemoteVolumeType() {} type DockerBasicAuth struct { - // Name of the user + // User name for authenticating to the Docker registry that hosts the image. Username *string - // Password of the user + // Password for authenticating to the Docker registry that hosts the image. Password *string } diff --git a/jobs/CHANGELOG.md b/jobs/CHANGELOG.md index 5a6119fb..804e2582 100644 --- a/jobs/CHANGELOG.md +++ b/jobs/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.15.0 (2026-11-17) + ## Release v0.14.0 (2026-11-16) ## Release v0.13.0 (2026-11-12) diff --git a/jobs/internal/version.go b/jobs/internal/version.go index b53a8d9f..3b895382 100644 --- a/jobs/internal/version.go +++ b/jobs/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-jobs" -const Version = "0.14.0" +const Version = "0.15.0" diff --git a/jobs/v2/model.go b/jobs/v2/model.go index 90a0bbd0..526d8ab0 100644 --- a/jobs/v2/model.go +++ b/jobs/v2/model.go @@ -1301,7 +1301,12 @@ type AwsAttributes struct { // does not affect cluster size and cannot currently be mutated over the // lifetime of a cluster. FirstOnDemand *int - Availability AwsAvailability + // Availability type used for all subsequent nodes past the `first_on_demand` + // ones. + // + // Note: If `first_on_demand` is zero, this availability type will be used for + // the entire cluster. + Availability AwsAvailability // Identifier for the availability zone/datacenter in which the cluster resides. // This string will be of a form like "us-west-2a". The provided availability // zone must be in the same region as the deployment. For example, @@ -1364,7 +1369,9 @@ type AwsAttributes struct { // Attributes set during cluster creation which are related to Microsoft Azure.. type AzureAttributes struct { - // Defines values necessary to configure and run Azure Log Analytics agent + // Defines values necessary to configure and run Azure Log Analytics agent. Both + // `log_analytics_workspace_id` and `log_analytics_primary_key` must be set, or + // neither. LogAnalyticsInfo *LogAnalyticsInfo // The first `first_on_demand` nodes of the cluster will be placed on on-demand // instances. This value should be greater than 0, to make sure the cluster @@ -1843,7 +1850,10 @@ type ClusterSpec_NewCluster struct { // `cluster_log_conf` is specified, init script logs are sent to // `//init_scripts`. InitScripts []InitScriptInfo - // Custom docker image BYOC + // Custom Docker image for the cluster's nodes, used with [Databricks Container + // Services]. + // + // [Databricks Container Services]: https://docs.databricks.com/compute/custom-containers.html DockerImage *DockerImage // The optional ID of the instance pool to which the cluster belongs. InstancePoolId *string @@ -1857,8 +1867,39 @@ type ClusterSpec_NewCluster struct { // The pool cluster uses the instance pool with id (instance_pool_id) if the // driver pool is not assigned. DriverInstancePoolId *string - WorkloadType *WorkloadType - DataSecurityMode DataSecurityMode + // Types of workloads that can use the cluster, such as notebooks or jobs. If + // omitted, the cluster can be used for both notebooks and jobs. + WorkloadType *WorkloadType + // Data security mode decides what data governance model to use when accessing + // data from a cluster. + // + // * `DATA_SECURITY_MODE_AUTO`: will choose the most appropriate + // access mode depending on your compute configuration. * + // `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by + // multiple users. Cluster users are fully isolated so that they cannot see each + // other's data and credentials. Most data governance features are supported in + // this mode. But programming languages and cluster features might be limited. * + // `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively + // used by a single user specified in `single_user_name`. Most programming + // languages, cluster features and data governance features are available in + // this mode. * `NONE`: No security isolation for multiple users sharing the + // cluster. Data governance features are not available in this mode. + // + // The following modes are legacy aliases for the above modes: + // + // * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * + // `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. + // + // The following modes are deprecated starting with Databricks Runtime 15.0 and + // will be removed for future Databricks Runtime versions: + // + // * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL + // clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from + // legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This + // mode is for users migrating from legacy Passthrough on standard clusters. * + // `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn't have UC + // or passthrough enabled. + DataSecurityMode DataSecurityMode // Determines the cluster's runtime engine, either standard or Photon. // // This field is not compatible with legacy `spark_version` values that contain @@ -1868,7 +1909,22 @@ type ClusterSpec_NewCluster struct { // If left unspecified, the runtime engine defaults to standard unless the // spark_version contains -photon-, in which case Photon will be used. RuntimeEngine RuntimeEngine - Kind ComputeKind + // The kind of compute described by this compute specification. + // + // Depending on `kind`, different validations and default values will be + // applied. + // + // Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas + // clusters with no specified `kind` do not. * [is_single_node] * + // [use_ml_runtime] + // + // By using the [simple form], your clusters are automatically using `kind = + // CLASSIC_PREVIEW`. + // + // [is_single_node]: https://docs.databricks.com/api/workspace/clusters/create#is_single_node + // [simple form]: https://docs.databricks.com/compute/simple-form.html + // [use_ml_runtime]: https://docs.databricks.com/api/workspace/clusters/create#use_ml_runtime + Kind ComputeKind // This field can only be used when `kind = CLASSIC_PREVIEW`. // // `effective_spark_version` is determined by `spark_version` (DBR release), @@ -2439,9 +2495,9 @@ type DeploymentSpec struct { } type DockerBasicAuth struct { - // Name of the user + // User name for authenticating to the Docker registry that hosts the image. Username *string - // Password of the user + // Password for authenticating to the Docker registry that hosts the image. Password *string } @@ -3651,6 +3707,8 @@ type JobsHealthRules struct { Rules []JobsHealthRule } +// A library to install on a cluster. Exactly one of the library-type fields +// must be set.. type Library struct { Lib isLibrary_Lib } @@ -3674,8 +3732,9 @@ type Library_Lib_Jar struct { func (*Library_Lib_Jar) isLibrary_Lib() {} // Library_Lib_Egg selects Egg for Library.Lib. -// Deprecated. URI of the egg library to install. Installing Python egg files is -// deprecated and is not supported in Databricks Runtime 14.0 and above. +// Deprecated: Installing Python egg files is not supported in Databricks +// Runtime 14.0 and above. Use a wheel (`whl`) or a PyPI package (`pypi`) +// instead. The value is the URI of the egg file to install. type Library_Lib_Egg struct { Egg string } @@ -3683,7 +3742,7 @@ type Library_Lib_Egg struct { func (*Library_Lib_Egg) isLibrary_Lib() {} // Library_Lib_Pypi selects Pypi for Library.Lib. -// Specification of a PyPi library to be installed. For example: `{ "package": +// Specification of a PyPI library to be installed. For example: `{ "package": // "simplejson" }` type Library_Lib_Pypi struct { Pypi PythonPyPiLibrary @@ -3701,7 +3760,7 @@ type Library_Lib_Maven struct { func (*Library_Lib_Maven) isLibrary_Lib() {} // Library_Lib_Cran selects Cran for Library.Lib. -// Specification of a CRAN library to be installed as part of the library +// Specification of a CRAN library to be installed. type Library_Lib_Cran struct { Cran RCranLibrary } @@ -3726,7 +3785,8 @@ func (*Library_Lib_Whl) isLibrary_Lib() {} // URI of the requirements.txt file to install. Only Workspace paths and Unity // Catalog Volumes paths are supported. For example: `{ "requirements": // "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : -// "/Volumes/path/to/requirements.txt" }` +// "/Volumes/path/to/requirements.txt" }`. Installing a requirements file +// requires Databricks Runtime 15.0 or above. type Library_Lib_Requirements struct { Requirements string } @@ -3883,8 +3943,12 @@ type LocalFileInfo struct { } type LogAnalyticsInfo struct { + // ID of the Azure Log Analytics workspace that the Log Analytics agent on the + // cluster's nodes reports to. LogAnalyticsWorkspaceId *string - LogAnalyticsPrimaryKey *string + // Primary key of the Azure Log Analytics workspace. The agent uses it to + // authenticate when reporting to the workspace. + LogAnalyticsPrimaryKey *string } // A recurring weekly time window during which platform-initiated maintenance is @@ -3910,7 +3974,7 @@ type MavenLibrary struct { // Maven repo to install the Maven package from. If omitted, both Maven Central // Repository and Spark Packages are searched. Repo *string - // List of dependences to exclude. For example: `["slf4j:slf4j", + // List of dependencies to exclude. For example: `["slf4j:slf4j", // "*:hadoop-client"]`. // // Maven dependency exclusions: @@ -4195,7 +4259,7 @@ type PythonOperatorTask_Parameter struct { } type PythonPyPiLibrary struct { - // The name of the pypi package to install. An optional exact version + // The name of the PyPI package to install. An optional exact version // specification is also supported. Examples: "simplejson" and // "simplejson==3.8.0". Package *string @@ -5881,7 +5945,8 @@ type SqlAlertState struct { } type SqlConditionConfiguration struct { - // The ID of the SQL query to evaluate as the trigger condition. + // The ID of the SQL query to evaluate as the trigger condition. Exactly one of + // `sql_query_id` or `file` must be set. SqlQueryId *string // The canonical identifier of the SQL warehouse to run the condition query // against. diff --git a/lakeview/CHANGELOG.md b/lakeview/CHANGELOG.md index 0dcca333..e48cab6f 100644 --- a/lakeview/CHANGELOG.md +++ b/lakeview/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.8.0 (2026-11-17) + ## Release v0.7.0 (2026-11-16) ## Release v0.6.0 (2026-11-12) diff --git a/lakeview/internal/version.go b/lakeview/internal/version.go index 81cdfe93..09a324e2 100644 --- a/lakeview/internal/version.go +++ b/lakeview/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-lakeview" -const Version = "0.7.0" +const Version = "0.8.0" diff --git a/lakeview/v1/client.go b/lakeview/v1/client.go index 221f0288..30512fbb 100644 --- a/lakeview/v1/client.go +++ b/lakeview/v1/client.go @@ -150,7 +150,8 @@ func (c *internalClient) CreateDashboard(ctx context.Context, req CreateDashboar return resp, nil } -// Create dashboard schedule. +// Creates a schedule for a dashboard. The dashboard must be published before a +// schedule can be created for it. // // Requires the [Databricks SQL access] entitlement. Grant Databricks SQL access // in addition to Workspace access. @@ -229,7 +230,13 @@ func (c *internalClient) CreateSchedule(ctx context.Context, req CreateScheduleR return resp, nil } -// Create schedule subscription. +// Create schedule subscription. Subscribes a workspace user or a notification +// destination to the schedule, so that it receives a snapshot of the dashboard +// each time the schedule runs. Subscribing yourself requires CAN VIEW +// permission on the dashboard; subscribing another user or a notification +// destination requires CAN EDIT. On dashboards published without embedded +// credentials, you can only subscribe yourself. If the subscriber is already +// subscribed to the schedule, the existing subscription is returned. // // The caller must be a workspace user with one of the following [entitlements]: // Workspace access, Databricks SQL access, or Consumer access. @@ -317,7 +324,7 @@ func (c *internalClient) CreateSubscription(ctx context.Context, req CreateSubsc return resp, nil } -// Delete dashboard schedule. +// Deletes a schedule together with all of its subscriptions. // // Requires the [Databricks SQL access] entitlement. Grant Databricks SQL access // in addition to Workspace access. @@ -387,7 +394,10 @@ func (c *internalClient) DeleteSchedule(ctx context.Context, req DeleteScheduleR return nil } -// Delete schedule subscription. +// Delete schedule subscription. The subscriber receives no further snapshots +// from the schedule. Deleting your own subscription requires CAN VIEW +// permission on the dashboard; deleting another user's or a notification +// destination's subscription requires CAN EDIT. // // The caller must be a workspace user with one of the following [entitlements]: // Workspace access, Databricks SQL access, or Consumer access. @@ -527,7 +537,8 @@ func (c *internalClient) GetDashboard(ctx context.Context, req GetDashboardReque return resp, nil } -// Get the current published dashboard. +// Get the current published dashboard. The request fails with NOT_FOUND if the +// dashboard has never been published or has been unpublished. // // The caller must be a workspace user with one of the following entitlements: // Workspace access, Databricks SQL access, or Consumer access. @@ -596,14 +607,20 @@ func (c *internalClient) GetPublishedDashboard(ctx context.Context, req GetPubli return resp, nil } -// Get a required authorization details and scopes of a published dashboard to -// mint an OAuth token. +// Get the OAuth scope and authorization details needed to mint a token that is +// scoped to viewing the published dashboard, for example to embed it in an +// external application. Pass the response fields, with `authorization_details` +// serialized as a JSON string, to the workspace OAuth token endpoint with the +// `client_credentials` grant to mint the token. See [Embedding for external +// users]. // // The caller must be a workspace user with one of the following entitlements: // Workspace access, Databricks SQL access, or Consumer access. // // Account-level users who are not members of the workspace cannot call this // endpoint, even if the dashboard has been shared with them. +// +// [Embedding for external users]: https://docs.databricks.com/dashboards/share/embedding/external-embed func (c *internalClient) GetPublishedDashboardTokenInfo(ctx context.Context, req GetPublishedDashboardTokenInfoRequest, opts ...call.Option) (*GetPublishedDashboardTokenInfoResponse, error) { wireReq, err := getPublishedDashboardTokenInfoRequestToWire(&req) if err != nil { @@ -1337,6 +1354,10 @@ func (c *internalClient) PublishDashboard(ctx context.Context, req PublishDashbo } // Revert a dashboard's definition in draft mode to the last published version. +// The draft's datasets, pages and widgets are replaced with those of the +// current published version, and the updated draft is returned. The dashboard +// must have been published at least once; the published version itself is not +// changed. func (c *internalClient) RevertDashboard(ctx context.Context, req RevertDashboardRequest, opts ...call.Option) (*RevertDashboardResponse, error) { wireReq, err := revertDashboardRequestToWire(&req) if err != nil { @@ -1465,7 +1486,9 @@ func (c *internalClient) TrashDashboard(ctx context.Context, req TrashDashboardR return resp, nil } -// Unpublish the dashboard. +// Unpublish the dashboard. This deletes the published version, so viewers can +// no longer open it, and pauses the dashboard's schedules. The draft is not +// changed; publish it again to make the dashboard available to viewers. func (c *internalClient) UnpublishDashboard(ctx context.Context, req UnpublishDashboardRequest, opts ...call.Option) (*UnpublishDashboardResponse, error) { headers := http.Header{} @@ -1601,7 +1624,8 @@ func (c *internalClient) UpdateDashboard(ctx context.Context, req UpdateDashboar return resp, nil } -// Update dashboard schedule. +// Updates a schedule. The request body replaces the stored schedule, so fields +// omitted from the body are reset to their defaults. // // Requires the [Databricks SQL access] entitlement. Grant Databricks SQL access // in addition to Workspace access. diff --git a/lakeview/v1/model.go b/lakeview/v1/model.go index a1b769ce..e657aab8 100644 --- a/lakeview/v1/model.go +++ b/lakeview/v1/model.go @@ -33,24 +33,24 @@ const ( ) type AuthorizationDetails struct { - // The type of authorization downscoping policy. Ex: `workspace_rule_set` - // defines access rules for a specific workspace resource + // The type of authorization downscoping policy. A `workspace_rule_set` policy + // defines access rules for a specific workspace resource. Type *string // The resource name to which the authorization rule applies. This field is - // specific to `workspace_rule_set` constraint. Format: - // `workspaces/{workspace_id}/dashboards/{dashboard_id}` + // specific to the `workspace_rule_set` constraint. Format: + // `workspaces/{workspace_id}` for a workspace-level rule, or + // `workspaces/{workspace_id}/dashboards/{dashboard_id}` for a rule on a + // dashboard. ResourceName *string - // The acl path of the tree store resource resource. + // The legacy ACL path of the workspace object that the rule applies to. ResourceLegacyAclPath *string // Represents downscoped permission rules with specific access rights. This - // field is specific to `workspace_rule_set` constraint. + // field is specific to the `workspace_rule_set` constraint. GrantRules []AuthorizationDetails_GrantRule } type AuthorizationDetails_GrantRule struct { - // Permission sets for dashboard are defined in - // iam-common/rbac-common/permission-sets/definitions/TreeStoreBasePermissionSets - // Ex: `permissionSets/dashboard.runner` + // The permission set that the rule grants on the resource. PermissionSet *string } @@ -78,10 +78,10 @@ type CreateSubscriptionRequest struct { } type CronSchedule struct { - // A cron expression using quartz syntax. EX: `0 0 8 * * ?` represents everyday - // at 8am. See [Cron Trigger] for details. + // A cron expression in Quartz syntax that defines when the schedule runs. See + // the [Quartz cron trigger tutorial] for the syntax. // - // [Cron Trigger]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html + // [Quartz cron trigger tutorial]: https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html QuartzCronExpression *string // A Java timezone id. The schedule will be resolved with respect to this // timezone. See [Java TimeZone] for details. @@ -181,12 +181,13 @@ type DeleteScheduleRequest struct { type DeleteSubscriptionRequest struct { // UUID identifying the subscription. SubscriptionId *string - // UUID identifying the schedule which the subscription belongs. + // UUID identifying the schedule to which the subscription belongs. ScheduleId *string - // UUID identifying the dashboard which the subscription belongs. + // UUID identifying the dashboard to which the subscription belongs. DashboardId *string - // The etag for the subscription. Can be optionally provided to ensure that the - // subscription has not been modified since the last read. + // The etag of the subscription, as returned when it was read. If it is provided + // and no longer matches, the request fails with ABORTED and the subscription is + // not deleted. Etag *string } @@ -203,21 +204,28 @@ type GetPublishedDashboardRequest struct { type GetPublishedDashboardTokenInfoRequest struct { // UUID identifying the published dashboard. DashboardId *string - // Provided external value to be included in the custom claim. + // Value to carry in the token for this viewer, such as a tenant or region key. + // Dashboard dataset queries can reference it as `__aibi_external_value` to + // filter data per viewer. The combined size of `external_value` and + // `external_viewer_id` must be less than 1 KB. ExternalValue *string - // Provided external viewer id to be included in the custom claim. + // Identifier of the viewer in the external application. It is recorded in + // dashboard audit logs, so it should be unique per viewer and must not contain + // personally identifiable information. ExternalViewerId *string } type GetPublishedDashboardTokenInfoResponse struct { - // Custom claim generated from external_value and external_viewer_id. Format: + // Custom claim that carries `external_value` and `external_viewer_id` into the + // minted token. It is only present when at least one of those query parameters + // was provided. Format: // `urn:aibi:external_data:::` CustomClaim *string - // Scope defining access permissions. + // The space-separated OAuth scopes to request when minting the token. Scope *string - // Authorization constraints for accessing the published dashboard. Currently - // includes `workspace_rule_set` and could be enriched with - // `unity_catalog_privileges` before oAuth token generation. + // Authorization constraints that restrict the minted token to viewing the + // published dashboard. Pass them to the OAuth token endpoint serialized as a + // JSON array. AuthorizationDetails []AuthorizationDetails } @@ -231,9 +239,9 @@ type GetScheduleRequest struct { type GetSubscriptionRequest struct { // UUID identifying the subscription. SubscriptionId *string - // UUID identifying the schedule which the subscription belongs. + // UUID identifying the schedule to which the subscription belongs. ScheduleId *string - // UUID identifying the dashboard which the subscription belongs. + // UUID identifying the dashboard to which the subscription belongs. DashboardId *string } @@ -258,7 +266,7 @@ type ListDashboardsResponse struct { } type ListSchedulesRequest struct { - // UUID identifying the dashboard to which the schedules belongs. + // UUID identifying the dashboard to which the schedules belong. DashboardId *string // The number of schedules to return per page. PageSize *int @@ -276,9 +284,9 @@ type ListSchedulesResponse struct { } type ListSubscriptionsRequest struct { - // UUID identifying the dashboard which the subscriptions belongs. + // UUID identifying the dashboard to which the subscriptions belong. DashboardId *string - // UUID identifying the schedule which the subscriptions belongs. + // UUID identifying the schedule to which the subscriptions belong. ScheduleId *string // The number of subscriptions to return per page. PageSize *int @@ -323,11 +331,17 @@ type PublishDashboardRequest struct { type PublishedDashboard struct { // The display name of the published dashboard. DisplayName *string - // The warehouse ID used to run the published dashboard. + // The ID of the SQL warehouse that runs the published dashboard's queries. It + // is the warehouse given when the dashboard was published, or the draft's + // warehouse if none was given. WarehouseId *string - // Indicates whether credentials are embedded in the published dashboard. + // Whether the publisher's credentials are embedded in the published dashboard. + // When true, viewers run the dashboard's queries with the publisher's data + // permissions (shared data permissions); when false, each viewer runs them with + // their own data permissions. EmbedCredentials *bool - // The timestamp of when the published dashboard was last revised. + // The timestamp of when the current published version was created, that is, + // when the dashboard was last published. RevisionCreateTime *types.Time } @@ -335,8 +349,9 @@ type PublishedDashboard struct { type RevertDashboardRequest struct { // UUID identifying the dashboard. DashboardId *string - // The etag for the dashboard. Optionally, it can be provided to verify that the - // dashboard has not been modified from its last retrieval. + // The etag of the draft dashboard, as returned by the get dashboard API. When + // provided, the request fails with ABORTED if the draft has been modified since + // that etag was read. Etag *string } @@ -346,17 +361,24 @@ type RevertDashboardResponse struct { Dashboard *Dashboard } +// A schedule periodically runs a dashboard on a cron cadence. Each run executes +// the dashboard's queries to refresh its cached data and sends a snapshot of +// the dashboard to the schedule's subscribers. See [Manage scheduled dashboard +// updates and subscriptions] for details. +// +// [Manage scheduled dashboard updates and subscriptions]: https://docs.databricks.com/dashboards/share/schedule-subscribe. type Schedule struct { // UUID identifying the schedule. ScheduleId *string // UUID identifying the dashboard to which the schedule belongs. DashboardId *string - // The cron expression describing the frequency of the periodic refresh for this - // schedule. + // The cron expression and time zone that define when the schedule runs. CronSchedule *CronSchedule - // The status indicates whether this schedule is paused or not. + // Whether the schedule is paused. A paused schedule keeps its configuration and + // subscribers, but the dashboard is not refreshed and no notifications are sent + // until the schedule is unpaused. PauseStatus SchedulePauseStatus - // The display name for schedule. + // The name of the schedule, as shown in the dashboard's list of schedules. DisplayName *string // The etag for the schedule. Must be left empty on create, must be provided on // updates to ensure that the schedule has not been modified since the last @@ -366,30 +388,48 @@ type Schedule struct { CreateTime *types.Time // A timestamp indicating when the schedule was last updated. UpdateTime *types.Time - // The warehouse id to run the dashboard with for the schedule. + // The ID of the SQL warehouse that runs the dashboard's queries during + // scheduled refreshes. If omitted, the warehouse of the published dashboard is + // used. WarehouseId *string } +// A schedule periodically runs a dashboard on a cron cadence. Each run executes +// the dashboard's queries to refresh its cached data and sends a snapshot of +// the dashboard to the schedule's subscribers. See [Manage scheduled dashboard +// updates and subscriptions] for details. +// +// [Manage scheduled dashboard updates and subscriptions]: https://docs.databricks.com/dashboards/share/schedule-subscribe. type ScheduleCreate struct { // UUID identifying the schedule. ScheduleId *string // UUID identifying the dashboard to which the schedule belongs. DashboardId *string - // The cron expression describing the frequency of the periodic refresh for this - // schedule. + // The cron expression and time zone that define when the schedule runs. CronSchedule *CronSchedule - // The status indicates whether this schedule is paused or not. + // Whether the schedule is paused. A paused schedule keeps its configuration and + // subscribers, but the dashboard is not refreshed and no notifications are sent + // until the schedule is unpaused. PauseStatus SchedulePauseStatus - // The display name for schedule. + // The name of the schedule, as shown in the dashboard's list of schedules. DisplayName *string // A timestamp indicating when the schedule was created. CreateTime *types.Time // A timestamp indicating when the schedule was last updated. UpdateTime *types.Time - // The warehouse id to run the dashboard with for the schedule. + // The ID of the SQL warehouse that runs the dashboard's queries during + // scheduled refreshes. If omitted, the warehouse of the published dashboard is + // used. WarehouseId *string } +// A subscription registers a workspace user or a notification destination as a +// subscriber of a dashboard schedule. Each time the schedule runs, the +// subscriber receives a snapshot of the published dashboard, unless the +// subscription is refresh-only. See [Manage scheduled dashboard updates and +// subscriptions]. +// +// [Manage scheduled dashboard updates and subscriptions]: https://docs.databricks.com/dashboards/share/schedule-subscribe. type Subscription struct { // UUID identifying the subscription. SubscriptionId *string @@ -397,45 +437,59 @@ type Subscription struct { ScheduleId *string // UUID identifying the dashboard to which the subscription belongs. DashboardId *string - // Subscriber details for users and destinations to be added as subscribers to - // the schedule. + // The subscriber that receives the dashboard snapshot. Exactly one of + // `user_subscriber` and `destination_subscriber` must be set. Subscriber *Subscription_Subscriber // UserId of the user who adds subscribers (users or notification destinations) // to the dashboard's schedule. CreatedByUserId *int64 - // The etag for the subscription. Must be left empty on create, can be - // optionally provided on delete to ensure that the subscription has not been - // deleted since the last read. + // The etag of the subscription. It must be left empty on create. It can be + // passed as the `etag` parameter of the delete method to make sure the + // subscription has not changed since it was read. Etag *string // A timestamp indicating when the subscription was created. CreateTime *types.Time // A timestamp indicating when the subscription was last updated. UpdateTime *types.Time - // Controls whether notifications are sent to the subscriber for scheduled - // dashboard refreshes. If not defined, defaults to false in the backend to - // match the current behavior (refresh and notify) + // Whether the subscription is refresh-only. When true, each scheduled run + // refreshes the dashboard data for the subscriber without sending a + // notification. Defaults to false. Refresh-only subscriptions are only allowed + // on dashboards published without embedded credentials. SkipNotify *bool } type Subscription_Subscriber struct { - // The user to receive the subscription email. This parameter is mutually - // exclusive with `destination_subscriber`. + // The workspace user who receives the subscription. UserSubscriber *Subscription_Subscriber_User - // The destination to receive the subscription email. This parameter is mutually - // exclusive with `user_subscriber`. + // The notification destination that receives the subscription, such as an email + // list or a Slack or Microsoft Teams channel. Workspace admins configure + // notification destinations. DestinationSubscriber *Subscription_Subscriber_Destination } type Subscription_Subscriber_Destination struct { - // The canonical identifier of the destination to receive email notification. + // The UUID of the notification destination that receives the subscription, as + // returned by the [Notification Destinations API]. + // + // [Notification Destinations API]: https://docs.databricks.com/api/workspace/notificationdestinations DestinationId *string } type Subscription_Subscriber_User struct { - // UserId of the subscriber. + // The ID of the workspace user who receives the subscription, as returned by + // the [Workspace Users API]. + // + // [Workspace Users API]: https://docs.databricks.com/api/workspace/users UserId *int64 } +// A subscription registers a workspace user or a notification destination as a +// subscriber of a dashboard schedule. Each time the schedule runs, the +// subscriber receives a snapshot of the published dashboard, unless the +// subscription is refresh-only. See [Manage scheduled dashboard updates and +// subscriptions]. +// +// [Manage scheduled dashboard updates and subscriptions]: https://docs.databricks.com/dashboards/share/schedule-subscribe. type SubscriptionCreate struct { // UUID identifying the subscription. SubscriptionId *string @@ -443,8 +497,8 @@ type SubscriptionCreate struct { ScheduleId *string // UUID identifying the dashboard to which the subscription belongs. DashboardId *string - // Subscriber details for users and destinations to be added as subscribers to - // the schedule. + // The subscriber that receives the dashboard snapshot. Exactly one of + // `user_subscriber` and `destination_subscriber` must be set. Subscriber *Subscription_Subscriber // UserId of the user who adds subscribers (users or notification destinations) // to the dashboard's schedule. @@ -453,9 +507,10 @@ type SubscriptionCreate struct { CreateTime *types.Time // A timestamp indicating when the subscription was last updated. UpdateTime *types.Time - // Controls whether notifications are sent to the subscriber for scheduled - // dashboard refreshes. If not defined, defaults to false in the backend to - // match the current behavior (refresh and notify) + // Whether the subscription is refresh-only. When true, each scheduled run + // refreshes the dashboard data for the subscriber without sending a + // notification. Defaults to false. Refresh-only subscriptions are only allowed + // on dashboards published without embedded credentials. SkipNotify *bool } diff --git a/lakewatch/CHANGELOG.md b/lakewatch/CHANGELOG.md index 7c412190..39f0daad 100644 --- a/lakewatch/CHANGELOG.md +++ b/lakewatch/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.6.0 (2026-11-17) + ## Release v0.5.0 (2026-11-12) ### New Features and Improvements diff --git a/lakewatch/internal/version.go b/lakewatch/internal/version.go index ea199b60..406df22f 100644 --- a/lakewatch/internal/version.go +++ b/lakewatch/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-lakewatch" -const Version = "0.5.0" +const Version = "0.6.0" diff --git a/lakewatch/v1/client.go b/lakewatch/v1/client.go index 531f5e1a..d42e45d3 100644 --- a/lakewatch/v1/client.go +++ b/lakewatch/v1/client.go @@ -78,8 +78,9 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Returns the alert delivery configuration for a catalog. -func (c *internalClient) GetAlertConfig(ctx context.Context, req GetAlertConfigRequest, opts ...call.Option) (*AlertConfig, error) { +// Returns a built-in parser's identity and latest-version pointer. Does not +// return the spec. +func (c *internalClient) GetBuiltinParser(ctx context.Context, req GetBuiltinParserRequest, opts ...call.Option) (*BuiltinParser, error) { headers := http.Header{} if c.workspaceID != "" { @@ -101,7 +102,7 @@ func (c *internalClient) GetAlertConfig(ctx context.Context, req GetAlertConfigR baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *AlertConfig + var resp *BuiltinParser call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -123,11 +124,11 @@ func (c *internalClient) GetAlertConfig(ctx context.Context, req GetAlertConfigR if err != nil { return err } - var wireResp alertConfigWire + var wireResp builtinParserWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = alertConfigFromWire(&wireResp) + resp, err = builtinParserFromWire(&wireResp) if err != nil { return err } @@ -140,8 +141,8 @@ func (c *internalClient) GetAlertConfig(ctx context.Context, req GetAlertConfigR return resp, nil } -// Returns the common Lakewatch configuration for a catalog. -func (c *internalClient) GetCommonConfig(ctx context.Context, req GetCommonConfigRequest, opts ...call.Option) (*CommonConfig, error) { +// Returns one version of a built-in parser, including its spec. +func (c *internalClient) GetBuiltinParserVersion(ctx context.Context, req GetBuiltinParserVersionRequest, opts ...call.Option) (*BuiltinParserVersion, error) { headers := http.Header{} if c.workspaceID != "" { @@ -163,7 +164,7 @@ func (c *internalClient) GetCommonConfig(ctx context.Context, req GetCommonConfi baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *CommonConfig + var resp *BuiltinParserVersion call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -185,11 +186,11 @@ func (c *internalClient) GetCommonConfig(ctx context.Context, req GetCommonConfi if err != nil { return err } - var wireResp commonConfigWire + var wireResp builtinParserVersionWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = commonConfigFromWire(&wireResp) + resp, err = builtinParserVersionFromWire(&wireResp) if err != nil { return err } @@ -202,8 +203,13 @@ func (c *internalClient) GetCommonConfig(ctx context.Context, req GetCommonConfi return resp, nil } -// Returns the datasource ingestion configuration for a catalog. -func (c *internalClient) GetDatasourceConfig(ctx context.Context, req GetDatasourceConfigRequest, opts ...call.Option) (*DatasourceConfig, error) { +// Lists a built-in parser's published versions, oldest first (ascending +// 1..resource_revision), without their specs. +func (c *internalClient) ListBuiltinParserVersions(ctx context.Context, req ListBuiltinParserVersionsRequest, opts ...call.Option) (*ListBuiltinParserVersionsResponse, error) { + wireReq, err := listBuiltinParserVersionsRequestToWire(&req) + if err != nil { + return nil, err + } headers := http.Header{} if c.workspaceID != "" { @@ -216,16 +222,23 @@ func (c *internalClient) GetDatasourceConfig(ctx context.Context, req GetDatasou } pb := pathBuilder{} pb.literal("/api/2.0/lakewatch/") - if req.Name == nil { - return nil, fmt.Errorf("path parameter %q is required", "name") + if req.Parent == nil { + return nil, fmt.Errorf("path parameter %q is required", "parent") } - pb.singleSegment(*req.Name) + pb.singleSegment(*req.Parent) + pb.literal("/versions") baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} + if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + return nil, err + } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *DatasourceConfig + var resp *ListBuiltinParserVersionsResponse call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -247,11 +260,11 @@ func (c *internalClient) GetDatasourceConfig(ctx context.Context, req GetDatasou if err != nil { return err } - var wireResp datasourceConfigWire + var wireResp listBuiltinParserVersionsResponseWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = datasourceConfigFromWire(&wireResp) + resp, err = listBuiltinParserVersionsResponseFromWire(&wireResp) if err != nil { return err } @@ -264,8 +277,53 @@ func (c *internalClient) GetDatasourceConfig(ctx context.Context, req GetDatasou return resp, nil } -// Returns the detection-rule configuration for a catalog. -func (c *internalClient) GetRuleConfig(ctx context.Context, req GetRuleConfigRequest, opts ...call.Option) (*RuleConfig, error) { +// ListBuiltinParserVersionsIter returns an iterator that iterates +// over the results of ListBuiltinParserVersions. +// +// For example: +// +// for item, err := range c.ListBuiltinParserVersionsIter(ctx, ListBuiltinParserVersionsRequest{}) { +// if err != nil { +// return err +// } +// fmt.Println(item) +// } +// +// Options opts are passed to each ListBuiltinParserVersions call +// made by the iterator under the hood. +// +// Callers who need custom pagination logic should use +// ListBuiltinParserVersions directly. +func (c *internalClient) ListBuiltinParserVersionsIter(ctx context.Context, req ListBuiltinParserVersionsRequest, opts ...call.Option) iter.Seq2[*BuiltinParserVersion, error] { + return func(yield func(*BuiltinParserVersion, error) bool) { + // Keep pagination state local to this traversal so reusing the iterator starts + // from the original request. Reference-bearing fields remain shared and read-only. + pageReq := req + for { + resp, err := c.ListBuiltinParserVersions(ctx, pageReq, opts...) + if err != nil { + yield(nil, err) + return + } + for i := range resp.BuiltinParserVersions { + if !yield(&resp.BuiltinParserVersions[i], nil) { + return + } + } + if resp.NextPageToken == nil || *resp.NextPageToken == "" { + return + } + pageReq.PageToken = resp.NextPageToken + } + } +} + +// Lists the built-in parsers Databricks publishes, without their specs. +func (c *internalClient) ListBuiltinParsers(ctx context.Context, req ListBuiltinParsersRequest, opts ...call.Option) (*ListBuiltinParsersResponse, error) { + wireReq, err := listBuiltinParsersRequestToWire(&req) + if err != nil { + return nil, err + } headers := http.Header{} if c.workspaceID != "" { @@ -276,18 +334,21 @@ func (c *internalClient) GetRuleConfig(ctx context.Context, req GetRuleConfigReq if err != nil { return nil, err } - pb := pathBuilder{} - pb.literal("/api/2.0/lakewatch/") - if req.Name == nil { - return nil, fmt.Errorf("path parameter %q is required", "name") - } - pb.singleSegment(*req.Name) - baseURL.Path, baseURL.RawPath = pb.build() + baseURL.Path = "/api/2.0/lakewatch/builtin-parsers" queryParams := url.Values{} + if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "filter", wireReq.Filter); err != nil { + return nil, err + } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *RuleConfig + var resp *ListBuiltinParsersResponse call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -309,11 +370,11 @@ func (c *internalClient) GetRuleConfig(ctx context.Context, req GetRuleConfigReq if err != nil { return err } - var wireResp ruleConfigWire + var wireResp listBuiltinParsersResponseWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = ruleConfigFromWire(&wireResp) + resp, err = listBuiltinParsersResponseFromWire(&wireResp) if err != nil { return err } @@ -326,19 +387,51 @@ func (c *internalClient) GetRuleConfig(ctx context.Context, req GetRuleConfigReq return resp, nil } -// Updates the alert delivery configuration for a catalog. -func (c *internalClient) UpdateAlertConfig(ctx context.Context, req UpdateAlertConfigRequest, opts ...call.Option) (*AlertConfig, error) { - wireReq, err := updateAlertConfigRequestToWire(&req) - if err != nil { - return nil, err - } - body, err := json.Marshal(wireReq.AlertConfig) - if err != nil { - return nil, err +// ListBuiltinParsersIter returns an iterator that iterates +// over the results of ListBuiltinParsers. +// +// For example: +// +// for item, err := range c.ListBuiltinParsersIter(ctx, ListBuiltinParsersRequest{}) { +// if err != nil { +// return err +// } +// fmt.Println(item) +// } +// +// Options opts are passed to each ListBuiltinParsers call +// made by the iterator under the hood. +// +// Callers who need custom pagination logic should use +// ListBuiltinParsers directly. +func (c *internalClient) ListBuiltinParsersIter(ctx context.Context, req ListBuiltinParsersRequest, opts ...call.Option) iter.Seq2[*BuiltinParser, error] { + return func(yield func(*BuiltinParser, error) bool) { + // Keep pagination state local to this traversal so reusing the iterator starts + // from the original request. Reference-bearing fields remain shared and read-only. + pageReq := req + for { + resp, err := c.ListBuiltinParsers(ctx, pageReq, opts...) + if err != nil { + yield(nil, err) + return + } + for i := range resp.BuiltinParsers { + if !yield(&resp.BuiltinParsers[i], nil) { + return + } + } + if resp.NextPageToken == nil || *resp.NextPageToken == "" { + return + } + pageReq.PageToken = resp.NextPageToken + } } +} + +// Returns the alert delivery configuration for a catalog. +func (c *internalClient) GetAlertConfig(ctx context.Context, req GetAlertConfigRequest, opts ...call.Option) (*AlertConfig, error) { headers := http.Header{} - headers.Set("Content-Type", "application/json") if c.workspaceID != "" { headers.Set("X-Databricks-Workspace-Id", c.workspaceID) } @@ -349,19 +442,12 @@ func (c *internalClient) UpdateAlertConfig(ctx context.Context, req UpdateAlertC } pb := pathBuilder{} pb.literal("/api/2.0/lakewatch/") - if req.AlertConfig == nil || req.AlertConfig.Name == nil { - pb.singleSegment("") - } else { - pb.singleSegment(*req.AlertConfig.Name) + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") } + pb.singleSegment(*req.Name) baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} - if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { - return nil, err - } - if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { - return nil, err - } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() @@ -369,12 +455,11 @@ func (c *internalClient) UpdateAlertConfig(ctx context.Context, req UpdateAlertC call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "PATCH", + Method: "GET", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, Headers: headers, - Body: bytes.NewBuffer(body), }) if err != nil { return err @@ -405,19 +490,10 @@ func (c *internalClient) UpdateAlertConfig(ctx context.Context, req UpdateAlertC return resp, nil } -// Updates the common Lakewatch configuration for a catalog. -func (c *internalClient) UpdateCommonConfig(ctx context.Context, req UpdateCommonConfigRequest, opts ...call.Option) (*CommonConfig, error) { - wireReq, err := updateCommonConfigRequestToWire(&req) - if err != nil { - return nil, err - } - body, err := json.Marshal(wireReq.CommonConfig) - if err != nil { - return nil, err - } +// Returns the common Lakewatch configuration for a catalog. +func (c *internalClient) GetCommonConfig(ctx context.Context, req GetCommonConfigRequest, opts ...call.Option) (*CommonConfig, error) { headers := http.Header{} - headers.Set("Content-Type", "application/json") if c.workspaceID != "" { headers.Set("X-Databricks-Workspace-Id", c.workspaceID) } @@ -428,19 +504,12 @@ func (c *internalClient) UpdateCommonConfig(ctx context.Context, req UpdateCommo } pb := pathBuilder{} pb.literal("/api/2.0/lakewatch/") - if req.CommonConfig == nil || req.CommonConfig.Name == nil { - pb.singleSegment("") - } else { - pb.singleSegment(*req.CommonConfig.Name) + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") } + pb.singleSegment(*req.Name) baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} - if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { - return nil, err - } - if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { - return nil, err - } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() @@ -448,12 +517,11 @@ func (c *internalClient) UpdateCommonConfig(ctx context.Context, req UpdateCommo call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "PATCH", + Method: "GET", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, Headers: headers, - Body: bytes.NewBuffer(body), }) if err != nil { return err @@ -484,19 +552,10 @@ func (c *internalClient) UpdateCommonConfig(ctx context.Context, req UpdateCommo return resp, nil } -// Updates the datasource ingestion configuration for a catalog. -func (c *internalClient) UpdateDatasourceConfig(ctx context.Context, req UpdateDatasourceConfigRequest, opts ...call.Option) (*DatasourceConfig, error) { - wireReq, err := updateDatasourceConfigRequestToWire(&req) - if err != nil { - return nil, err - } - body, err := json.Marshal(wireReq.DatasourceConfig) - if err != nil { - return nil, err - } +// Returns the datasource ingestion configuration for a catalog. +func (c *internalClient) GetDatasourceConfig(ctx context.Context, req GetDatasourceConfigRequest, opts ...call.Option) (*DatasourceConfig, error) { headers := http.Header{} - headers.Set("Content-Type", "application/json") if c.workspaceID != "" { headers.Set("X-Databricks-Workspace-Id", c.workspaceID) } @@ -507,19 +566,12 @@ func (c *internalClient) UpdateDatasourceConfig(ctx context.Context, req UpdateD } pb := pathBuilder{} pb.literal("/api/2.0/lakewatch/") - if req.DatasourceConfig == nil || req.DatasourceConfig.Name == nil { - pb.singleSegment("") - } else { - pb.singleSegment(*req.DatasourceConfig.Name) + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") } + pb.singleSegment(*req.Name) baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} - if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { - return nil, err - } - if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { - return nil, err - } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() @@ -527,12 +579,11 @@ func (c *internalClient) UpdateDatasourceConfig(ctx context.Context, req UpdateD call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "PATCH", + Method: "GET", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, Headers: headers, - Body: bytes.NewBuffer(body), }) if err != nil { return err @@ -563,19 +614,10 @@ func (c *internalClient) UpdateDatasourceConfig(ctx context.Context, req UpdateD return resp, nil } -// Updates the detection-rule configuration for a catalog. -func (c *internalClient) UpdateRuleConfig(ctx context.Context, req UpdateRuleConfigRequest, opts ...call.Option) (*RuleConfig, error) { - wireReq, err := updateRuleConfigRequestToWire(&req) - if err != nil { - return nil, err - } - body, err := json.Marshal(wireReq.RuleConfig) - if err != nil { - return nil, err - } +// Returns the detection-rule configuration for a catalog. +func (c *internalClient) GetRuleConfig(ctx context.Context, req GetRuleConfigRequest, opts ...call.Option) (*RuleConfig, error) { headers := http.Header{} - headers.Set("Content-Type", "application/json") if c.workspaceID != "" { headers.Set("X-Databricks-Workspace-Id", c.workspaceID) } @@ -586,19 +628,12 @@ func (c *internalClient) UpdateRuleConfig(ctx context.Context, req UpdateRuleCon } pb := pathBuilder{} pb.literal("/api/2.0/lakewatch/") - if req.RuleConfig == nil || req.RuleConfig.Name == nil { - pb.singleSegment("") - } else { - pb.singleSegment(*req.RuleConfig.Name) + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") } + pb.singleSegment(*req.Name) baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} - if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { - return nil, err - } - if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { - return nil, err - } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() @@ -606,12 +641,11 @@ func (c *internalClient) UpdateRuleConfig(ctx context.Context, req UpdateRuleCon call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "PATCH", + Method: "GET", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, Headers: headers, - Body: bytes.NewBuffer(body), }) if err != nil { return err @@ -642,13 +676,13 @@ func (c *internalClient) UpdateRuleConfig(ctx context.Context, req UpdateRuleCon return resp, nil } -// Creates a new datasource. -func (c *internalClient) CreateDatasource(ctx context.Context, req CreateDatasourceRequest, opts ...call.Option) (*Datasource, error) { - wireReq, err := createDatasourceRequestToWire(&req) +// Updates the alert delivery configuration for a catalog. +func (c *internalClient) UpdateAlertConfig(ctx context.Context, req UpdateAlertConfigRequest, opts ...call.Option) (*AlertConfig, error) { + wireReq, err := updateAlertConfigRequestToWire(&req) if err != nil { return nil, err } - body, err := json.Marshal(wireReq.Datasource) + body, err := json.Marshal(wireReq.AlertConfig) if err != nil { return nil, err } @@ -663,16 +697,29 @@ func (c *internalClient) CreateDatasource(ctx context.Context, req CreateDatasou if err != nil { return nil, err } - baseURL.Path = "/api/2.0/lakewatch/datasources" + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.AlertConfig == nil || req.AlertConfig.Name == nil { + pb.singleSegment("") + } else { + pb.singleSegment(*req.AlertConfig.Name) + } + baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} + if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + return nil, err + } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *Datasource + var resp *AlertConfig call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "POST", + Method: "PATCH", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, @@ -691,7 +738,627 @@ func (c *internalClient) CreateDatasource(ctx context.Context, req CreateDatasou if err != nil { return err } - var wireResp datasourceWire + var wireResp alertConfigWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = alertConfigFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Updates the common Lakewatch configuration for a catalog. +func (c *internalClient) UpdateCommonConfig(ctx context.Context, req UpdateCommonConfigRequest, opts ...call.Option) (*CommonConfig, error) { + wireReq, err := updateCommonConfigRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.CommonConfig) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.CommonConfig == nil || req.CommonConfig.Name == nil { + pb.singleSegment("") + } else { + pb.singleSegment(*req.CommonConfig.Name) + } + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + return nil, err + } + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *CommonConfig + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "PATCH", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp commonConfigWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = commonConfigFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Updates the datasource ingestion configuration for a catalog. +func (c *internalClient) UpdateDatasourceConfig(ctx context.Context, req UpdateDatasourceConfigRequest, opts ...call.Option) (*DatasourceConfig, error) { + wireReq, err := updateDatasourceConfigRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.DatasourceConfig) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.DatasourceConfig == nil || req.DatasourceConfig.Name == nil { + pb.singleSegment("") + } else { + pb.singleSegment(*req.DatasourceConfig.Name) + } + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + return nil, err + } + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *DatasourceConfig + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "PATCH", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp datasourceConfigWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = datasourceConfigFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Updates the detection-rule configuration for a catalog. +func (c *internalClient) UpdateRuleConfig(ctx context.Context, req UpdateRuleConfigRequest, opts ...call.Option) (*RuleConfig, error) { + wireReq, err := updateRuleConfigRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.RuleConfig) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.RuleConfig == nil || req.RuleConfig.Name == nil { + pb.singleSegment("") + } else { + pb.singleSegment(*req.RuleConfig.Name) + } + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + return nil, err + } + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *RuleConfig + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "PATCH", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp ruleConfigWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = ruleConfigFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Creates a new datasource. +func (c *internalClient) CreateDatasource(ctx context.Context, req CreateDatasourceRequest, opts ...call.Option) (*Datasource, error) { + wireReq, err := createDatasourceRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.Datasource) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + baseURL.Path = "/api/2.0/lakewatch/datasources" + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *Datasource + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "POST", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp datasourceWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = datasourceFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Deletes a datasource by its resource name. +func (c *internalClient) DeleteDatasource(ctx context.Context, req DeleteDatasourceRequest, opts ...call.Option) error { + + headers := http.Header{} + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Name == nil { + return fmt.Errorf("path parameter %q is required", "name") + } + pb.singleSegment(*req.Name) + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "DELETE", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + _ = respBody + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return err + } + return nil +} + +// Returns a single datasource by its resource name. +func (c *internalClient) GetDatasource(ctx context.Context, req GetDatasourceRequest, opts ...call.Option) (*Datasource, error) { + + headers := http.Header{} + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") + } + pb.singleSegment(*req.Name) + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *Datasource + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "GET", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp datasourceWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = datasourceFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Lists the datasources in a Unity Catalog catalog, optionally narrowed to one +// schema or filtered by name. Results are paginated. Datasources that are still +// being created or deleted are not listed. +func (c *internalClient) ListDatasources(ctx context.Context, req ListDatasourcesRequest, opts ...call.Option) (*ListDatasourcesResponse, error) { + wireReq, err := listDatasourcesRequestToWire(&req) + if err != nil { + return nil, err + } + + headers := http.Header{} + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + baseURL.Path = "/api/2.0/lakewatch/datasources" + queryParams := url.Values{} + if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "filter", wireReq.Filter); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "catalog_name", wireReq.CatalogName); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "schema_name", wireReq.SchemaName); err != nil { + return nil, err + } + if wireReq.View != "" { + if err := addQueryValue(queryParams, "view", wireReq.View); err != nil { + return nil, err + } + } + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *ListDatasourcesResponse + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "GET", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp listDatasourcesResponseWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = listDatasourcesResponseFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// ListDatasourcesIter returns an iterator that iterates +// over the results of ListDatasources. +// +// For example: +// +// for item, err := range c.ListDatasourcesIter(ctx, ListDatasourcesRequest{}) { +// if err != nil { +// return err +// } +// fmt.Println(item) +// } +// +// Options opts are passed to each ListDatasources call +// made by the iterator under the hood. +// +// Callers who need custom pagination logic should use +// ListDatasources directly. +func (c *internalClient) ListDatasourcesIter(ctx context.Context, req ListDatasourcesRequest, opts ...call.Option) iter.Seq2[*Datasource, error] { + return func(yield func(*Datasource, error) bool) { + // Keep pagination state local to this traversal so reusing the iterator starts + // from the original request. Reference-bearing fields remain shared and read-only. + pageReq := req + for { + resp, err := c.ListDatasources(ctx, pageReq, opts...) + if err != nil { + yield(nil, err) + return + } + for i := range resp.Datasources { + if !yield(&resp.Datasources[i], nil) { + return + } + } + if resp.NextPageToken == nil || *resp.NextPageToken == "" { + return + } + pageReq.PageToken = resp.NextPageToken + } + } +} + +// Updates an existing datasource. +func (c *internalClient) UpdateDatasource(ctx context.Context, req UpdateDatasourceRequest, opts ...call.Option) (*Datasource, error) { + wireReq, err := updateDatasourceRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.Datasource) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Datasource == nil || req.Datasource.Name == nil { + pb.singleSegment("") + } else { + pb.singleSegment(*req.Datasource.Name) + } + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + return nil, err + } + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *Datasource + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "PATCH", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp datasourceWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } @@ -708,8 +1375,149 @@ func (c *internalClient) CreateDatasource(ctx context.Context, req CreateDatasou return resp, nil } -// Deletes a datasource by its resource name. -func (c *internalClient) DeleteDatasource(ctx context.Context, req DeleteDatasourceRequest, opts ...call.Option) error { +// Creates a parser and its first version in one call. +func (c *internalClient) CreateParser(ctx context.Context, req CreateParserRequest, opts ...call.Option) (*Parser, error) { + wireReq, err := createParserRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.Parser) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + baseURL.Path = "/api/2.0/lakewatch/parsers" + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *Parser + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "POST", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp parserWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = parserFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Appends a new immutable version, advancing the parser's latest-version +// pointer. This is how a parser is edited; versions are never changed in place. +func (c *internalClient) CreateParserVersion(ctx context.Context, req CreateParserVersionRequest, opts ...call.Option) (*ParserVersion, error) { + wireReq, err := createParserVersionRequestToWire(&req) + if err != nil { + return nil, err + } + body, err := json.Marshal(wireReq.ParserVersion) + if err != nil { + return nil, err + } + + headers := http.Header{} + headers.Set("Content-Type", "application/json") + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return nil, err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Parent == nil { + return nil, fmt.Errorf("path parameter %q is required", "parent") + } + pb.singleSegment(*req.Parent) + pb.literal("/versions") + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *ParserVersion + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "POST", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + Body: bytes.NewBuffer(body), + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp parserVersionWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = parserVersionFromWire(&wireResp) + if err != nil { + return err + } + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return nil, err + } + return resp, nil +} + +// Deletes a parser and all of its versions. Refused while any data source +// references it. +func (c *internalClient) DeleteParser(ctx context.Context, req DeleteParserRequest, opts ...call.Option) error { headers := http.Header{} if c.workspaceID != "" { @@ -761,8 +1569,65 @@ func (c *internalClient) DeleteDatasource(ctx context.Context, req DeleteDatasou return nil } -// Returns a single datasource by its resource name. -func (c *internalClient) GetDatasource(ctx context.Context, req GetDatasourceRequest, opts ...call.Option) (*Datasource, error) { +// Deletes one version of a parser. Returns NOT_FOUND when the parser or the +// version does not exist, and a conflict (HTTP 409) only while a data source +// references the version. Deleting the last version leaves the parser in place +// with no versions. +func (c *internalClient) DeleteParserVersion(ctx context.Context, req DeleteParserVersionRequest, opts ...call.Option) error { + + headers := http.Header{} + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { + return err + } + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Name == nil { + return fmt.Errorf("path parameter %q is required", "name") + } + pb.singleSegment(*req.Name) + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "DELETE", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + _ = respBody + return nil + } + + if err := c.executeCall(ctx, call, opts); err != nil { + return err + } + return nil +} + +// Returns a parser's identity and latest-version pointer. Does not return the +// spec; read a spec with GetParserVersion. +func (c *internalClient) GetParser(ctx context.Context, req GetParserRequest, opts ...call.Option) (*Parser, error) { headers := http.Header{} if c.workspaceID != "" { @@ -784,7 +1649,7 @@ func (c *internalClient) GetDatasource(ctx context.Context, req GetDatasourceReq baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *Datasource + var resp *Parser call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -806,11 +1671,11 @@ func (c *internalClient) GetDatasource(ctx context.Context, req GetDatasourceReq if err != nil { return err } - var wireResp datasourceWire + var wireResp parserWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = datasourceFromWire(&wireResp) + resp, err = parserFromWire(&wireResp) if err != nil { return err } @@ -823,14 +1688,8 @@ func (c *internalClient) GetDatasource(ctx context.Context, req GetDatasourceReq return resp, nil } -// Lists the datasources in a Unity Catalog catalog, optionally narrowed to one -// schema or filtered by name. Results are paginated. Datasources that are still -// being created or deleted are not listed. -func (c *internalClient) ListDatasources(ctx context.Context, req ListDatasourcesRequest, opts ...call.Option) (*ListDatasourcesResponse, error) { - wireReq, err := listDatasourcesRequestToWire(&req) - if err != nil { - return nil, err - } +// Returns one version of a parser, including its spec. +func (c *internalClient) GetParserVersion(ctx context.Context, req GetParserVersionRequest, opts ...call.Option) (*ParserVersion, error) { headers := http.Header{} if c.workspaceID != "" { @@ -841,32 +1700,91 @@ func (c *internalClient) ListDatasources(ctx context.Context, req ListDatasource if err != nil { return nil, err } - baseURL.Path = "/api/2.0/lakewatch/datasources" + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Name == nil { + return nil, fmt.Errorf("path parameter %q is required", "name") + } + pb.singleSegment(*req.Name) + baseURL.Path, baseURL.RawPath = pb.build() queryParams := url.Values{} - if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { - return nil, err + baseURL.RawQuery = queryParams.Encode() + urlStr := baseURL.String() + + var resp *ParserVersion + + call := func(ctx context.Context) error { + httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ + Method: "GET", + URL: urlStr, + Credentials: c.credentials, + UserAgent: c.userAgent, + Headers: headers, + }) + if err != nil { + return err + } + + respBody, _, err := executeHTTPCall(httpCallOptions{ + req: httpReq, + client: c.httpClient, + logger: c.logger, + }) + if err != nil { + return err + } + var wireResp parserVersionWire + if err := json.Unmarshal(respBody, &wireResp); err != nil { + return err + } + resp, err = parserVersionFromWire(&wireResp) + if err != nil { + return err + } + return nil } - if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + + if err := c.executeCall(ctx, call, opts); err != nil { return nil, err } - if err := addQueryValue(queryParams, "filter", wireReq.Filter); err != nil { + return resp, nil +} + +// Lists a parser's versions, oldest first, without their specs. +func (c *internalClient) ListParserVersions(ctx context.Context, req ListParserVersionsRequest, opts ...call.Option) (*ListParserVersionsResponse, error) { + wireReq, err := listParserVersionsRequestToWire(&req) + if err != nil { return nil, err } - if err := addQueryValue(queryParams, "catalog_name", wireReq.CatalogName); err != nil { + + headers := http.Header{} + if c.workspaceID != "" { + headers.Set("X-Databricks-Workspace-Id", c.workspaceID) + } + + baseURL, err := url.Parse(c.host) + if err != nil { return nil, err } - if err := addQueryValue(queryParams, "schema_name", wireReq.SchemaName); err != nil { + pb := pathBuilder{} + pb.literal("/api/2.0/lakewatch/") + if req.Parent == nil { + return nil, fmt.Errorf("path parameter %q is required", "parent") + } + pb.singleSegment(*req.Parent) + pb.literal("/versions") + baseURL.Path, baseURL.RawPath = pb.build() + queryParams := url.Values{} + if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { return nil, err } - if wireReq.View != "" { - if err := addQueryValue(queryParams, "view", wireReq.View); err != nil { - return nil, err - } + if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + return nil, err } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *ListDatasourcesResponse + var resp *ListParserVersionsResponse call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ @@ -888,11 +1806,11 @@ func (c *internalClient) ListDatasources(ctx context.Context, req ListDatasource if err != nil { return err } - var wireResp listDatasourcesResponseWire + var wireResp listParserVersionsResponseWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = listDatasourcesResponseFromWire(&wireResp) + resp, err = listParserVersionsResponseFromWire(&wireResp) if err != nil { return err } @@ -905,36 +1823,36 @@ func (c *internalClient) ListDatasources(ctx context.Context, req ListDatasource return resp, nil } -// ListDatasourcesIter returns an iterator that iterates -// over the results of ListDatasources. +// ListParserVersionsIter returns an iterator that iterates +// over the results of ListParserVersions. // // For example: // -// for item, err := range c.ListDatasourcesIter(ctx, ListDatasourcesRequest{}) { +// for item, err := range c.ListParserVersionsIter(ctx, ListParserVersionsRequest{}) { // if err != nil { // return err // } // fmt.Println(item) // } // -// Options opts are passed to each ListDatasources call +// Options opts are passed to each ListParserVersions call // made by the iterator under the hood. // // Callers who need custom pagination logic should use -// ListDatasources directly. -func (c *internalClient) ListDatasourcesIter(ctx context.Context, req ListDatasourcesRequest, opts ...call.Option) iter.Seq2[*Datasource, error] { - return func(yield func(*Datasource, error) bool) { +// ListParserVersions directly. +func (c *internalClient) ListParserVersionsIter(ctx context.Context, req ListParserVersionsRequest, opts ...call.Option) iter.Seq2[*ParserVersion, error] { + return func(yield func(*ParserVersion, error) bool) { // Keep pagination state local to this traversal so reusing the iterator starts // from the original request. Reference-bearing fields remain shared and read-only. pageReq := req for { - resp, err := c.ListDatasources(ctx, pageReq, opts...) + resp, err := c.ListParserVersions(ctx, pageReq, opts...) if err != nil { yield(nil, err) return } - for i := range resp.Datasources { - if !yield(&resp.Datasources[i], nil) { + for i := range resp.ParserVersions { + if !yield(&resp.ParserVersions[i], nil) { return } } @@ -946,19 +1864,14 @@ func (c *internalClient) ListDatasourcesIter(ctx context.Context, req ListDataso } } -// Updates an existing datasource. -func (c *internalClient) UpdateDatasource(ctx context.Context, req UpdateDatasourceRequest, opts ...call.Option) (*Datasource, error) { - wireReq, err := updateDatasourceRequestToWire(&req) - if err != nil { - return nil, err - } - body, err := json.Marshal(wireReq.Datasource) +// Lists parsers in a catalog, without their specs. +func (c *internalClient) ListParsers(ctx context.Context, req ListParsersRequest, opts ...call.Option) (*ListParsersResponse, error) { + wireReq, err := listParsersRequestToWire(&req) if err != nil { return nil, err } headers := http.Header{} - headers.Set("Content-Type", "application/json") if c.workspaceID != "" { headers.Set("X-Databricks-Workspace-Id", c.workspaceID) } @@ -967,34 +1880,35 @@ func (c *internalClient) UpdateDatasource(ctx context.Context, req UpdateDatasou if err != nil { return nil, err } - pb := pathBuilder{} - pb.literal("/api/2.0/lakewatch/") - if req.Datasource == nil || req.Datasource.Name == nil { - pb.singleSegment("") - } else { - pb.singleSegment(*req.Datasource.Name) - } - baseURL.Path, baseURL.RawPath = pb.build() + baseURL.Path = "/api/2.0/lakewatch/parsers" queryParams := url.Values{} - if err := addQueryValue(queryParams, "update_mask", wireReq.UpdateMask); err != nil { + if err := addQueryValue(queryParams, "page_size", wireReq.PageSize); err != nil { return nil, err } - if err := addQueryValue(queryParams, "etag", wireReq.Etag); err != nil { + if err := addQueryValue(queryParams, "page_token", wireReq.PageToken); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "filter", wireReq.Filter); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "catalog_name", wireReq.CatalogName); err != nil { + return nil, err + } + if err := addQueryValue(queryParams, "schema_name", wireReq.SchemaName); err != nil { return nil, err } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() - var resp *Datasource + var resp *ListParsersResponse call := func(ctx context.Context) error { httpReq, err := newHTTPRequest(ctx, httpRequestOptions{ - Method: "PATCH", + Method: "GET", URL: urlStr, Credentials: c.credentials, UserAgent: c.userAgent, Headers: headers, - Body: bytes.NewBuffer(body), }) if err != nil { return err @@ -1008,11 +1922,11 @@ func (c *internalClient) UpdateDatasource(ctx context.Context, req UpdateDatasou if err != nil { return err } - var wireResp datasourceWire + var wireResp listParsersResponseWire if err := json.Unmarshal(respBody, &wireResp); err != nil { return err } - resp, err = datasourceFromWire(&wireResp) + resp, err = listParsersResponseFromWire(&wireResp) if err != nil { return err } @@ -1025,6 +1939,47 @@ func (c *internalClient) UpdateDatasource(ctx context.Context, req UpdateDatasou return resp, nil } +// ListParsersIter returns an iterator that iterates +// over the results of ListParsers. +// +// For example: +// +// for item, err := range c.ListParsersIter(ctx, ListParsersRequest{}) { +// if err != nil { +// return err +// } +// fmt.Println(item) +// } +// +// Options opts are passed to each ListParsers call +// made by the iterator under the hood. +// +// Callers who need custom pagination logic should use +// ListParsers directly. +func (c *internalClient) ListParsersIter(ctx context.Context, req ListParsersRequest, opts ...call.Option) iter.Seq2[*Parser, error] { + return func(yield func(*Parser, error) bool) { + // Keep pagination state local to this traversal so reusing the iterator starts + // from the original request. Reference-bearing fields remain shared and read-only. + pageReq := req + for { + resp, err := c.ListParsers(ctx, pageReq, opts...) + if err != nil { + yield(nil, err) + return + } + for i := range resp.Parsers { + if !yield(&resp.Parsers[i], nil) { + return + } + } + if resp.NextPageToken == nil || *resp.NextPageToken == "" { + return + } + pageReq.PageToken = resp.NextPageToken + } + } +} + // Creates a new rule. func (c *internalClient) CreateRule(ctx context.Context, req CreateRuleRequest, opts ...call.Option) (*Rule, error) { wireReq, err := createRuleRequestToWire(&req) diff --git a/lakewatch/v1/model.go b/lakewatch/v1/model.go index 6db1e791..686f82ed 100644 --- a/lakewatch/v1/model.go +++ b/lakewatch/v1/model.go @@ -255,6 +255,55 @@ type autoLoaderCredentialsFieldMaskMetadata struct { *AutoLoader_Credentials_AwsSecretScope } +// A Databricks-authored parser, shared by every customer. +// +// Same container-plus-versions shape as the custom `Parser`, and addressed the +// same way, by UUID. Built-ins are global rather than catalog-scoped, so they +// carry no owning catalog or schema.. +type BuiltinParser struct { + // Resource name. Format: `builtin-parsers/{parser_id}`. + Name *string + // The parser's UUID, authored into the built-in's published definition rather + // than minted per customer, so every workspace addresses the same built-in by + // the same id. It is the leaf of `name`. + ParserId *string + // The built-in's own leaf name, as published in content-marketplace. Stable and + // human-readable, which is what a UI shows; `name` addresses the resource by + // id. + ParserName *string + // Key/value tags describing this parser, e.g. `cloud_provider=aws` or + // `log_source=cloudtrail`. A parser may carry up to 32. Filterable on list. + Tags []ParserTag + // Newest published version number. + LatestVersion *int64 +} + +// One published version of a built-in parser, carrying the spec. +// +// The content pipeline owns versioning, so unlike the custom version there is +// no authoring principal and no append timestamp here.. +type BuiltinParserVersion struct { + // Resource name. Format: `builtin-parsers/{parser_id}/versions/{version}`. + Name *string + // Version number. + Version *int64 + // The parser spec: the field definitions for this log source, as a YAML + // document. Read-only; built-in specs are authored by Databricks, so no + // client-facing size cap is enforced here. + Spec *string + // The parent built-in's published leaf name, copied from the container on read. + // Denormalized so a caller that needs identity alongside the spec reads the + // version alone rather than the container as well. Populated by + // `GetBuiltinParserVersion`; List omits it, as List omits `spec`. + ParserName *string + // The parent built-in's tags, copied from the container on read. Rendering + // reads these for connector-specific details, such as the source table a + // Lakeflow Connect parser reads. + Tags []ParserTag + // Free-form description of this parser, authored by Databricks. + Description *string +} + // Unwraps an AWS CloudWatch Logs delivery envelope: each element of `logEvents` // becomes its own row, with the element's `message` parsed as the event body.. type CloudWatchLogsEnvelope struct { @@ -295,6 +344,21 @@ type CreateDatasourceRequest struct { Datasource *Datasource } +// Request to create a parser and its first version in one call.. +type CreateParserRequest struct { + // The parser to create. `spec` and `description` become version 1. + Parser *Parser +} + +// Request to append a new immutable version to a parser.. +type CreateParserVersionRequest struct { + // Resource name of the parser to append a version to. Format: + // `parsers/{parser_id}`. + Parent *string + // The version to append. + ParserVersion *ParserVersion +} + type CreateRuleRequest struct { // The rule to create. Server-assigned fields (`name`, `rule_id`, timestamps, // `etag`, …) are ignored on the request. @@ -486,6 +550,22 @@ type DeleteDatasourceRequest struct { Name *string } +// Request to delete a parser and all of its versions.. +type DeleteParserRequest struct { + // Resource name of the parser. Format: `parsers/{parser_id}`. + Name *string +} + +// Request to delete one version of a parser.. +type DeleteParserVersionRequest struct { + // Resource name of the version. Format: + // `parsers/{parser_id}/versions/{version}`, where `{version}` is a version + // number written without sign or leading zeros, such as `3`. Unlike + // GetParserVersion, the alias `latest` and forms such as `007` or `+3` are + // rejected, so a delete always names one fixed version. + Name *string +} + type DeleteRuleRequest struct { // Resource name of the rule to delete. Format: `rules/{rule_id}`. Name *string @@ -623,6 +703,20 @@ type GetAlertConfigRequest struct { Name *string } +// Request for one built-in parser's identity and latest-version pointer.. +type GetBuiltinParserRequest struct { + // Resource name. Format: `builtin-parsers/{parser_id}`. + Name *string +} + +// Request for one version of a built-in parser, including its spec.. +type GetBuiltinParserVersionRequest struct { + // Resource name of the version. Format: + // `builtin-parsers/{parser_id}/versions/{version}`, where `{version}` is either + // a version number or the alias `latest`. + Name *string +} + type GetCommonConfigRequest struct { Name *string } @@ -635,6 +729,21 @@ type GetDatasourceRequest struct { Name *string } +// Request for one parser's identity and latest-version pointer.. +type GetParserRequest struct { + // Resource name of the parser. Format: `parsers/{parser_id}`. + Name *string +} + +// Request for one version of a parser, including its spec.. +type GetParserVersionRequest struct { + // Resource name of the version. Format: + // `parsers/{parser_id}/versions/{version}`, where `{version}` is either a + // version number or the alias `latest`. `latest` is NOT_FOUND when every + // version of the parser has been deleted. + Name *string +} + type GetRuleConfigRequest struct { Name *string } @@ -670,6 +779,55 @@ type JsonArrayEnvelope struct { Path *string `fieldmask:"path"` } +// Request to list a built-in parser's published versions.. +type ListBuiltinParserVersionsRequest struct { + // Resource name of the built-in parser whose versions are listed. Format: + // `builtin-parsers/{parser_id}`. + Parent *string + // Maximum number of results to return in one page. If unset or 0, at most 100 + // versions are returned. The maximum allowed value is 1000. + PageSize *int + // Page token from a previous `ListBuiltinParserVersions` response, used to + // fetch the next page of results. + PageToken *string +} + +// A page of built-in version metadata, without the specs.. +type ListBuiltinParserVersionsResponse struct { + // Version metadata only, oldest first, without the spec. + BuiltinParserVersions []BuiltinParserVersion + // Opaque token for fetching the next page; pass it as `page_token` on the next + // call. It is empty when there are no more results. + NextPageToken *string +} + +// Request to list the built-in parsers Databricks publishes.. +type ListBuiltinParsersRequest struct { + // Maximum number of results to return in one page. This is a best-effort upper + // bound, not a guarantee: the server may return fewer items than requested + // (even zero) while more results still exist. If unset or 0, the server picks a + // default. + PageSize *int + // Page token from a previous `ListBuiltinParsers` response, used to fetch the + // next page of results. + PageToken *string + // AIP-160 filter expression. Built-ins are filterable by their `tags`; the + // exact tag-filter grammar is defined by the server. + Filter *string +} + +// A page of built-in parsers, without their specs.. +type ListBuiltinParsersResponse struct { + // Summary rows: the container fields only, without the spec. Read a spec with + // `GetBuiltinParserVersion`. + BuiltinParsers []BuiltinParser + // Opaque token for fetching the next page; pass it as `page_token` on the next + // call. It is empty only when there are no more results. Page size is + // best-effort, so a short or even empty page does not mean the end; keep paging + // until this token is empty, which is the only reliable end-of-results signal. + NextPageToken *string +} + type ListDatasourcesRequest struct { // Maximum number of results to return in one page. This is a best-effort upper // bound, not a guarantee: the server may return fewer items than requested @@ -705,6 +863,71 @@ type ListDatasourcesResponse struct { NextPageToken *string } +// Request to list a parser's versions.. +type ListParserVersionsRequest struct { + // Resource name of the parser whose versions are listed. Format: + // `parsers/{parser_id}`. + Parent *string + // Maximum number of results to return in one page. This is a best-effort upper + // bound, not a guarantee: the server may return fewer items than requested + // (even zero) while more results still exist. If unset or 0, the server picks a + // default. + PageSize *int + // Page token from a previous `ListParserVersions` response, used to fetch the + // next page of results. Send the same `parent` when requesting later pages; a + // token from a request for a different parser is rejected. + PageToken *string +} + +// A page of version metadata, without the specs.. +type ListParserVersionsResponse struct { + // Version metadata only, oldest first, without the spec. Each row's + // `create_time` and `created_by` are that version's own. + ParserVersions []ParserVersion + // Opaque token for fetching the next page; pass it as `page_token` on the next + // call. It is empty only when there are no more results. Page size is + // best-effort, so a short or even empty page does not mean the end; keep paging + // until this token is empty, which is the only reliable end-of-results signal. + NextPageToken *string +} + +// Request to list the parsers in a UC catalog, optionally narrowed by schema +// and/or an exact `parser_name` filter.. +type ListParsersRequest struct { + // Maximum number of results to return in one page. This is a best-effort upper + // bound, not a guarantee: the server may return fewer items than requested + // (even zero) while more results still exist. If unset or 0, the server picks a + // default. + PageSize *int + // Page token from a previous `ListParsers` response, used to fetch the next + // page of results. Send the other request fields unchanged when requesting + // later pages; a token from a request with different fields is rejected. + PageToken *string + // AIP-160 filter expression. Parsers are filterable by `parser_name`. + Filter *string + // Name of the parent UC catalog to list parsers from. Required: List is scoped + // to a single catalog, mirroring the Unity Catalog tables/functions APIs, which + // take catalog_name as a request field (bound to a query parameter) rather than + // a catalog path segment. + CatalogName *string + // Name of the parent UC schema. Optional: when set, results are narrowed to + // this schema; when omitted, parsers across all schemas in `catalog_name` are + // returned. + SchemaName *string +} + +// A page of parsers, without their specs.. +type ListParsersResponse struct { + // Summary rows: the container fields only, without the spec. Read a spec with + // `GetParserVersion`. + Parsers []Parser + // Opaque token for fetching the next page; pass it as `page_token` on the next + // call. It is empty only when there are no more results. Page size is + // best-effort, so a short or even empty page does not mean the end; keep paging + // until this token is empty, which is the only reliable end-of-results signal. + NextPageToken *string +} + type ListRulesRequest struct { // Maximum number of rules to return. The server may return fewer. If unset or // 0, the server picks a default page size. @@ -736,6 +959,61 @@ type ListRulesResponse struct { NextPageToken *string } +// A customer-authored, versioned contract describing the fields of an ingested +// log source. +// +// The parser is a container: it carries the identity and a pointer to the +// newest version. Specs live on the immutable `ParserVersion` children, so one +// parser can be attached to many data sources and each attachment pins a +// version.. +type Parser struct { + // Resource name. Format: `parsers/{parser_id}`, where the leaf is the immutable + // server-assigned UUID. Create-time placement names are stored separately and + // do not affect the resource name. + Name *string + // The parser's server-assigned UUID. It is not currently a UC securable ID, is + // immutable for the life of the resource, is the leaf of `name`, and is + // returned by CreateParser. + ParserId *string + // Name of the parent UC catalog, set on create. Reads return the value as + // supplied; the current UC catalog name is reported in + // `effective_catalog_name`. + CatalogName *string + // Name of the parent UC schema, set on create. Reads return the value as + // supplied; the current UC schema name is reported in `effective_schema_name`. + SchemaName *string + // Client-supplied parser name, stored as supplied and fixed at create time. + // Must be a valid UC identifier. A custom parser is not currently a UC + // securable and this field has no rename path. + ParserName *string + // Newest version of this parser. Read its spec by passing this to + // `GetParserVersion`, or by using the `latest` alias. Absent when every version + // has been deleted. + LatestVersion *int64 + // When the parser was created. + CreateTime *types.Time + // The principal that created this parser. Each version records its own + // `created_by`. + CreatedBy *Principal + // The principal that owns this parser. Server-set from the authenticated + // creator, never supplied by the client. Beta reads by UUID perform no + // authorization. Owner-gated write RPCs are outside this API's current + // implementation scope. + Owner *Principal + // The current name of the parser's UC catalog. Tracks UC renames; when the + // current name cannot be looked up, the name as of the parser's last write. + EffectiveCatalogName *string + // The current name of the parser's UC schema. Tracks UC renames; when the + // current name cannot be looked up, the name as of the parser's last write. + EffectiveSchemaName *string + // Spec of version 1, as a YAML document. Accepted only by `CreateParser`; read + // a version's spec with `GetParserVersion`. + Spec *string + // Description of version 1. Accepted only by `CreateParser`; read a version's + // description with `GetParserVersion`. + Description *string +} + // Reference to a parser that the datasource uses. Each parser defines the // schema of the parsed table that it writes.. type ParserRef struct { @@ -755,6 +1033,43 @@ type ParserRef struct { EffectiveTargetTable *TableName } +// A key/value tag on a built-in parser, e.g. `key = "cloud_provider"`, `value = +// "aws"`. Key/value pairs match the tag shape used across Databricks (MLflow, +// model-serving endpoints, clusters) rather than a bare string list.. +type ParserTag struct { + // The tag key, e.g. `cloud_provider`. Required: a keyless tag cannot be + // filtered on and is meaningless. + Key *string + // The tag value, e.g. `aws`. + Value *string +} + +// One immutable version of a parser, carrying the spec. +// +// Immutable: an edit creates a new version rather than mutating an existing +// one, so a data source that pins a version is never affected. Version numbers +// are never reused, including after a delete.. +type ParserVersion struct { + // Resource name. Format: `parsers/{parser_id}/versions/{version}`. + Name *string + // Version number, assigned by the server when the version is appended. + Version *int64 + // The field definitions for this log source, as a YAML document. Validated on + // write, so an invalid spec is never stored. + Spec *string + // When this version was appended. + CreateTime *types.Time + // The principal that appended this version. + CreatedBy *Principal + // Free-form description of this version, set when the version is created. + Description *string + // The parent parser's name, copied from the container on read. Denormalized so + // a caller that needs identity alongside the spec reads the version alone + // rather than the container as well. Populated by `GetParserVersion`; List + // omits it, as List omits `spec`. + ParserName *string +} + // A principal — user or service-principal — used as the value type for // `created_by` / `updated_by` / `assignee` etc. across v2 resources.. type Principal struct { diff --git a/lakewatch/v1/wire.go b/lakewatch/v1/wire.go index b34580e1..e55faef7 100644 --- a/lakewatch/v1/wire.go +++ b/lakewatch/v1/wire.go @@ -296,6 +296,157 @@ func (v *AutoLoader) UnmarshalJSON(data []byte) error { return nil } +type builtinParserWire struct { + Name *string `json:"name,omitempty"` + ParserId *string `json:"parser_id,omitempty"` + ParserName *string `json:"parser_name,omitempty"` + Tags []parserTagWire `json:"tags,omitempty"` + LatestVersion *wireInt64 `json:"latest_version,omitempty"` +} + +func builtinParserToWire(v *BuiltinParser) (*builtinParserWire, error) { + if v == nil { + return nil, nil + } + tagsWireValue, err := convertSlice(v.Tags, parserTagToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParser.Tags", err) + } + latestVersionWireValue, err := int64ToWire(v.LatestVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParser.LatestVersion", err) + } + return &builtinParserWire{ + Name: v.Name, + ParserId: v.ParserId, + ParserName: v.ParserName, + Tags: tagsWireValue, + LatestVersion: latestVersionWireValue, + }, nil +} + +func builtinParserFromWire(w *builtinParserWire) (*BuiltinParser, error) { + if w == nil { + return nil, nil + } + tagsPublicValue, err := convertSlice(w.Tags, parserTagFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParser.Tags", err) + } + latestVersionPublicValue, err := int64FromWire(w.LatestVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParser.LatestVersion", err) + } + return &BuiltinParser{ + Name: w.Name, + ParserId: w.ParserId, + ParserName: w.ParserName, + Tags: tagsPublicValue, + LatestVersion: latestVersionPublicValue, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v BuiltinParser) MarshalJSON() ([]byte, error) { + w, err := builtinParserToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *BuiltinParser) UnmarshalJSON(data []byte) error { + var w builtinParserWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := builtinParserFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type builtinParserVersionWire struct { + Name *string `json:"name,omitempty"` + Version *wireInt64 `json:"version,omitempty"` + Spec *string `json:"spec,omitempty"` + ParserName *string `json:"parser_name,omitempty"` + Tags []parserTagWire `json:"tags,omitempty"` + Description *string `json:"description,omitempty"` +} + +func builtinParserVersionToWire(v *BuiltinParserVersion) (*builtinParserVersionWire, error) { + if v == nil { + return nil, nil + } + versionWireValue, err := int64ToWire(v.Version) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParserVersion.Version", err) + } + tagsWireValue, err := convertSlice(v.Tags, parserTagToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParserVersion.Tags", err) + } + return &builtinParserVersionWire{ + Name: v.Name, + Version: versionWireValue, + Spec: v.Spec, + ParserName: v.ParserName, + Tags: tagsWireValue, + Description: v.Description, + }, nil +} + +func builtinParserVersionFromWire(w *builtinParserVersionWire) (*BuiltinParserVersion, error) { + if w == nil { + return nil, nil + } + versionPublicValue, err := int64FromWire(w.Version) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParserVersion.Version", err) + } + tagsPublicValue, err := convertSlice(w.Tags, parserTagFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "BuiltinParserVersion.Tags", err) + } + return &BuiltinParserVersion{ + Name: w.Name, + Version: versionPublicValue, + Spec: w.Spec, + ParserName: w.ParserName, + Tags: tagsPublicValue, + Description: w.Description, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v BuiltinParserVersion) MarshalJSON() ([]byte, error) { + w, err := builtinParserVersionToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *BuiltinParserVersion) UnmarshalJSON(data []byte) error { + var w builtinParserVersionWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := builtinParserVersionFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type cloudWatchLogsEnvelopeWire struct { HeaderRetention *bool `json:"header_retention,omitempty"` } @@ -522,6 +673,117 @@ func (v *CreateDatasourceRequest) UnmarshalJSON(data []byte) error { return nil } +type createParserRequestWire struct { + Parser *parserWire `json:"parser,omitempty"` +} + +func createParserRequestToWire(v *CreateParserRequest) (*createParserRequestWire, error) { + if v == nil { + return nil, nil + } + parserWireValue, err := parserToWire(v.Parser) + if err != nil { + return nil, fmt.Errorf("%s: %w", "CreateParserRequest.Parser", err) + } + return &createParserRequestWire{ + Parser: parserWireValue, + }, nil +} + +func createParserRequestFromWire(w *createParserRequestWire) (*CreateParserRequest, error) { + if w == nil { + return nil, nil + } + parserPublicValue, err := parserFromWire(w.Parser) + if err != nil { + return nil, fmt.Errorf("%s: %w", "CreateParserRequest.Parser", err) + } + return &CreateParserRequest{ + Parser: parserPublicValue, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v CreateParserRequest) MarshalJSON() ([]byte, error) { + w, err := createParserRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *CreateParserRequest) UnmarshalJSON(data []byte) error { + var w createParserRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := createParserRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type createParserVersionRequestWire struct { + Parent *string `json:"parent,omitempty"` + ParserVersion *parserVersionWire `json:"parser_version,omitempty"` +} + +func createParserVersionRequestToWire(v *CreateParserVersionRequest) (*createParserVersionRequestWire, error) { + if v == nil { + return nil, nil + } + parserVersionWireValue, err := parserVersionToWire(v.ParserVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "CreateParserVersionRequest.ParserVersion", err) + } + return &createParserVersionRequestWire{ + Parent: v.Parent, + ParserVersion: parserVersionWireValue, + }, nil +} + +func createParserVersionRequestFromWire(w *createParserVersionRequestWire) (*CreateParserVersionRequest, error) { + if w == nil { + return nil, nil + } + parserVersionPublicValue, err := parserVersionFromWire(w.ParserVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "CreateParserVersionRequest.ParserVersion", err) + } + return &CreateParserVersionRequest{ + Parent: w.Parent, + ParserVersion: parserVersionPublicValue, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v CreateParserVersionRequest) MarshalJSON() ([]byte, error) { + w, err := createParserVersionRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *CreateParserVersionRequest) UnmarshalJSON(data []byte) error { + var w createParserVersionRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := createParserVersionRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type createRuleRequestWire struct { Rule *ruleWire `json:"rule,omitempty"` } @@ -1017,6 +1279,98 @@ func (v *DeleteDatasourceRequest) UnmarshalJSON(data []byte) error { return nil } +type deleteParserRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func deleteParserRequestToWire(v *DeleteParserRequest) (*deleteParserRequestWire, error) { + if v == nil { + return nil, nil + } + return &deleteParserRequestWire{ + Name: v.Name, + }, nil +} + +func deleteParserRequestFromWire(w *deleteParserRequestWire) (*DeleteParserRequest, error) { + if w == nil { + return nil, nil + } + return &DeleteParserRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v DeleteParserRequest) MarshalJSON() ([]byte, error) { + w, err := deleteParserRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *DeleteParserRequest) UnmarshalJSON(data []byte) error { + var w deleteParserRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := deleteParserRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type deleteParserVersionRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func deleteParserVersionRequestToWire(v *DeleteParserVersionRequest) (*deleteParserVersionRequestWire, error) { + if v == nil { + return nil, nil + } + return &deleteParserVersionRequestWire{ + Name: v.Name, + }, nil +} + +func deleteParserVersionRequestFromWire(w *deleteParserVersionRequestWire) (*DeleteParserVersionRequest, error) { + if w == nil { + return nil, nil + } + return &DeleteParserVersionRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v DeleteParserVersionRequest) MarshalJSON() ([]byte, error) { + w, err := deleteParserVersionRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *DeleteParserVersionRequest) UnmarshalJSON(data []byte) error { + var w deleteParserVersionRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := deleteParserVersionRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type deleteRuleRequestWire struct { Name *string `json:"name,omitempty"` } @@ -1431,32 +1785,32 @@ func (v *GetAlertConfigRequest) UnmarshalJSON(data []byte) error { return nil } -type getCommonConfigRequestWire struct { +type getBuiltinParserRequestWire struct { Name *string `json:"name,omitempty"` } -func getCommonConfigRequestToWire(v *GetCommonConfigRequest) (*getCommonConfigRequestWire, error) { +func getBuiltinParserRequestToWire(v *GetBuiltinParserRequest) (*getBuiltinParserRequestWire, error) { if v == nil { return nil, nil } - return &getCommonConfigRequestWire{ + return &getBuiltinParserRequestWire{ Name: v.Name, }, nil } -func getCommonConfigRequestFromWire(w *getCommonConfigRequestWire) (*GetCommonConfigRequest, error) { +func getBuiltinParserRequestFromWire(w *getBuiltinParserRequestWire) (*GetBuiltinParserRequest, error) { if w == nil { return nil, nil } - return &GetCommonConfigRequest{ + return &GetBuiltinParserRequest{ Name: w.Name, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v GetCommonConfigRequest) MarshalJSON() ([]byte, error) { - w, err := getCommonConfigRequestToWire(&v) +func (v GetBuiltinParserRequest) MarshalJSON() ([]byte, error) { + w, err := getBuiltinParserRequestToWire(&v) if err != nil { return nil, err } @@ -1464,12 +1818,12 @@ func (v GetCommonConfigRequest) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *GetCommonConfigRequest) UnmarshalJSON(data []byte) error { - var w getCommonConfigRequestWire +func (v *GetBuiltinParserRequest) UnmarshalJSON(data []byte) error { + var w getBuiltinParserRequestWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := getCommonConfigRequestFromWire(&w) + decoded, err := getBuiltinParserRequestFromWire(&w) if err != nil { return err } @@ -1477,32 +1831,32 @@ func (v *GetCommonConfigRequest) UnmarshalJSON(data []byte) error { return nil } -type getDatasourceConfigRequestWire struct { +type getBuiltinParserVersionRequestWire struct { Name *string `json:"name,omitempty"` } -func getDatasourceConfigRequestToWire(v *GetDatasourceConfigRequest) (*getDatasourceConfigRequestWire, error) { +func getBuiltinParserVersionRequestToWire(v *GetBuiltinParserVersionRequest) (*getBuiltinParserVersionRequestWire, error) { if v == nil { return nil, nil } - return &getDatasourceConfigRequestWire{ + return &getBuiltinParserVersionRequestWire{ Name: v.Name, }, nil } -func getDatasourceConfigRequestFromWire(w *getDatasourceConfigRequestWire) (*GetDatasourceConfigRequest, error) { +func getBuiltinParserVersionRequestFromWire(w *getBuiltinParserVersionRequestWire) (*GetBuiltinParserVersionRequest, error) { if w == nil { return nil, nil } - return &GetDatasourceConfigRequest{ + return &GetBuiltinParserVersionRequest{ Name: w.Name, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v GetDatasourceConfigRequest) MarshalJSON() ([]byte, error) { - w, err := getDatasourceConfigRequestToWire(&v) +func (v GetBuiltinParserVersionRequest) MarshalJSON() ([]byte, error) { + w, err := getBuiltinParserVersionRequestToWire(&v) if err != nil { return nil, err } @@ -1510,12 +1864,12 @@ func (v GetDatasourceConfigRequest) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *GetDatasourceConfigRequest) UnmarshalJSON(data []byte) error { - var w getDatasourceConfigRequestWire +func (v *GetBuiltinParserVersionRequest) UnmarshalJSON(data []byte) error { + var w getBuiltinParserVersionRequestWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := getDatasourceConfigRequestFromWire(&w) + decoded, err := getBuiltinParserVersionRequestFromWire(&w) if err != nil { return err } @@ -1523,15 +1877,107 @@ func (v *GetDatasourceConfigRequest) UnmarshalJSON(data []byte) error { return nil } -type getDatasourceRequestWire struct { +type getCommonConfigRequestWire struct { Name *string `json:"name,omitempty"` } -func getDatasourceRequestToWire(v *GetDatasourceRequest) (*getDatasourceRequestWire, error) { +func getCommonConfigRequestToWire(v *GetCommonConfigRequest) (*getCommonConfigRequestWire, error) { if v == nil { return nil, nil } - return &getDatasourceRequestWire{ + return &getCommonConfigRequestWire{ + Name: v.Name, + }, nil +} + +func getCommonConfigRequestFromWire(w *getCommonConfigRequestWire) (*GetCommonConfigRequest, error) { + if w == nil { + return nil, nil + } + return &GetCommonConfigRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v GetCommonConfigRequest) MarshalJSON() ([]byte, error) { + w, err := getCommonConfigRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetCommonConfigRequest) UnmarshalJSON(data []byte) error { + var w getCommonConfigRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getCommonConfigRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type getDatasourceConfigRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func getDatasourceConfigRequestToWire(v *GetDatasourceConfigRequest) (*getDatasourceConfigRequestWire, error) { + if v == nil { + return nil, nil + } + return &getDatasourceConfigRequestWire{ + Name: v.Name, + }, nil +} + +func getDatasourceConfigRequestFromWire(w *getDatasourceConfigRequestWire) (*GetDatasourceConfigRequest, error) { + if w == nil { + return nil, nil + } + return &GetDatasourceConfigRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v GetDatasourceConfigRequest) MarshalJSON() ([]byte, error) { + w, err := getDatasourceConfigRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetDatasourceConfigRequest) UnmarshalJSON(data []byte) error { + var w getDatasourceConfigRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getDatasourceConfigRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type getDatasourceRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func getDatasourceRequestToWire(v *GetDatasourceRequest) (*getDatasourceRequestWire, error) { + if v == nil { + return nil, nil + } + return &getDatasourceRequestWire{ Name: v.Name, }, nil } @@ -1569,6 +2015,98 @@ func (v *GetDatasourceRequest) UnmarshalJSON(data []byte) error { return nil } +type getParserRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func getParserRequestToWire(v *GetParserRequest) (*getParserRequestWire, error) { + if v == nil { + return nil, nil + } + return &getParserRequestWire{ + Name: v.Name, + }, nil +} + +func getParserRequestFromWire(w *getParserRequestWire) (*GetParserRequest, error) { + if w == nil { + return nil, nil + } + return &GetParserRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v GetParserRequest) MarshalJSON() ([]byte, error) { + w, err := getParserRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetParserRequest) UnmarshalJSON(data []byte) error { + var w getParserRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getParserRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type getParserVersionRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func getParserVersionRequestToWire(v *GetParserVersionRequest) (*getParserVersionRequestWire, error) { + if v == nil { + return nil, nil + } + return &getParserVersionRequestWire{ + Name: v.Name, + }, nil +} + +func getParserVersionRequestFromWire(w *getParserVersionRequestWire) (*GetParserVersionRequest, error) { + if w == nil { + return nil, nil + } + return &GetParserVersionRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v GetParserVersionRequest) MarshalJSON() ([]byte, error) { + w, err := getParserVersionRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetParserVersionRequest) UnmarshalJSON(data []byte) error { + var w getParserVersionRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getParserVersionRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type getRuleConfigRequestWire struct { Name *string `json:"name,omitempty"` } @@ -1593,8 +2131,437 @@ func getRuleConfigRequestFromWire(w *getRuleConfigRequestWire) (*GetRuleConfigRe // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v GetRuleConfigRequest) MarshalJSON() ([]byte, error) { - w, err := getRuleConfigRequestToWire(&v) +func (v GetRuleConfigRequest) MarshalJSON() ([]byte, error) { + w, err := getRuleConfigRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetRuleConfigRequest) UnmarshalJSON(data []byte) error { + var w getRuleConfigRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getRuleConfigRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type getRuleRequestWire struct { + Name *string `json:"name,omitempty"` +} + +func getRuleRequestToWire(v *GetRuleRequest) (*getRuleRequestWire, error) { + if v == nil { + return nil, nil + } + return &getRuleRequestWire{ + Name: v.Name, + }, nil +} + +func getRuleRequestFromWire(w *getRuleRequestWire) (*GetRuleRequest, error) { + if w == nil { + return nil, nil + } + return &GetRuleRequest{ + Name: w.Name, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v GetRuleRequest) MarshalJSON() ([]byte, error) { + w, err := getRuleRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *GetRuleRequest) UnmarshalJSON(data []byte) error { + var w getRuleRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := getRuleRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type jiraExportConfigWire struct { + InstanceUrl *string `json:"instance_url,omitempty"` + ProjectKey *string `json:"project_key,omitempty"` + UserEmail *string `json:"user_email,omitempty"` + DefaultIssueType *string `json:"default_issue_type,omitempty"` + ApiTokenUcSecret *string `json:"api_token_uc_secret,omitempty"` +} + +func jiraExportConfigToWire(v *JiraExportConfig) (*jiraExportConfigWire, error) { + if v == nil { + return nil, nil + } + return &jiraExportConfigWire{ + InstanceUrl: v.InstanceUrl, + ProjectKey: v.ProjectKey, + UserEmail: v.UserEmail, + DefaultIssueType: v.DefaultIssueType, + ApiTokenUcSecret: v.ApiTokenUcSecret, + }, nil +} + +func jiraExportConfigFromWire(w *jiraExportConfigWire) (*JiraExportConfig, error) { + if w == nil { + return nil, nil + } + return &JiraExportConfig{ + InstanceUrl: w.InstanceUrl, + ProjectKey: w.ProjectKey, + UserEmail: w.UserEmail, + DefaultIssueType: w.DefaultIssueType, + ApiTokenUcSecret: w.ApiTokenUcSecret, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v JiraExportConfig) MarshalJSON() ([]byte, error) { + w, err := jiraExportConfigToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *JiraExportConfig) UnmarshalJSON(data []byte) error { + var w jiraExportConfigWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := jiraExportConfigFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type jsonArrayEnvelopeWire struct { + Path *string `json:"path,omitempty"` +} + +func jsonArrayEnvelopeToWire(v *JsonArrayEnvelope) (*jsonArrayEnvelopeWire, error) { + if v == nil { + return nil, nil + } + return &jsonArrayEnvelopeWire{ + Path: v.Path, + }, nil +} + +func jsonArrayEnvelopeFromWire(w *jsonArrayEnvelopeWire) (*JsonArrayEnvelope, error) { + if w == nil { + return nil, nil + } + return &JsonArrayEnvelope{ + Path: w.Path, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v JsonArrayEnvelope) MarshalJSON() ([]byte, error) { + w, err := jsonArrayEnvelopeToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *JsonArrayEnvelope) UnmarshalJSON(data []byte) error { + var w jsonArrayEnvelopeWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := jsonArrayEnvelopeFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type listBuiltinParserVersionsRequestWire struct { + Parent *string `json:"parent,omitempty"` + PageSize *int `json:"page_size,omitempty"` + PageToken *string `json:"page_token,omitempty"` +} + +func listBuiltinParserVersionsRequestToWire(v *ListBuiltinParserVersionsRequest) (*listBuiltinParserVersionsRequestWire, error) { + if v == nil { + return nil, nil + } + return &listBuiltinParserVersionsRequestWire{ + Parent: v.Parent, + PageSize: v.PageSize, + PageToken: v.PageToken, + }, nil +} + +func listBuiltinParserVersionsRequestFromWire(w *listBuiltinParserVersionsRequestWire) (*ListBuiltinParserVersionsRequest, error) { + if w == nil { + return nil, nil + } + return &ListBuiltinParserVersionsRequest{ + Parent: w.Parent, + PageSize: w.PageSize, + PageToken: w.PageToken, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ListBuiltinParserVersionsRequest) MarshalJSON() ([]byte, error) { + w, err := listBuiltinParserVersionsRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ListBuiltinParserVersionsRequest) UnmarshalJSON(data []byte) error { + var w listBuiltinParserVersionsRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := listBuiltinParserVersionsRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type listBuiltinParserVersionsResponseWire struct { + BuiltinParserVersions []builtinParserVersionWire `json:"builtin_parser_versions,omitempty"` + NextPageToken *string `json:"next_page_token,omitempty"` +} + +func listBuiltinParserVersionsResponseToWire(v *ListBuiltinParserVersionsResponse) (*listBuiltinParserVersionsResponseWire, error) { + if v == nil { + return nil, nil + } + builtinParserVersionsWireValue, err := convertSlice(v.BuiltinParserVersions, builtinParserVersionToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListBuiltinParserVersionsResponse.BuiltinParserVersions", err) + } + return &listBuiltinParserVersionsResponseWire{ + BuiltinParserVersions: builtinParserVersionsWireValue, + NextPageToken: v.NextPageToken, + }, nil +} + +func listBuiltinParserVersionsResponseFromWire(w *listBuiltinParserVersionsResponseWire) (*ListBuiltinParserVersionsResponse, error) { + if w == nil { + return nil, nil + } + builtinParserVersionsPublicValue, err := convertSlice(w.BuiltinParserVersions, builtinParserVersionFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListBuiltinParserVersionsResponse.BuiltinParserVersions", err) + } + return &ListBuiltinParserVersionsResponse{ + BuiltinParserVersions: builtinParserVersionsPublicValue, + NextPageToken: w.NextPageToken, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ListBuiltinParserVersionsResponse) MarshalJSON() ([]byte, error) { + w, err := listBuiltinParserVersionsResponseToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ListBuiltinParserVersionsResponse) UnmarshalJSON(data []byte) error { + var w listBuiltinParserVersionsResponseWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := listBuiltinParserVersionsResponseFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type listBuiltinParsersRequestWire struct { + PageSize *int `json:"page_size,omitempty"` + PageToken *string `json:"page_token,omitempty"` + Filter *string `json:"filter,omitempty"` +} + +func listBuiltinParsersRequestToWire(v *ListBuiltinParsersRequest) (*listBuiltinParsersRequestWire, error) { + if v == nil { + return nil, nil + } + return &listBuiltinParsersRequestWire{ + PageSize: v.PageSize, + PageToken: v.PageToken, + Filter: v.Filter, + }, nil +} + +func listBuiltinParsersRequestFromWire(w *listBuiltinParsersRequestWire) (*ListBuiltinParsersRequest, error) { + if w == nil { + return nil, nil + } + return &ListBuiltinParsersRequest{ + PageSize: w.PageSize, + PageToken: w.PageToken, + Filter: w.Filter, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ListBuiltinParsersRequest) MarshalJSON() ([]byte, error) { + w, err := listBuiltinParsersRequestToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ListBuiltinParsersRequest) UnmarshalJSON(data []byte) error { + var w listBuiltinParsersRequestWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := listBuiltinParsersRequestFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type listBuiltinParsersResponseWire struct { + BuiltinParsers []builtinParserWire `json:"builtin_parsers,omitempty"` + NextPageToken *string `json:"next_page_token,omitempty"` +} + +func listBuiltinParsersResponseToWire(v *ListBuiltinParsersResponse) (*listBuiltinParsersResponseWire, error) { + if v == nil { + return nil, nil + } + builtinParsersWireValue, err := convertSlice(v.BuiltinParsers, builtinParserToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListBuiltinParsersResponse.BuiltinParsers", err) + } + return &listBuiltinParsersResponseWire{ + BuiltinParsers: builtinParsersWireValue, + NextPageToken: v.NextPageToken, + }, nil +} + +func listBuiltinParsersResponseFromWire(w *listBuiltinParsersResponseWire) (*ListBuiltinParsersResponse, error) { + if w == nil { + return nil, nil + } + builtinParsersPublicValue, err := convertSlice(w.BuiltinParsers, builtinParserFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListBuiltinParsersResponse.BuiltinParsers", err) + } + return &ListBuiltinParsersResponse{ + BuiltinParsers: builtinParsersPublicValue, + NextPageToken: w.NextPageToken, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ListBuiltinParsersResponse) MarshalJSON() ([]byte, error) { + w, err := listBuiltinParsersResponseToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ListBuiltinParsersResponse) UnmarshalJSON(data []byte) error { + var w listBuiltinParsersResponseWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := listBuiltinParsersResponseFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type listDatasourcesRequestWire struct { + PageSize *int `json:"page_size,omitempty"` + PageToken *string `json:"page_token,omitempty"` + Filter *string `json:"filter,omitempty"` + CatalogName *string `json:"catalog_name,omitempty"` + SchemaName *string `json:"schema_name,omitempty"` + View DatasourceView `json:"view,omitempty"` +} + +func listDatasourcesRequestToWire(v *ListDatasourcesRequest) (*listDatasourcesRequestWire, error) { + if v == nil { + return nil, nil + } + return &listDatasourcesRequestWire{ + PageSize: v.PageSize, + PageToken: v.PageToken, + Filter: v.Filter, + CatalogName: v.CatalogName, + SchemaName: v.SchemaName, + View: v.View, + }, nil +} + +func listDatasourcesRequestFromWire(w *listDatasourcesRequestWire) (*ListDatasourcesRequest, error) { + if w == nil { + return nil, nil + } + return &ListDatasourcesRequest{ + PageSize: w.PageSize, + PageToken: w.PageToken, + Filter: w.Filter, + CatalogName: w.CatalogName, + SchemaName: w.SchemaName, + View: w.View, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ListDatasourcesRequest) MarshalJSON() ([]byte, error) { + w, err := listDatasourcesRequestToWire(&v) if err != nil { return nil, err } @@ -1602,12 +2569,12 @@ func (v GetRuleConfigRequest) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *GetRuleConfigRequest) UnmarshalJSON(data []byte) error { - var w getRuleConfigRequestWire +func (v *ListDatasourcesRequest) UnmarshalJSON(data []byte) error { + var w listDatasourcesRequestWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := getRuleConfigRequestFromWire(&w) + decoded, err := listDatasourcesRequestFromWire(&w) if err != nil { return err } @@ -1615,32 +2582,43 @@ func (v *GetRuleConfigRequest) UnmarshalJSON(data []byte) error { return nil } -type getRuleRequestWire struct { - Name *string `json:"name,omitempty"` +type listDatasourcesResponseWire struct { + Datasources []datasourceWire `json:"datasources,omitempty"` + NextPageToken *string `json:"next_page_token,omitempty"` } -func getRuleRequestToWire(v *GetRuleRequest) (*getRuleRequestWire, error) { +func listDatasourcesResponseToWire(v *ListDatasourcesResponse) (*listDatasourcesResponseWire, error) { if v == nil { return nil, nil } - return &getRuleRequestWire{ - Name: v.Name, + datasourcesWireValue, err := convertSlice(v.Datasources, datasourceToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListDatasourcesResponse.Datasources", err) + } + return &listDatasourcesResponseWire{ + Datasources: datasourcesWireValue, + NextPageToken: v.NextPageToken, }, nil } -func getRuleRequestFromWire(w *getRuleRequestWire) (*GetRuleRequest, error) { +func listDatasourcesResponseFromWire(w *listDatasourcesResponseWire) (*ListDatasourcesResponse, error) { if w == nil { return nil, nil } - return &GetRuleRequest{ - Name: w.Name, + datasourcesPublicValue, err := convertSlice(w.Datasources, datasourceFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListDatasourcesResponse.Datasources", err) + } + return &ListDatasourcesResponse{ + Datasources: datasourcesPublicValue, + NextPageToken: w.NextPageToken, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v GetRuleRequest) MarshalJSON() ([]byte, error) { - w, err := getRuleRequestToWire(&v) +func (v ListDatasourcesResponse) MarshalJSON() ([]byte, error) { + w, err := listDatasourcesResponseToWire(&v) if err != nil { return nil, err } @@ -1648,12 +2626,12 @@ func (v GetRuleRequest) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *GetRuleRequest) UnmarshalJSON(data []byte) error { - var w getRuleRequestWire +func (v *ListDatasourcesResponse) UnmarshalJSON(data []byte) error { + var w listDatasourcesResponseWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := getRuleRequestFromWire(&w) + decoded, err := listDatasourcesResponseFromWire(&w) if err != nil { return err } @@ -1661,44 +2639,38 @@ func (v *GetRuleRequest) UnmarshalJSON(data []byte) error { return nil } -type jiraExportConfigWire struct { - InstanceUrl *string `json:"instance_url,omitempty"` - ProjectKey *string `json:"project_key,omitempty"` - UserEmail *string `json:"user_email,omitempty"` - DefaultIssueType *string `json:"default_issue_type,omitempty"` - ApiTokenUcSecret *string `json:"api_token_uc_secret,omitempty"` +type listParserVersionsRequestWire struct { + Parent *string `json:"parent,omitempty"` + PageSize *int `json:"page_size,omitempty"` + PageToken *string `json:"page_token,omitempty"` } -func jiraExportConfigToWire(v *JiraExportConfig) (*jiraExportConfigWire, error) { +func listParserVersionsRequestToWire(v *ListParserVersionsRequest) (*listParserVersionsRequestWire, error) { if v == nil { return nil, nil } - return &jiraExportConfigWire{ - InstanceUrl: v.InstanceUrl, - ProjectKey: v.ProjectKey, - UserEmail: v.UserEmail, - DefaultIssueType: v.DefaultIssueType, - ApiTokenUcSecret: v.ApiTokenUcSecret, + return &listParserVersionsRequestWire{ + Parent: v.Parent, + PageSize: v.PageSize, + PageToken: v.PageToken, }, nil } -func jiraExportConfigFromWire(w *jiraExportConfigWire) (*JiraExportConfig, error) { +func listParserVersionsRequestFromWire(w *listParserVersionsRequestWire) (*ListParserVersionsRequest, error) { if w == nil { return nil, nil } - return &JiraExportConfig{ - InstanceUrl: w.InstanceUrl, - ProjectKey: w.ProjectKey, - UserEmail: w.UserEmail, - DefaultIssueType: w.DefaultIssueType, - ApiTokenUcSecret: w.ApiTokenUcSecret, + return &ListParserVersionsRequest{ + Parent: w.Parent, + PageSize: w.PageSize, + PageToken: w.PageToken, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v JiraExportConfig) MarshalJSON() ([]byte, error) { - w, err := jiraExportConfigToWire(&v) +func (v ListParserVersionsRequest) MarshalJSON() ([]byte, error) { + w, err := listParserVersionsRequestToWire(&v) if err != nil { return nil, err } @@ -1706,12 +2678,12 @@ func (v JiraExportConfig) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *JiraExportConfig) UnmarshalJSON(data []byte) error { - var w jiraExportConfigWire +func (v *ListParserVersionsRequest) UnmarshalJSON(data []byte) error { + var w listParserVersionsRequestWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := jiraExportConfigFromWire(&w) + decoded, err := listParserVersionsRequestFromWire(&w) if err != nil { return err } @@ -1719,32 +2691,43 @@ func (v *JiraExportConfig) UnmarshalJSON(data []byte) error { return nil } -type jsonArrayEnvelopeWire struct { - Path *string `json:"path,omitempty"` +type listParserVersionsResponseWire struct { + ParserVersions []parserVersionWire `json:"parser_versions,omitempty"` + NextPageToken *string `json:"next_page_token,omitempty"` } -func jsonArrayEnvelopeToWire(v *JsonArrayEnvelope) (*jsonArrayEnvelopeWire, error) { +func listParserVersionsResponseToWire(v *ListParserVersionsResponse) (*listParserVersionsResponseWire, error) { if v == nil { return nil, nil } - return &jsonArrayEnvelopeWire{ - Path: v.Path, + parserVersionsWireValue, err := convertSlice(v.ParserVersions, parserVersionToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListParserVersionsResponse.ParserVersions", err) + } + return &listParserVersionsResponseWire{ + ParserVersions: parserVersionsWireValue, + NextPageToken: v.NextPageToken, }, nil } -func jsonArrayEnvelopeFromWire(w *jsonArrayEnvelopeWire) (*JsonArrayEnvelope, error) { +func listParserVersionsResponseFromWire(w *listParserVersionsResponseWire) (*ListParserVersionsResponse, error) { if w == nil { return nil, nil } - return &JsonArrayEnvelope{ - Path: w.Path, + parserVersionsPublicValue, err := convertSlice(w.ParserVersions, parserVersionFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListParserVersionsResponse.ParserVersions", err) + } + return &ListParserVersionsResponse{ + ParserVersions: parserVersionsPublicValue, + NextPageToken: w.NextPageToken, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v JsonArrayEnvelope) MarshalJSON() ([]byte, error) { - w, err := jsonArrayEnvelopeToWire(&v) +func (v ListParserVersionsResponse) MarshalJSON() ([]byte, error) { + w, err := listParserVersionsResponseToWire(&v) if err != nil { return nil, err } @@ -1752,12 +2735,12 @@ func (v JsonArrayEnvelope) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *JsonArrayEnvelope) UnmarshalJSON(data []byte) error { - var w jsonArrayEnvelopeWire +func (v *ListParserVersionsResponse) UnmarshalJSON(data []byte) error { + var w listParserVersionsResponseWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := jsonArrayEnvelopeFromWire(&w) + decoded, err := listParserVersionsResponseFromWire(&w) if err != nil { return err } @@ -1765,47 +2748,44 @@ func (v *JsonArrayEnvelope) UnmarshalJSON(data []byte) error { return nil } -type listDatasourcesRequestWire struct { - PageSize *int `json:"page_size,omitempty"` - PageToken *string `json:"page_token,omitempty"` - Filter *string `json:"filter,omitempty"` - CatalogName *string `json:"catalog_name,omitempty"` - SchemaName *string `json:"schema_name,omitempty"` - View DatasourceView `json:"view,omitempty"` +type listParsersRequestWire struct { + PageSize *int `json:"page_size,omitempty"` + PageToken *string `json:"page_token,omitempty"` + Filter *string `json:"filter,omitempty"` + CatalogName *string `json:"catalog_name,omitempty"` + SchemaName *string `json:"schema_name,omitempty"` } -func listDatasourcesRequestToWire(v *ListDatasourcesRequest) (*listDatasourcesRequestWire, error) { +func listParsersRequestToWire(v *ListParsersRequest) (*listParsersRequestWire, error) { if v == nil { return nil, nil } - return &listDatasourcesRequestWire{ + return &listParsersRequestWire{ PageSize: v.PageSize, PageToken: v.PageToken, Filter: v.Filter, CatalogName: v.CatalogName, SchemaName: v.SchemaName, - View: v.View, }, nil } -func listDatasourcesRequestFromWire(w *listDatasourcesRequestWire) (*ListDatasourcesRequest, error) { +func listParsersRequestFromWire(w *listParsersRequestWire) (*ListParsersRequest, error) { if w == nil { return nil, nil } - return &ListDatasourcesRequest{ + return &ListParsersRequest{ PageSize: w.PageSize, PageToken: w.PageToken, Filter: w.Filter, CatalogName: w.CatalogName, SchemaName: w.SchemaName, - View: w.View, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v ListDatasourcesRequest) MarshalJSON() ([]byte, error) { - w, err := listDatasourcesRequestToWire(&v) +func (v ListParsersRequest) MarshalJSON() ([]byte, error) { + w, err := listParsersRequestToWire(&v) if err != nil { return nil, err } @@ -1813,12 +2793,12 @@ func (v ListDatasourcesRequest) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *ListDatasourcesRequest) UnmarshalJSON(data []byte) error { - var w listDatasourcesRequestWire +func (v *ListParsersRequest) UnmarshalJSON(data []byte) error { + var w listParsersRequestWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := listDatasourcesRequestFromWire(&w) + decoded, err := listParsersRequestFromWire(&w) if err != nil { return err } @@ -1826,43 +2806,43 @@ func (v *ListDatasourcesRequest) UnmarshalJSON(data []byte) error { return nil } -type listDatasourcesResponseWire struct { - Datasources []datasourceWire `json:"datasources,omitempty"` - NextPageToken *string `json:"next_page_token,omitempty"` +type listParsersResponseWire struct { + Parsers []parserWire `json:"parsers,omitempty"` + NextPageToken *string `json:"next_page_token,omitempty"` } -func listDatasourcesResponseToWire(v *ListDatasourcesResponse) (*listDatasourcesResponseWire, error) { +func listParsersResponseToWire(v *ListParsersResponse) (*listParsersResponseWire, error) { if v == nil { return nil, nil } - datasourcesWireValue, err := convertSlice(v.Datasources, datasourceToWire) + parsersWireValue, err := convertSlice(v.Parsers, parserToWire) if err != nil { - return nil, fmt.Errorf("%s: %w", "ListDatasourcesResponse.Datasources", err) + return nil, fmt.Errorf("%s: %w", "ListParsersResponse.Parsers", err) } - return &listDatasourcesResponseWire{ - Datasources: datasourcesWireValue, + return &listParsersResponseWire{ + Parsers: parsersWireValue, NextPageToken: v.NextPageToken, }, nil } -func listDatasourcesResponseFromWire(w *listDatasourcesResponseWire) (*ListDatasourcesResponse, error) { +func listParsersResponseFromWire(w *listParsersResponseWire) (*ListParsersResponse, error) { if w == nil { return nil, nil } - datasourcesPublicValue, err := convertSlice(w.Datasources, datasourceFromWire) + parsersPublicValue, err := convertSlice(w.Parsers, parserFromWire) if err != nil { - return nil, fmt.Errorf("%s: %w", "ListDatasourcesResponse.Datasources", err) + return nil, fmt.Errorf("%s: %w", "ListParsersResponse.Parsers", err) } - return &ListDatasourcesResponse{ - Datasources: datasourcesPublicValue, + return &ListParsersResponse{ + Parsers: parsersPublicValue, NextPageToken: w.NextPageToken, }, nil } // MarshalJSON returns the JSON form that the API uses for this message, with // fields that a request sends in the URL path or query as regular fields. -func (v ListDatasourcesResponse) MarshalJSON() ([]byte, error) { - w, err := listDatasourcesResponseToWire(&v) +func (v ListParsersResponse) MarshalJSON() ([]byte, error) { + w, err := listParsersResponseToWire(&v) if err != nil { return nil, err } @@ -1870,12 +2850,12 @@ func (v ListDatasourcesResponse) MarshalJSON() ([]byte, error) { } // UnmarshalJSON sets the message from the JSON that MarshalJSON writes. -func (v *ListDatasourcesResponse) UnmarshalJSON(data []byte) error { - var w listDatasourcesResponseWire +func (v *ListParsersResponse) UnmarshalJSON(data []byte) error { + var w listParsersResponseWire if err := json.Unmarshal(data, &w); err != nil { return err } - decoded, err := listDatasourcesResponseFromWire(&w) + decoded, err := listParsersResponseFromWire(&w) if err != nil { return err } @@ -2001,6 +2981,112 @@ func (v *ListRulesResponse) UnmarshalJSON(data []byte) error { return nil } +type parserWire struct { + Name *string `json:"name,omitempty"` + ParserId *string `json:"parser_id,omitempty"` + CatalogName *string `json:"catalog_name,omitempty"` + SchemaName *string `json:"schema_name,omitempty"` + ParserName *string `json:"parser_name,omitempty"` + LatestVersion *wireInt64 `json:"latest_version,omitempty"` + CreateTime *types.Time `json:"create_time,omitempty"` + CreatedBy *principalWire `json:"created_by,omitempty"` + Owner *principalWire `json:"owner,omitempty"` + EffectiveCatalogName *string `json:"effective_catalog_name,omitempty"` + EffectiveSchemaName *string `json:"effective_schema_name,omitempty"` + Spec *string `json:"spec,omitempty"` + Description *string `json:"description,omitempty"` +} + +func parserToWire(v *Parser) (*parserWire, error) { + if v == nil { + return nil, nil + } + latestVersionWireValue, err := int64ToWire(v.LatestVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.LatestVersion", err) + } + createdByWireValue, err := principalToWire(v.CreatedBy) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.CreatedBy", err) + } + ownerWireValue, err := principalToWire(v.Owner) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.Owner", err) + } + return &parserWire{ + Name: v.Name, + ParserId: v.ParserId, + CatalogName: v.CatalogName, + SchemaName: v.SchemaName, + ParserName: v.ParserName, + LatestVersion: latestVersionWireValue, + CreateTime: v.CreateTime, + CreatedBy: createdByWireValue, + Owner: ownerWireValue, + EffectiveCatalogName: v.EffectiveCatalogName, + EffectiveSchemaName: v.EffectiveSchemaName, + Spec: v.Spec, + Description: v.Description, + }, nil +} + +func parserFromWire(w *parserWire) (*Parser, error) { + if w == nil { + return nil, nil + } + latestVersionPublicValue, err := int64FromWire(w.LatestVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.LatestVersion", err) + } + createdByPublicValue, err := principalFromWire(w.CreatedBy) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.CreatedBy", err) + } + ownerPublicValue, err := principalFromWire(w.Owner) + if err != nil { + return nil, fmt.Errorf("%s: %w", "Parser.Owner", err) + } + return &Parser{ + Name: w.Name, + ParserId: w.ParserId, + CatalogName: w.CatalogName, + SchemaName: w.SchemaName, + ParserName: w.ParserName, + LatestVersion: latestVersionPublicValue, + CreateTime: w.CreateTime, + CreatedBy: createdByPublicValue, + Owner: ownerPublicValue, + EffectiveCatalogName: w.EffectiveCatalogName, + EffectiveSchemaName: w.EffectiveSchemaName, + Spec: w.Spec, + Description: w.Description, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v Parser) MarshalJSON() ([]byte, error) { + w, err := parserToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *Parser) UnmarshalJSON(data []byte) error { + var w parserWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := parserFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type parserRefWire struct { Id *string `json:"id,omitempty"` Version *string `json:"version,omitempty"` @@ -2072,6 +3158,135 @@ func (v *ParserRef) UnmarshalJSON(data []byte) error { return nil } +type parserTagWire struct { + Key *string `json:"key,omitempty"` + Value *string `json:"value,omitempty"` +} + +func parserTagToWire(v *ParserTag) (*parserTagWire, error) { + if v == nil { + return nil, nil + } + return &parserTagWire{ + Key: v.Key, + Value: v.Value, + }, nil +} + +func parserTagFromWire(w *parserTagWire) (*ParserTag, error) { + if w == nil { + return nil, nil + } + return &ParserTag{ + Key: w.Key, + Value: w.Value, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ParserTag) MarshalJSON() ([]byte, error) { + w, err := parserTagToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ParserTag) UnmarshalJSON(data []byte) error { + var w parserTagWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := parserTagFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + +type parserVersionWire struct { + Name *string `json:"name,omitempty"` + Version *wireInt64 `json:"version,omitempty"` + Spec *string `json:"spec,omitempty"` + CreateTime *types.Time `json:"create_time,omitempty"` + CreatedBy *principalWire `json:"created_by,omitempty"` + Description *string `json:"description,omitempty"` + ParserName *string `json:"parser_name,omitempty"` +} + +func parserVersionToWire(v *ParserVersion) (*parserVersionWire, error) { + if v == nil { + return nil, nil + } + versionWireValue, err := int64ToWire(v.Version) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ParserVersion.Version", err) + } + createdByWireValue, err := principalToWire(v.CreatedBy) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ParserVersion.CreatedBy", err) + } + return &parserVersionWire{ + Name: v.Name, + Version: versionWireValue, + Spec: v.Spec, + CreateTime: v.CreateTime, + CreatedBy: createdByWireValue, + Description: v.Description, + ParserName: v.ParserName, + }, nil +} + +func parserVersionFromWire(w *parserVersionWire) (*ParserVersion, error) { + if w == nil { + return nil, nil + } + versionPublicValue, err := int64FromWire(w.Version) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ParserVersion.Version", err) + } + createdByPublicValue, err := principalFromWire(w.CreatedBy) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ParserVersion.CreatedBy", err) + } + return &ParserVersion{ + Name: w.Name, + Version: versionPublicValue, + Spec: w.Spec, + CreateTime: w.CreateTime, + CreatedBy: createdByPublicValue, + Description: w.Description, + ParserName: w.ParserName, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v ParserVersion) MarshalJSON() ([]byte, error) { + w, err := parserVersionToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *ParserVersion) UnmarshalJSON(data []byte) error { + var w parserVersionWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := parserVersionFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type principalWire struct { Id *string `json:"id,omitempty"` PrincipalType PrincipalType `json:"principal_type,omitempty"` diff --git a/modelregistry/CHANGELOG.md b/modelregistry/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/modelregistry/CHANGELOG.md +++ b/modelregistry/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/modelregistry/internal/version.go b/modelregistry/internal/version.go index a2594dcc..0b526c39 100644 --- a/modelregistry/internal/version.go +++ b/modelregistry/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-modelregistry" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/modelregistry/v1/client.go b/modelregistry/v1/client.go index c20bfa09..20328c31 100644 --- a/modelregistry/v1/client.go +++ b/modelregistry/v1/client.go @@ -2204,7 +2204,7 @@ func (c *internalClient) SetRegisteredModelTag(ctx context.Context, req SetRegis return resp, nil } -// Updates the model version. +// Updates the description of a model version. func (c *internalClient) UpdateModelVersion(ctx context.Context, req UpdateModelVersionRequest, opts ...call.Option) (*UpdateModelVersionResponse, error) { wireReq, err := updateModelVersionRequestToWire(&req) if err != nil { diff --git a/modelregistry/v1/model.go b/modelregistry/v1/model.go index d51b32a7..36bafc03 100644 --- a/modelregistry/v1/model.go +++ b/modelregistry/v1/model.go @@ -250,24 +250,29 @@ type CreateCommentResponse struct { } type CreateModelVersionRequest struct { - // Register model under this name + // Name of the existing registered model to create the version under. Name *string - // URI indicating the location of the model artifacts. + // URI of the model artifacts to register, such as the artifact path of an + // MLflow run or a `models://` URI of an existing `READY` + // model version. The registry copies the artifacts asynchronously, so the new + // version starts in `PENDING_REGISTRATION` status. Source *string - // MLflow run ID for correlation, if `source` was generated by an experiment run - // in MLflow tracking server + // ID of the MLflow run that produced the model artifacts, if `source` comes + // from a run in the MLflow tracking server. The ID is at most 32 characters + // long. RunId *string - // Additional metadata for model version. + // Tags to set on the new model version. Tags []ModelVersionTag // MLflow run link - this is the exact link of the run that generated this model // version, potentially hosted at another instance of MLflow. RunLink *string - // Optional description for model version. + // Description of the model version. Description *string } type CreateModelVersionResponse struct { - // Return new version number generated for this model in registry. + // The new model version, including the version number that the registry + // assigned. ModelVersion *ModelVersion } @@ -582,34 +587,43 @@ type ListTransitionResponse struct { Requests []Activity } +// A version of a registered model in the Workspace Model Registry. Each version +// points to the model artifacts it was registered from and moves through the +// stages `None`, `Staging`, `Production` and `Archived`.. type ModelVersion struct { - // Unique name of the model + // Name of the registered model that the version belongs to. Name *string - // Model's version number. + // Version number of the model version, as a string. Version numbers start at 1 + // for each registered model. Version *string - // Timestamp recorded when this `model_version` was created. + // Time when the model version was created, as a Unix timestamp in milliseconds. CreationTimestamp *int64 - // Timestamp recorded when metadata for this `model_version` was last updated. + // Time when the metadata of the model version was last updated, as a Unix + // timestamp in milliseconds. LastUpdatedTimestamp *int64 // User that created this `model_version`. UserId *string - // Current stage for this `model_version`. + // Current stage of the model version. Valid values are `None`, `Staging`, + // `Production` and `Archived`. A new model version starts in `None`. CurrentStage *string - // Description of this `model_version`. + // User-provided description of the model version. Description *string - // URI indicating the location of the source model artifacts, used when creating - // `model_version` + // URI of the model artifacts that the model version was registered from, as + // given in the create request. Source *string - // MLflow run ID used when creating `model_version`, if `source` was generated - // by an experiment run stored in MLflow tracking server. + // ID of the MLflow run that produced the model artifacts, if `source` comes + // from a run in the MLflow tracking server. RunId *string - // Current status of `model_version` + // Registration status of the model version. The version can be used once the + // status is `READY`. Status ModelVersionStatus - // Details on current `status`, if it is pending or failed. + // Details on the current `status`, such as the reason why registration failed. StatusMessage *string - // Tags: Additional metadata key-value pairs for this `model_version`. + // Key-value tags attached to the model version. Tags []ModelVersionTag - // Run Link: Direct link to the run that generated this version + // URL of the MLflow run that generated the model version, as given in the + // create request. It is typically set when the run is hosted on a different + // tracking server than the registry. RunLink *string } @@ -623,7 +637,9 @@ type ModelVersionDatabricks struct { // Time of the object at last update, as a Unix timestamp in milliseconds. LastUpdatedTimestamp *int64 // The username of the user that created the object. - UserId *string + UserId *string + // Current stage of the model version. Valid values are `None`, `Staging`, + // `Production` and `Archived`. CurrentStage *string // User-specified description for the object. Description *string @@ -632,14 +648,16 @@ type ModelVersionDatabricks struct { Source *string // Unique identifier for the MLflow tracking run associated with the source // model artifacts. - RunId *string + RunId *string + // Registration status of the model version. The version can be used once the + // status is `READY`. Status ModelVersionStatus // Details on the current status, for example why registration failed. StatusMessage *string - // Open requests for this `model_versions`. Gap in sequence number is - // intentional and is done in order to match field sequence numbers of - // `ModelVersion` proto message - OpenRequests []Activity + // Open stage transition requests for the model version. + OpenRequests []Activity + // Permission level of the requesting user on the registered model that the + // version belongs to. PermissionLevel PermissionLevel // Array of tags that are associated with the model version. Tags []ModelVersionTag @@ -775,22 +793,25 @@ type RenameRegisteredModelResponse struct { } type SearchModelVersionsRequest struct { - // String filter condition, like "name='my-model-name'". Must be a single - // boolean condition, with string values wrapped in single quotes. + // Filter condition on `name`, `run_id`, `source` or `tags.`. The filter + // must constrain at least one of `name`, `run_id` or `source`. Conditions use + // `=`, `!=`, `LIKE` or `ILIKE`, and `run_id` also supports `IN`. String values + // are wrapped in single quotes, and several conditions can be combined with + // `AND`. Filter *string // Maximum number of models desired. Max threshold is 10K. MaxResults *int64 - // List of columns to be ordered by including model name, version, stage with an - // optional "DESC" or "ASC" annotation, where "ASC" is the default. Tiebreaks - // are done by latest stage transition timestamp, followed by name ASC, followed - // by version DESC. + // Columns to order the results by, each given as `name`, `version`, + // `current_stage` or `last_updated_timestamp` with an optional `ASC` or `DESC` + // suffix, where `ASC` is the default. Tiebreaks are done by latest stage + // transition timestamp, followed by name ASC, followed by version DESC. OrderBy []string // Pagination token to go to next page based on previous search query. PageToken *string } type SearchModelVersionsResponse struct { - // Models that match the search criteria + // Model versions that match the search filter. ModelVersions []ModelVersion // Pagination token to request next page of models for the same search query. NextPageToken *string @@ -945,12 +966,12 @@ type UpdateModelVersionRequest struct { Name *string // Model version number Version *string - // If provided, updates the description for this `registered_model`. + // New description for the model version. Description *string } type UpdateModelVersionResponse struct { - // Return new version number generated for this model in registry. + // The updated model version. ModelVersion *ModelVersion } diff --git a/pipelines/CHANGELOG.md b/pipelines/CHANGELOG.md index d62f2bcd..dbf4e451 100644 --- a/pipelines/CHANGELOG.md +++ b/pipelines/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.14.0 (2026-11-17) + ## Release v0.13.0 (2026-11-16) ## Release v0.12.0 (2026-11-12) diff --git a/pipelines/internal/version.go b/pipelines/internal/version.go index ba56ff3f..c7377d35 100644 --- a/pipelines/internal/version.go +++ b/pipelines/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-pipelines" -const Version = "0.13.0" +const Version = "0.14.0" diff --git a/pipelines/v2/model.go b/pipelines/v2/model.go index e47eac27..951cebf8 100644 --- a/pipelines/v2/model.go +++ b/pipelines/v2/model.go @@ -86,6 +86,7 @@ const ( IngestionSourceType_Jira IngestionSourceType = "JIRA" IngestionSourceType_Confluence IngestionSourceType = "CONFLUENCE" IngestionSourceType_MetaMarketing IngestionSourceType = "META_MARKETING" + IngestionSourceType_GoogleAds IngestionSourceType = "GOOGLE_ADS" IngestionSourceType_TiktokAds IngestionSourceType = "TIKTOK_ADS" IngestionSourceType_Zendesk IngestionSourceType = "ZENDESK" IngestionSourceType_Smartsheet IngestionSourceType = "SMARTSHEET" diff --git a/postgres/CHANGELOG.md b/postgres/CHANGELOG.md index 6686b310..9fbe12a5 100644 --- a/postgres/CHANGELOG.md +++ b/postgres/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.13.0 (2026-11-17) + ## Release v0.12.0 (2026-11-16) ## Release v0.11.0 (2026-11-12) diff --git a/postgres/internal/version.go b/postgres/internal/version.go index 05de5a4e..78b37f0c 100644 --- a/postgres/internal/version.go +++ b/postgres/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-postgres" -const Version = "0.12.0" +const Version = "0.13.0" diff --git a/postgres/v1/client.go b/postgres/v1/client.go index ae9c354a..30714010 100644 --- a/postgres/v1/client.go +++ b/postgres/v1/client.go @@ -2023,7 +2023,8 @@ func (o *CreateRoleOperation) Wait(ctx context.Context, opts ...lro.Option) (*Ro } // Creates a snapshot, an immutable point-in-time copy of a branch's data, -// within the project. +// within the project. Returns a long-running operation. Poll it until done is +// true; its response holds the new snapshot. func (c *internalClient) createSnapshotBase(ctx context.Context, req CreateSnapshotRequest, opts ...call.Option) (*Operation, error) { wireReq, err := createSnapshotRequestToWire(&req) if err != nil { @@ -2100,7 +2101,8 @@ func (c *internalClient) createSnapshotBase(ctx context.Context, req CreateSnaps } // Creates a snapshot, an immutable point-in-time copy of a branch's data, -// within the project. +// within the project. Returns a long-running operation. Poll it until done is +// true; its response holds the new snapshot. func (c *internalClient) CreateSnapshot(ctx context.Context, req CreateSnapshotRequest, opts ...call.Option) (*CreateSnapshotOperation, error) { operation, err := c.createSnapshotBase(ctx, req, opts...) if err != nil { @@ -2997,7 +2999,7 @@ func (o *DeleteDataApiOperation) Wait(ctx context.Context, opts ...lro.Option) e return nil } -// Delete a Database. +// Deletes the specified Postgres database. func (c *internalClient) deleteDatabaseBase(ctx context.Context, req DeleteDatabaseRequest, opts ...call.Option) (*Operation, error) { headers := http.Header{} @@ -3059,7 +3061,7 @@ func (c *internalClient) deleteDatabaseBase(ctx context.Context, req DeleteDatab return resp, nil } -// Delete a Database. +// Deletes the specified Postgres database. func (c *internalClient) DeleteDatabase(ctx context.Context, req DeleteDatabaseRequest, opts ...call.Option) (*DeleteDatabaseOperation, error) { operation, err := c.deleteDatabaseBase(ctx, req, opts...) if err != nil { @@ -3791,7 +3793,8 @@ func (o *DeleteRoleOperation) Wait(ctx context.Context, opts ...lro.Option) erro return nil } -// Deletes the specified snapshot. +// Deletes the specified snapshot. A deleted snapshot cannot be recovered. +// Returns a long-running operation. Poll it until done is true. func (c *internalClient) deleteSnapshotBase(ctx context.Context, req DeleteSnapshotRequest, opts ...call.Option) (*Operation, error) { headers := http.Header{} @@ -3853,7 +3856,8 @@ func (c *internalClient) deleteSnapshotBase(ctx context.Context, req DeleteSnaps return resp, nil } -// Deletes the specified snapshot. +// Deletes the specified snapshot. A deleted snapshot cannot be recovered. +// Returns a long-running operation. Poll it until done is true. func (c *internalClient) DeleteSnapshot(ctx context.Context, req DeleteSnapshotRequest, opts ...call.Option) (*DeleteSnapshotOperation, error) { operation, err := c.deleteSnapshotBase(ctx, req, opts...) if err != nil { @@ -4536,7 +4540,7 @@ func (c *internalClient) GetDataApi(ctx context.Context, req GetDataApiRequest, return resp, nil } -// Get a Database. +// Retrieves information about the specified Postgres database. func (c *internalClient) GetDatabase(ctx context.Context, req GetDatabaseRequest, opts ...call.Option) (*Database, error) { headers := http.Header{} @@ -5562,7 +5566,7 @@ func (c *internalClient) ListCdfStatusesIter(ctx context.Context, req ListCdfSta } } -// List Databases. +// Returns a list of Postgres databases in the branch. func (c *internalClient) ListDatabases(ctx context.Context, req ListDatabasesRequest, opts ...call.Option) (*ListDatabasesResponse, error) { wireReq, err := listDatabasesRequestToWire(&req) if err != nil { @@ -7347,7 +7351,8 @@ func (o *UpdateDataApiOperation) Wait(ctx context.Context, opts ...lro.Option) ( return result, nil } -// Update a Database. +// Updates the specified Postgres database. Only the fields named in +// `update_mask` change. func (c *internalClient) updateDatabaseBase(ctx context.Context, req UpdateDatabaseRequest, opts ...call.Option) (*Operation, error) { wireReq, err := updateDatabaseRequestToWire(&req) if err != nil { @@ -7423,7 +7428,8 @@ func (c *internalClient) updateDatabaseBase(ctx context.Context, req UpdateDatab return resp, nil } -// Update a Database. +// Updates the specified Postgres database. Only the fields named in +// `update_mask` change. func (c *internalClient) UpdateDatabase(ctx context.Context, req UpdateDatabaseRequest, opts ...call.Option) (*UpdateDatabaseOperation, error) { operation, err := c.updateDatabaseBase(ctx, req, opts...) if err != nil { diff --git a/postgres/v1/model.go b/postgres/v1/model.go index b4f03366..b3fb7aad 100644 --- a/postgres/v1/model.go +++ b/postgres/v1/model.go @@ -1175,8 +1175,9 @@ type CreateDatabaseRequest struct { // The ID used to identify and manage the Database resource through the API. The // Postgres database name is specified in `spec.postgres_database`. // - // This value should be 4-63 characters, and only use characters available in - // DNS names, as defined by RFC-1123 + // This value should be 3-63 characters, start with a lowercase letter, end with + // a lowercase letter or digit, and contain only lowercase letters, digits, and + // hyphens. // // If database_id is not specified in the request, it is generated // automatically. @@ -1184,7 +1185,9 @@ type CreateDatabaseRequest struct { // The desired specification of a Database. Database *Database // If true, update the database if it already exists instead of returning an - // error. + // error. This requires `database_id` to be set. Without `database_id`, the + // request follows the normal create path and fails if the database name already + // exists. ReplaceExisting *bool } @@ -1265,10 +1268,13 @@ type CreateRoleRequest struct { type CreateSnapshotRequest struct { // The project in which to create the snapshot. Format: projects/{project_id} Parent *string - // The snapshot to create. + // The snapshot to create. Only `spec` is read from this message; the system + // sets the other fields. Snapshot *Snapshot // Client-chosen ID for the snapshot. It becomes the final segment of the - // snapshot resource name and cannot be changed after creation. + // snapshot resource name and cannot be changed after creation. The ID must be 3 + // to 63 characters long, start with a lowercase letter, end with a lowercase + // letter or digit, and contain only lowercase letters, digits, and hyphens. SnapshotId *string } @@ -1402,7 +1408,8 @@ type Database_DatabaseSpec struct { // The name of the role that owns the database. Format: // projects/{project_id}/branches/{branch_id}/roles/{role_id} // - // To change the owner, pass valid existing Role name when updating the Database + // The role must be in the same branch as the database. To change the owner, + // pass valid existing Role name when updating the Database // // Required when creating the Database. Role *string `fieldmask:"role"` @@ -2604,7 +2611,10 @@ type ScheduleCadence_Schedule_MonthlySchedule struct { func (*ScheduleCadence_Schedule_MonthlySchedule) isScheduleCadence_Schedule() {} // An immutable, point-in-time copy of a branch's data within a project. It -// remains available after the source branch is deleted.. +// remains available after the source branch is deleted. To restore a snapshot, +// create a new branch with `spec.source_snapshot` set to the snapshot's +// resource name. For an overview, see +// https://docs.databricks.com/aws/en/oltp/projects/snapshots.. type Snapshot struct { // The resource name of the snapshot. Format: // projects/{project_id}/snapshots/{snapshot_id} @@ -2613,7 +2623,8 @@ type Snapshot struct { Uid *string // When the snapshot was created. CreateTime *types.Time - // Client-provided configuration of the snapshot. + // Client-provided configuration of the snapshot. When you create a snapshot, + // set exactly one of `expire_time`, `ttl`, or `no_expiry`. Spec *SnapshotSpec // Server-observed state of the snapshot. Status *SnapshotStatus @@ -2646,8 +2657,8 @@ type SnapshotScheduleOperationMetadata struct { // Client-provided configuration of the snapshot.. type SnapshotSpec struct { - // The source branch to snapshot. Format: - // projects/{project_id}/branches/{branch_id} + // The source branch to snapshot. The branch must belong to the same project as + // the snapshot. Format: projects/{project_id}/branches/{branch_id} SourceBranch *string // The point in time to snapshot from. If unset, the current head of the source // branch is used. The chosen LSN or timestamp must fall within the project's @@ -2663,8 +2674,10 @@ type isSnapshotSpec_PointInTime interface { } // SnapshotSpec_PointInTime_SourceBranchLsn selects SourceBranchLsn for SnapshotSpec.PointInTime. -// LSN to snapshot from, e.g. `16/B374D848`. Mutually exclusive with -// `source_branch_time`. +// Postgres log sequence number (LSN) to snapshot from. It must fall within the +// project's history retention window. If neither this field nor +// `source_branch_time` is set, the snapshot captures the current head of the +// source branch. Mutually exclusive with `source_branch_time`. type SnapshotSpec_PointInTime_SourceBranchLsn struct { SourceBranchLsn string } @@ -2672,7 +2685,8 @@ type SnapshotSpec_PointInTime_SourceBranchLsn struct { func (*SnapshotSpec_PointInTime_SourceBranchLsn) isSnapshotSpec_PointInTime() {} // SnapshotSpec_PointInTime_SourceBranchTime selects SourceBranchTime for SnapshotSpec.PointInTime. -// Timestamp to snapshot from. Mutually exclusive with `source_branch_lsn`. +// Timestamp to snapshot from. It must fall within the project's history +// retention window. Mutually exclusive with `source_branch_lsn`. type SnapshotSpec_PointInTime_SourceBranchTime struct { SourceBranchTime types.Time } @@ -2684,8 +2698,8 @@ type isSnapshotSpec_Expiration interface { } // SnapshotSpec_Expiration_ExpireTime selects ExpireTime for SnapshotSpec.Expiration. -// Absolute time at which the snapshot is deleted. Mutually exclusive with `ttl` -// and `no_expiry`. +// Absolute time at which the snapshot is deleted. It must not be in the past. +// Mutually exclusive with `ttl` and `no_expiry`. type SnapshotSpec_Expiration_ExpireTime struct { ExpireTime types.Time } @@ -2693,9 +2707,10 @@ type SnapshotSpec_Expiration_ExpireTime struct { func (*SnapshotSpec_Expiration_ExpireTime) isSnapshotSpec_Expiration() {} // SnapshotSpec_Expiration_Ttl selects Ttl for SnapshotSpec.Expiration. -// Time-to-live. The snapshot expires this long after it is created. Mutually -// exclusive with `expire_time` and `no_expiry`. Reads report the resolved -// absolute `expire_time` instead. +// Time-to-live, as a positive duration in seconds with an `s` suffix. The +// snapshot expires this long after it is created. Mutually exclusive with +// `expire_time` and `no_expiry`. Reads report the resolved absolute +// `expire_time` instead. type SnapshotSpec_Expiration_Ttl struct { Ttl types.Duration } @@ -2703,7 +2718,8 @@ type SnapshotSpec_Expiration_Ttl struct { func (*SnapshotSpec_Expiration_Ttl) isSnapshotSpec_Expiration() {} // SnapshotSpec_Expiration_NoExpiry selects NoExpiry for SnapshotSpec.Expiration. -// If true, the snapshot never expires. Mutually exclusive with `ttl` and +// If true, the snapshot never expires. The value `false` is rejected; set +// `expire_time` or `ttl` instead. Mutually exclusive with `ttl` and // `expire_time`. type SnapshotSpec_Expiration_NoExpiry struct { NoExpiry bool @@ -2983,7 +2999,8 @@ type UpdateDatabaseRequest struct { // The database's `name` field is used to identify the database to update. // Format: projects/{project_id}/branches/{branch_id}/databases/{database_id} Database *Database - // The list of fields to update. + // The list of fields to update. The updatable fields are `spec.role` and + // `spec.postgres_database`. UpdateMask *types.FieldMask[Database] } diff --git a/queryhistory/CHANGELOG.md b/queryhistory/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/queryhistory/CHANGELOG.md +++ b/queryhistory/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/queryhistory/internal/version.go b/queryhistory/internal/version.go index 41725c6b..c3dc9b5f 100644 --- a/queryhistory/internal/version.go +++ b/queryhistory/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-queryhistory" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/queryhistory/v1/model.go b/queryhistory/v1/model.go index 2160426b..e56611a5 100644 --- a/queryhistory/v1/model.go +++ b/queryhistory/v1/model.go @@ -109,7 +109,9 @@ type ExternalQuerySource struct { NotebookId *string // The canonical identifier for this SQL query SqlQueryId *string - JobInfo *ExternalQuerySource_JobInfo + // Identifiers of the job, job run, and task run that executed this statement, + // when it ran as part of a job. + JobInfo *ExternalQuerySource_JobInfo // The canonical identifier for this Genie space GenieSpaceId *string } @@ -131,15 +133,15 @@ type ExternalQuerySource_JobInfo struct { // conforms to the 10 second max query time limit.. type ListQueriesRequest struct { // An optional filter object to limit query history results. Accepts parameters - // such as user IDs, endpoint IDs, and statuses to narrow the returned data. In + // such as user IDs, warehouse IDs, and statuses to narrow the returned data. In // a URL, the parameters of this filter are specified with dot notation. For // example: `filter_by.statement_ids`. FilterBy *QueryFilter - // Limit the number of results returned in one page. Must be less than 1000 and + // Limit the number of results returned in one page. Must be no more than 1000; // the default is 100. MaxResults *int // A token that can be used to get the next page of results. The token can - // contains characters that need to be encoded before using it in a URL. For + // contain characters that need to be encoded before using it in a URL. For // example, the character '+' needs to be replaced by %2B. This field is // optional. PageToken *string @@ -174,20 +176,22 @@ type QueryFilter struct { } type QueryInfo struct { - // The query ID. + // The UUID that uniquely identifies the query execution. It matches the + // statement ID shown in the Query History UI and the values accepted by the + // `statement_ids` filter. QueryId *string - // Query status with one the following values: - `QUEUED`: Query has been + // Query status with one of the following values: - `QUEUED`: Query has been // received and queued. - `RUNNING`: Query has started. - `CANCELED`: Query has // been cancelled by the user. - `FAILED`: Query has failed. - `FINISHED`: Query // has completed. Status QueryStatus // The text of the query. QueryText *string - // The time the query started. + // The time the query started, in milliseconds since the Unix epoch. QueryStartTimeMs *int64 - // The time execution of the query ended. + // The time execution of the query ended, in milliseconds since the Unix epoch. ExecutionEndTimeMs *int64 - // The time the query ended. + // The time the query ended, in milliseconds since the Unix epoch. QueryEndTimeMs *int64 // The ID of the user who ran the query. UserId *int64 @@ -203,7 +207,8 @@ type QueryInfo struct { ErrorMessage *string // A key that can be used to look up query details. LookupKey *string - // Metrics about query execution. + // Metrics about query execution. Only returned when `include_metrics` is true + // in the request. Metrics *QueryMetrics // The ID of the user whose credentials were used to run the query. ExecutedAsUserId *int64 @@ -213,7 +218,8 @@ type QueryInfo struct { // The spark session UUID that query ran on. This is either the Spark Connect, // DBSQL, or SDP session ID. SessionId *string - // Whether more updates for the query are expected. + // Whether the query has reached a final state, so no further updates to this + // record are expected. IsFinal *bool // SQL Warehouse channel information at the time of query execution ChannelUsed *ChannelInfo @@ -221,11 +227,11 @@ type QueryInfo struct { PlansState PlansState // Type of statement for this query StatementType QueryStatementType - // Warehouse ID. + // ID of the warehouse that ran the query. WarehouseId *string - // Total time of the statement execution. This value does not include the time - // taken to retrieve the results, which can result in a discrepancy between this - // value and the start-to-finish wall-clock time. + // Total time of the statement execution, in milliseconds. This value does not + // include the time taken to retrieve the results, which can result in a + // discrepancy between this value and the start-to-finish wall-clock time. Duration *int64 // Client application that ran the statement. For example: Databricks SQL // Editor, Tableau, and Power BI. This field is derived from information @@ -236,9 +242,12 @@ type QueryInfo struct { // that were involved in the execution of this statement, such as jobs, // notebooks, or dashboards. This field only records entities. QuerySource *ExternalQuerySource - // The ID of the cached query if this result retrieved from cache + // The ID of the query whose cached result was returned, if this result was + // retrieved from the cache. CacheQueryId *string - // A query execution can be optionally annotated with query tags + // Key-value tags attached to the query execution, for example with the `SET + // QUERY_TAGS` statement, for grouping and filtering. See + // https://docs.databricks.com/aws/en/sql/user/queries/query-tags. QueryTags []QueryTag } @@ -259,7 +268,7 @@ type QueryMetrics struct { // Size of persistent data read from cloud object storage on your cloud tenant, // in bytes. ReadRemoteBytes *int64 - // Size pf persistent data written to cloud object storage in your cloud tenant, + // Size of persistent data written to cloud object storage in your cloud tenant, // in bytes. WriteRemoteBytes *int64 // Size of persistent data read from the cache, in bytes. @@ -290,25 +299,26 @@ type QueryMetrics struct { // Total number of files from all tables not read due to pruning PrunedFilesCount *int64 // Timestamp of when the query was enqueued waiting for a cluster to be - // provisioned for the warehouse. This field is optional and will not appear if - // the query skipped the provisioning queue. + // provisioned for the warehouse, in milliseconds since the Unix epoch. This + // field is optional and will not appear if the query skipped the provisioning + // queue. ProvisioningQueueStartTimestamp *int64 // Timestamp of when the query was enqueued waiting while the warehouse was at - // max load. This field is optional and will not appear if the query skipped the - // overloading queue. + // max load, in milliseconds since the Unix epoch. This field is optional and + // will not appear if the query skipped the overloading queue. OverloadingQueueStartTimestamp *int64 - // Timestamp of when the underlying compute started compilation of the query. + // Timestamp of when the underlying compute started compilation of the query, in + // milliseconds since the Unix epoch. QueryCompilationStartTimestamp *int64 // sum of task times completed in a range of wall clock time, approximated to a // configurable number of points aggregated over all stages and jobs in the // query (based on task_total_time_ms) TaskTimeOverTimeRange *TaskTimeOverRange - // remaining work to be done across all stages in the query, calculated by - // autoscaler StatementAnalysis.scala, in milliseconds deprecated: using - // projected_remaining_task_total_time_ms instead + // Deprecated: use `projected_remaining_task_total_time_ms` instead. Remaining + // work to be done across all stages in the query, in milliseconds. WorkToBeDone *int64 - // number of remaining tasks to complete, calculated by autoscaler - // StatementAnalysis.scala deprecated: use remaining_task_count instead + // Deprecated: use `remaining_task_count` instead. Number of remaining tasks to + // complete. RunnableTasks *int64 // projected remaining work to be done aggregated across all stages in the // query, in milliseconds @@ -327,11 +337,15 @@ type QueryMetrics struct { // users to attribute the executions by key and optional value to filter by. // QueryTag is the user-facing representation.. type QueryTag struct { - Key *string + // The key of the tag. + Key *string + // The value of the tag. Key-only tags have no value. Value *string } type TaskTimeOverRange struct { + // Each entry covers one consecutive time bucket of length `interval`, starting + // at `query_start_time_ms`. Entries []TaskTimeOverRangeEntry // interval length for all entries (difference in start time and end time of an // entry range) the same for all entries start time of first interval is @@ -346,8 +360,8 @@ type TaskTimeOverRangeEntry struct { } type TimeRange struct { - // The start time in milliseconds. + // Start of the range, in milliseconds since the Unix epoch. StartTimeMs *int64 - // The end time in milliseconds. + // End of the range, in milliseconds since the Unix epoch. EndTimeMs *int64 } diff --git a/scim/CHANGELOG.md b/scim/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/scim/CHANGELOG.md +++ b/scim/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/scim/internal/version.go b/scim/internal/version.go index 4b711d6f..97e0e246 100644 --- a/scim/internal/version.go +++ b/scim/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-scim" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/scim/v1/client.go b/scim/v1/client.go index 40a808a2..507f9432 100644 --- a/scim/v1/client.go +++ b/scim/v1/client.go @@ -78,7 +78,7 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Creates a group in the account with a unique name, using the +// Creates a group in the Databricks account with a unique name, using the // supplied group details. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) CreateAccountGroup(ctx context.Context, req CreateAccountGroupRequest, opts ...call.Option) (*AccountGroup, error) { @@ -151,7 +151,7 @@ func (c *internalClient) CreateAccountGroup(ctx context.Context, req CreateAccou return resp, nil } -// Deletes a group from the account. +// Deletes a group from the Databricks account. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) DeleteAccountGroup(ctx context.Context, req DeleteAccountGroupRequest, opts ...call.Option) error { @@ -208,7 +208,7 @@ func (c *internalClient) DeleteAccountGroup(ctx context.Context, req DeleteAccou return nil } -// Gets the information for a specific group in the account. +// Gets the information for a specific group in the Databricks account. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) GetAccountGroup(ctx context.Context, req GetAccountGroupRequest, opts ...call.Option) (*AccountGroup, error) { @@ -274,8 +274,8 @@ func (c *internalClient) GetAccountGroup(ctx context.Context, req GetAccountGrou return resp, nil } -// Gets all details of the groups associated with the account. As -// of 08/22/2025, this endpoint will no longer return members. Instead, members +// Gets all details of the groups associated with the Databricks account. As of +// 08/22/2025, this endpoint will no longer return members. Instead, members // should be retrieved by iterating through `Get group details`. Existing // accounts that rely on this attribute will not be impacted and will continue // receiving member data as before. @@ -620,7 +620,7 @@ func (c *internalClient) CreateAccountServicePrincipal(ctx context.Context, req return resp, nil } -// Delete a single service principal in the account. +// Deletes a single service principal from the account. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) DeleteAccountServicePrincipal(ctx context.Context, req DeleteAccountServicePrincipalRequest, opts ...call.Option) error { @@ -677,8 +677,7 @@ func (c *internalClient) DeleteAccountServicePrincipal(ctx context.Context, req return nil } -// Gets the details for a single service principal define in the -// account. +// Gets the details for a single service principal in the account. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) GetAccountServicePrincipal(ctx context.Context, req GetAccountServicePrincipalRequest, opts ...call.Option) (*AccountServicePrincipal, error) { @@ -949,7 +948,7 @@ func (c *internalClient) PatchAccountServicePrincipal(ctx context.Context, req P // Updates the details of a single service principal. // -// This action replaces the existing service principal with the same name. +// This action replaces the existing service principal that has the given ID. // Account-level method. Uses the Client's accountID, overridable per call via req.AccountId. func (c *internalClient) UpdateAccountServicePrincipal(ctx context.Context, req UpdateAccountServicePrincipalRequest, opts ...call.Option) error { wireReq, err := updateAccountServicePrincipalRequestToWire(&req) diff --git a/scim/v1/model.go b/scim/v1/model.go index ece2ca70..da40b69d 100644 --- a/scim/v1/model.go +++ b/scim/v1/model.go @@ -108,29 +108,48 @@ const ( ) type AccountComplexValue struct { + // Provides a human-readable name for the value, such as the display name of a + // group member. Display *string + // Indicates whether this is the primary or preferred value of a multi-valued + // attribute. At most one value can be primary. Primary *bool - Ref *string - Type *string - Value *string + // References the resource that the value refers to by its URI, such as the + // user, service principal or group entry of a group member. + Ref *string + // Indicates the function or kind of the value within a multi-valued attribute. + Type *string + // Holds the value of the attribute. For references to other resources, such as + // group members, this is the ID of the referenced resource. + Value *string } type AccountGetSortOrder struct { } +// Represents a group of users, service principals and other groups in the +// Databricks account. Members inherit the permissions that are granted to the +// group.. type AccountGroup struct { - // String that represents a human-readable group name + // Identifies the group by a human-readable name. The name must be unique within + // the account and must not exceed 255 characters. DisplayName *string - // external_id should be unique for identifying groups + // Identifies the group in an external identity provider. Identity providers use + // it to match the groups they manage to Databricks groups, so it should be + // unique among the groups in the account. ExternalId *string - // group ID - Id *string + // Identifies the group within Databricks. Databricks generates the ID when the + // group is created. + Id *string + // Lists the members of the group. Each entry identifies a user, service + // principal or group in the account by its ID in the value sub-attribute. Members []AccountComplexValue // Container for the group identifier. Workspace local versus account. Meta *AccountResourceMeta - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the group. The only supported + // role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account that the group belongs to. AccountId *string } @@ -149,9 +168,15 @@ type AccountName struct { type AccountPatch struct { // Type of patch operation. Op AccountPatchOp_PatchOp - // Selection of patch operation + // Specifies the attribute that the operation targets, optionally with a filter + // that selects one value of a multi-valued attribute. The path is required for + // remove operations. When it is omitted, the operation applies to the resource + // itself and the value holds the attributes to add or replace. Path *string - // Value to modify + // Holds the value that the operation adds or sets. Depending on the target, it + // is a single value, a list of entries for a multi-valued attribute, or an + // object with the attributes to change when the path is omitted. Remove + // operations do not need a value. Value json.RawMessage } @@ -167,19 +192,31 @@ type AccountResourceMeta struct { ResourceType *string } +// Represents a service principal in the Databricks account. Service principals +// are identities for jobs, automated tools and systems such as scripts, apps +// and CI/CD platforms.. type AccountServicePrincipal struct { - // If this user is active + // Indicates whether the service principal is active. A deactivated service + // principal cannot authenticate or access Databricks APIs, but it keeps its + // permissions. Active *bool - // UUID relating to the service principal + // Identifies the service principal when it authenticates to Databricks. + // Databricks generates the application ID, except on Azure, where it can be the + // application ID of a Microsoft Entra ID service principal. ApplicationId *string - // String that represents a concatenation of given and family names. + // Identifies the service principal by a human-readable name. DisplayName *string - ExternalId *string - // service principal ID. + // Identifies the service principal in an external identity provider. Identity + // providers use it to match the service principals they manage to Databricks + // service principals. + ExternalId *string + // Identifies the service principal within Databricks. Databricks generates the + // ID when the service principal is created. Id *string - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the service principal. The only + // supported role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account that the service principal belongs to. AccountId *string } @@ -216,33 +253,46 @@ type ComplexValue struct { } type CreateAccountGroupRequest struct { - // String that represents a human-readable group name + // Sets the human-readable name of the group. The name must be unique within the + // account and must not exceed 255 characters. DisplayName *string - ExternalId *string - // group ID - Id *string + // Sets the identifier of the group in an external identity provider. + ExternalId *string + // Identifies the group within Databricks. + Id *string + // Lists the members of the group. Each entry identifies a user, service + // principal or group in the account by its ID in the value sub-attribute. Members []AccountComplexValue // Container for the group identifier. Workspace local versus account. Meta *AccountResourceMeta - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the group. The only supported + // role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account. AccountId *string } type CreateAccountServicePrincipalRequest struct { - // If this user is active + // Indicates whether the service principal is active. A deactivated service + // principal cannot authenticate or access Databricks APIs, but it keeps its + // permissions. Active *bool - // UUID relating to the service principal + // Sets the application ID of a Microsoft Entra ID service principal. Only Azure + // accepts this field. On AWS and GCP, Databricks generates the application ID + // and rejects requests that set it. ApplicationId *string - // String that represents a concatenation of given and family names. + // Sets the human-readable name of the service principal. The name is required + // unless the request sets `applicationId`. DisplayName *string - ExternalId *string - // service principal ID. + // Sets the identifier of the service principal in an external identity + // provider. + ExternalId *string + // Identifies the service principal within Databricks. Id *string - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the service principal. The only + // supported role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account. AccountId *string } @@ -343,9 +393,9 @@ type CreateUserRequest struct { // Delete a group. type DeleteAccountGroupRequest struct { - // Unique ID for a group in the account. + // Unique ID for a group in the Databricks account. Id *string - // account ID + // Identifies the Databricks account. AccountId *string } @@ -353,7 +403,7 @@ type DeleteAccountGroupRequest struct { type DeleteAccountServicePrincipalRequest struct { // Unique ID for a service principal in the account. Id *string - // account ID + // Identifies the Databricks account. AccountId *string } @@ -385,9 +435,9 @@ type DeleteUserRequest struct { // Get group details. type GetAccountGroupRequest struct { - // Unique ID for a group in the account. + // Unique ID for a group in the Databricks account. Id *string - // account ID + // Identifies the Databricks account. AccountId *string } @@ -395,7 +445,7 @@ type GetAccountGroupRequest struct { type GetAccountServicePrincipalRequest struct { // Unique ID for a service principal in the account. Id *string - // account ID + // Identifies the Databricks account. AccountId *string } @@ -503,7 +553,7 @@ type Group struct { // List group details. type ListAccountGroupsRequest struct { - // account ID + // Identifies the Databricks account. AccountId *string // Query by which the results have to be filtered. Supported operators are // equals(`eq`), contains(`co`), starts with(`sw`) and not equals(`ne`). @@ -519,7 +569,8 @@ type ListAccountGroupsRequest struct { ExcludedAttributes *string // Specifies the index of the first result. First item is number 1. StartIndex *int64 - // Desired number of results per page. Default is 10000. + // Specifies the number of results per page. When omitted, the server uses its + // default page size. Count *int64 // Attribute to sort the results. SortBy *string @@ -530,7 +581,7 @@ type ListAccountGroupsRequest struct { type ListAccountGroupsResponse struct { // Total results returned in the response. ItemsPerPage *int - // User objects returned in the response. + // Group objects returned in the response. Resources []AccountGroup // Starting index of all the results that matched the request filters. First // item is number 1. @@ -561,14 +612,14 @@ type ListAccountServicePrincipalsRequest struct { SortOrder AccountListSort_Order // Specifies the index of the first result. First item is number 1. StartIndex *int64 - // account ID + // Identifies the Databricks account. AccountId *string } type ListAccountServicePrincipalsResponse struct { // Total results returned in the response. ItemsPerPage *int - // User objects returned in the response. + // Lists the service principals that are returned in the response. Resources []AccountServicePrincipal // Starting index of all the results that matched the request filters. First // item is number 1. @@ -807,24 +858,26 @@ type Patch struct { } type PatchAccountGroupRequest struct { - // Unique ID in the workspace. - Id *string + // Unique ID for a group in the Databricks account. + Id *string + // Lists the patch operations to apply to the group. Operations []AccountPatch // The schema of the patch request. Must be // ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]. Schemas []AccountPatchSchema_PatchSchema - // account ID + // Identifies the Databricks account. AccountId *string } type PatchAccountServicePrincipalRequest struct { - // Unique ID in the workspace. - Id *string + // Unique ID for a service principal in the Databricks account. + Id *string + // Lists the patch operations to apply to the service principal. Operations []AccountPatch // The schema of the patch request. Must be // ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]. Schemas []AccountPatchSchema_PatchSchema - // account ID + // Identifies the Databricks account. AccountId *string } @@ -895,33 +948,46 @@ type ServicePrincipal struct { } type UpdateAccountGroupRequest struct { - // String that represents a human-readable group name + // Sets the human-readable name of the group. The name must be unique within the + // account and must not exceed 255 characters. DisplayName *string - ExternalId *string - // group ID - Id *string + // Sets the identifier of the group in an external identity provider. + ExternalId *string + // Identifies the group within Databricks. + Id *string + // Lists the members of the group. Each entry identifies a user, service + // principal or group in the account by its ID in the value sub-attribute. Members []AccountComplexValue // Container for the group identifier. Workspace local versus account. Meta *AccountResourceMeta - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the group. The only supported + // role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account. AccountId *string } type UpdateAccountServicePrincipalRequest struct { - // If this user is active + // Indicates whether the service principal is active. A deactivated service + // principal cannot authenticate or access Databricks APIs, but it keeps its + // permissions. Active *bool - // UUID relating to the service principal + // Sets the application ID of a Microsoft Entra ID service principal. Only Azure + // accepts this field. On AWS and GCP, Databricks generates the application ID + // and rejects requests that set it. ApplicationId *string - // String that represents a concatenation of given and family names. + // Sets the human-readable name of the service principal. The name is required + // unless the request sets `applicationId`. DisplayName *string - ExternalId *string - // service principal ID. + // Sets the identifier of the service principal in an external identity + // provider. + ExternalId *string + // Identifies the service principal within Databricks. Id *string - // Indicates if the group has the admin role. + // Lists the account roles that are assigned to the service principal. The only + // supported role value is `account_admin`. Roles []AccountComplexValue - // account ID + // Identifies the Databricks account. AccountId *string } diff --git a/secrets/CHANGELOG.md b/secrets/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/secrets/CHANGELOG.md +++ b/secrets/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/secrets/internal/version.go b/secrets/internal/version.go index 55983a9f..4feb74bb 100644 --- a/secrets/internal/version.go +++ b/secrets/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-secrets" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/secrets/v1/client.go b/secrets/v1/client.go index 13af6d80..fea724af 100644 --- a/secrets/v1/client.go +++ b/secrets/v1/client.go @@ -319,12 +319,6 @@ func (c *internalClient) DeleteScope(ctx context.Context, req DeleteScopeRequest // Deletes the secret stored in this secret scope. You must have “WRITE“ or // “MANAGE“ permission on the Secret Scope. // -// Example request: -// -// .. code:: -// -// { "scope": "my-secret-scope", "key": "my-secret-key" } -// // Throws “RESOURCE_DOES_NOT_EXIST“ if no such secret scope or secret exists. // Throws “PERMISSION_DENIED“ if the user does not have permission to make // this API call. Throws “BAD_REQUEST“ if system user attempts to delete an @@ -467,33 +461,20 @@ func (c *internalClient) GetAcl(ctx context.Context, req GetAclRequest, opts ... return resp, nil } -// Gets a secret for a given key and scope. This API can only be called from the -// DBUtils interface. Users need the READ permission to make this call. +// Gets the bytes representation of a secret value for the specified scope and +// key. Users need the READ permission to make this call. // -// Example response: -// -// .. code:: -// -// { "key": "my-string-key", "value": } -// -// Note that the secret value returned is in bytes. The interpretation of the -// bytes is determined by the caller in DBUtils and the type the data is decoded -// into. +// The secret value is returned as bytes regardless of whether it was written +// with “string_value“ or “bytes_value“, so the caller decides how to decode +// it. Values returned by this API are not redacted. // // Throws “RESOURCE_DOES_NOT_EXIST“ if no such secret or secret scope exists. // Throws “PERMISSION_DENIED“ if the user does not have permission to make -// this API call. -// -// Note: This is explicitly an undocumented API. It also doesn't need to be -// supported for the /preview prefix, because it's not a customer-facing API -// (i.e. only used for DBUtils SecretUtils to fetch secrets). -// -// Throws “RESOURCE_DOES_NOT_EXIST“ if no such secret scope or secret exists. -// Throws “BAD_REQUEST“ if normal user calls get secret outside of a notebook. -// AKV specific errors: Throws “INVALID_PARAMETER_VALUE“ if secret name is not -// alphanumeric or too long. Throws “PERMISSION_DENIED“ if secret manager -// cannot access AKV with 403 error Throws “MALFORMED_REQUEST“ if secret -// manager cannot access AKV with any other 4xx error +// this API call. Azure Key Vault-backed scopes can also return the following +// errors: Throws “INVALID_PARAMETER_VALUE“ if the secret name is not +// alphanumeric or is too long. Throws “PERMISSION_DENIED“ if +// cannot access the key vault (HTTP 403). Throws “MALFORMED_REQUEST“ if +// cannot access the key vault with any other 4xx error. func (c *internalClient) GetSecret(ctx context.Context, req GetSecretRequest, opts ...call.Option) (*GetSecretResponse, error) { wireReq, err := getSecretRequestToWire(&req) if err != nil { @@ -705,16 +686,6 @@ func (c *internalClient) ListScopes(ctx context.Context, req ListScopesRequest, // operation; secret data cannot be retrieved using this API. Users need the // READ permission to make this call. // -// Example response: -// -// .. code:: -// -// { "secrets": [ { "key": "my-string-key"", "last_updated_timestamp": -// "1520467595000" }, { "key": "my-byte-key", "last_updated_timestamp": -// "1520467595000" }, ] } -// -// The lastUpdatedTimestamp returned is in milliseconds since epoch. -// // Throws “RESOURCE_DOES_NOT_EXIST“ if no such secret scope exists. Throws // “PERMISSION_DENIED“ if the user does not have permission to make this API // call. @@ -882,16 +853,9 @@ func (c *internalClient) PutAcl(ctx context.Context, req PutAclRequest, opts ... // and periods, and cannot exceed 128 characters. The maximum allowed secret // value size is 128 KB. The maximum number of secrets in a given scope is 1000. // -// Example request: -// -// .. code:: -// -// { "scope": "my-databricks-scope", "key": "my-string-key", "string_value": -// "foobar" } -// -// The input fields "string_value" or "bytes_value" specify the type of the -// secret, which will determine the value returned when the secret value is -// requested. Exactly one must be specified. +// Exactly one of “string_value“ or “bytes_value“ must be specified. The +// server stores both kinds of value as bytes and does not record which one was +// used. // // Throws “RESOURCE_DOES_NOT_EXIST“ if no such secret scope exists. Throws // “RESOURCE_LIMIT_EXCEEDED“ if maximum number of secrets in scope is diff --git a/secrets/v1/model.go b/secrets/v1/model.go index 1ba24223..ba3b7f4e 100644 --- a/secrets/v1/model.go +++ b/secrets/v1/model.go @@ -110,7 +110,8 @@ type GetSecretRequest struct { type GetSecretResponse struct { // A unique name to identify the secret. Key *string - // The value of the secret in its byte representation. + // The secret value, base64-encoded. A secret written with ``string_value`` is + // returned as the base64 encoding of its UTF-8 bytes. Value []byte } @@ -155,7 +156,7 @@ type PutAclResponse struct { } type PutSecretRequest struct { - // The name of the scope to which the secret will be associated with. + // The name of the scope to store the secret in. Scope *string // A unique name to identify the secret. Key *string @@ -167,7 +168,8 @@ type isPutSecretRequest_Value interface { } // PutSecretRequest_Value_StringValue selects StringValue for PutSecretRequest.Value. -// If specified, note that the value will be stored in UTF-8 (MB4) form. +// The secret value as text. The server stores it as the UTF-8 encoding of the +// string. type PutSecretRequest_Value_StringValue struct { StringValue string } @@ -175,7 +177,8 @@ type PutSecretRequest_Value_StringValue struct { func (*PutSecretRequest_Value_StringValue) isPutSecretRequest_Value() {} // PutSecretRequest_Value_BytesValue selects BytesValue for PutSecretRequest.Value. -// If specified, value will be stored as bytes. +// The secret value as raw bytes. The bytes must be base64-encoded in the JSON +// request body. type PutSecretRequest_Value_BytesValue struct { BytesValue []byte } @@ -190,7 +193,7 @@ type PutSecretResponse struct { type SecretMetadata struct { // A unique name to identify the secret. Key *string - // The last updated timestamp (in milliseconds) for the secret. + // The time the secret was last written, in milliseconds since the Unix epoch. LastUpdatedTimestamp *int64 } diff --git a/sharing/CHANGELOG.md b/sharing/CHANGELOG.md index 3fbf2d71..f06f06d5 100644 --- a/sharing/CHANGELOG.md +++ b/sharing/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.9.0 (2026-11-17) + ## Release v0.8.0 (2026-11-12) ### New Features and Improvements diff --git a/sharing/internal/version.go b/sharing/internal/version.go index 759ded63..b93fdcbc 100644 --- a/sharing/internal/version.go +++ b/sharing/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-sharing" -const Version = "0.8.0" +const Version = "0.9.0" diff --git a/sharing/v1/client.go b/sharing/v1/client.go index b64a0449..e8969ef9 100644 --- a/sharing/v1/client.go +++ b/sharing/v1/client.go @@ -1091,6 +1091,9 @@ func (c *internalClient) ListProviderShareAssets(ctx context.Context, req ListPr if err := addQueryValue(queryParams, "notebook_max_results", wireReq.NotebookMaxResults); err != nil { return nil, err } + if err := addQueryValue(queryParams, "skill_max_results", wireReq.SkillMaxResults); err != nil { + return nil, err + } baseURL.RawQuery = queryParams.Encode() urlStr := baseURL.String() diff --git a/sharing/v1/model.go b/sharing/v1/model.go index ff189ebb..5577c2ff 100644 --- a/sharing/v1/model.go +++ b/sharing/v1/model.go @@ -483,6 +483,8 @@ type ListProviderShareAssetsRequest struct { VolumeMaxResults *int // Maximum number of notebooks to return. NotebookMaxResults *int + // Maximum number of skills to return. + SkillMaxResults *int } // Response to ListProviderShareAssets, which contains the list of assets of a @@ -496,6 +498,8 @@ type ListProviderShareAssetsResponse struct { Notebooks []NotebookFile // The list of volumes in the share. Volumes []Volume + // The list of skills in the share. + Skills []Skill // The metadata of the share. Share *Share } @@ -1024,6 +1028,30 @@ type SharedDataObject_HistoryDataSharingStatus struct { type SharedDataObject_Status struct { } +// An agent skill shared through a Delta Sharing share, as served to a +// recipient.. +type Skill struct { + // The name of the skill. + Name *string + // The shared-object ID that recipients match replicas on. For a skill shared + // through its schema, which has no shared-object row, this is derived from the + // shared schema and the skill. + Id *string + // The name of the schema that the skill belongs to. + Schema *string + // The name of the share that the skill belongs to. + Share *string + // The id of the share that the skill belongs to. + ShareId *string + // The comment of the skill. + Comment *string + // The bundle description parsed from the skill bundle's SKILL.md frontmatter. + Description *string + // The name from the skill bundle frontmatter, which may differ from the + // resource name. + BundleName *string +} + type Table struct { // The name of the table. Name *string diff --git a/sharing/v1/wire.go b/sharing/v1/wire.go index ee9d592d..eb693033 100644 --- a/sharing/v1/wire.go +++ b/sharing/v1/wire.go @@ -1877,6 +1877,7 @@ type listProviderShareAssetsRequestWire struct { FunctionMaxResults *int `json:"function_max_results,omitempty"` VolumeMaxResults *int `json:"volume_max_results,omitempty"` NotebookMaxResults *int `json:"notebook_max_results,omitempty"` + SkillMaxResults *int `json:"skill_max_results,omitempty"` } func listProviderShareAssetsRequestToWire(v *ListProviderShareAssetsRequest) (*listProviderShareAssetsRequestWire, error) { @@ -1890,6 +1891,7 @@ func listProviderShareAssetsRequestToWire(v *ListProviderShareAssetsRequest) (*l FunctionMaxResults: v.FunctionMaxResults, VolumeMaxResults: v.VolumeMaxResults, NotebookMaxResults: v.NotebookMaxResults, + SkillMaxResults: v.SkillMaxResults, }, nil } @@ -1904,6 +1906,7 @@ func listProviderShareAssetsRequestFromWire(w *listProviderShareAssetsRequestWir FunctionMaxResults: w.FunctionMaxResults, VolumeMaxResults: w.VolumeMaxResults, NotebookMaxResults: w.NotebookMaxResults, + SkillMaxResults: w.SkillMaxResults, }, nil } @@ -1936,6 +1939,7 @@ type listProviderShareAssetsResponseWire struct { Functions []functionWire `json:"functions,omitempty"` Notebooks []notebookFileWire `json:"notebooks,omitempty"` Volumes []volumeWire `json:"volumes,omitempty"` + Skills []skillWire `json:"skills,omitempty"` Share *shareWire `json:"share,omitempty"` } @@ -1959,6 +1963,10 @@ func listProviderShareAssetsResponseToWire(v *ListProviderShareAssetsResponse) ( if err != nil { return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Volumes", err) } + skillsWireValue, err := convertSlice(v.Skills, skillToWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Skills", err) + } shareWireValue, err := shareToWire(v.Share) if err != nil { return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Share", err) @@ -1968,6 +1976,7 @@ func listProviderShareAssetsResponseToWire(v *ListProviderShareAssetsResponse) ( Functions: functionsWireValue, Notebooks: notebooksWireValue, Volumes: volumesWireValue, + Skills: skillsWireValue, Share: shareWireValue, }, nil } @@ -1992,6 +2001,10 @@ func listProviderShareAssetsResponseFromWire(w *listProviderShareAssetsResponseW if err != nil { return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Volumes", err) } + skillsPublicValue, err := convertSlice(w.Skills, skillFromWire) + if err != nil { + return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Skills", err) + } sharePublicValue, err := shareFromWire(w.Share) if err != nil { return nil, fmt.Errorf("%s: %w", "ListProviderShareAssetsResponse.Share", err) @@ -2001,6 +2014,7 @@ func listProviderShareAssetsResponseFromWire(w *listProviderShareAssetsResponseW Functions: functionsPublicValue, Notebooks: notebooksPublicValue, Volumes: volumesPublicValue, + Skills: skillsPublicValue, Share: sharePublicValue, }, nil } @@ -4039,6 +4053,73 @@ func (v *SharedDataObject_Status) UnmarshalJSON(data []byte) error { return nil } +type skillWire struct { + Name *string `json:"name,omitempty"` + Id *string `json:"id,omitempty"` + Schema *string `json:"schema,omitempty"` + Share *string `json:"share,omitempty"` + ShareId *string `json:"share_id,omitempty"` + Comment *string `json:"comment,omitempty"` + Description *string `json:"description,omitempty"` + BundleName *string `json:"bundle_name,omitempty"` +} + +func skillToWire(v *Skill) (*skillWire, error) { + if v == nil { + return nil, nil + } + return &skillWire{ + Name: v.Name, + Id: v.Id, + Schema: v.Schema, + Share: v.Share, + ShareId: v.ShareId, + Comment: v.Comment, + Description: v.Description, + BundleName: v.BundleName, + }, nil +} + +func skillFromWire(w *skillWire) (*Skill, error) { + if w == nil { + return nil, nil + } + return &Skill{ + Name: w.Name, + Id: w.Id, + Schema: w.Schema, + Share: w.Share, + ShareId: w.ShareId, + Comment: w.Comment, + Description: w.Description, + BundleName: w.BundleName, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v Skill) MarshalJSON() ([]byte, error) { + w, err := skillToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *Skill) UnmarshalJSON(data []byte) error { + var w skillWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := skillFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type tableWire struct { Name *string `json:"name,omitempty"` Schema *string `json:"schema,omitempty"` diff --git a/softwareartifacts/CHANGELOG.md b/softwareartifacts/CHANGELOG.md index c0720d69..eb5522da 100644 --- a/softwareartifacts/CHANGELOG.md +++ b/softwareartifacts/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.5.0 (2026-11-17) + ## Release v0.4.0 (2026-11-12) ### New Features and Improvements diff --git a/softwareartifacts/internal/version.go b/softwareartifacts/internal/version.go index 7acb32ad..a214d4f8 100644 --- a/softwareartifacts/internal/version.go +++ b/softwareartifacts/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-softwareartifacts" -const Version = "0.4.0" +const Version = "0.5.0" diff --git a/softwareartifacts/v1/client.go b/softwareartifacts/v1/client.go index a1070ed0..2125f0e7 100644 --- a/softwareartifacts/v1/client.go +++ b/softwareartifacts/v1/client.go @@ -78,7 +78,8 @@ func NewClient(ctx context.Context, opts ...client.Option) (*Client, error) { }, nil } -// Gets a software artifact. +// Gets a software artifact by its resource name. Returns `NOT_FOUND` if the +// artifact does not exist. func (c *internalClient) GetSoftwareArtifact(ctx context.Context, req GetSoftwareArtifactRequest, opts ...call.Option) (*SoftwareArtifact, error) { headers := http.Header{} @@ -140,7 +141,8 @@ func (c *internalClient) GetSoftwareArtifact(ctx context.Context, req GetSoftwar return resp, nil } -// Gets a software artifact version. +// Gets a software artifact version by its resource name. Returns `NOT_FOUND` if +// the version does not exist. func (c *internalClient) GetSoftwareArtifactVersion(ctx context.Context, req GetSoftwareArtifactVersionRequest, opts ...call.Option) (*SoftwareArtifactVersion, error) { headers := http.Header{} @@ -426,7 +428,8 @@ func (c *internalClient) ListSoftwareArtifactsIter(ctx context.Context, req List } } -// Resolves a software artifact version from its tag alias. +// Resolves a software artifact version from its tag alias. Returns `NOT_FOUND` +// if the artifact or the alias does not exist. func (c *internalClient) ResolveSoftwareArtifactVersion(ctx context.Context, req ResolveSoftwareArtifactVersionRequest, opts ...call.Option) (*ResolveSoftwareArtifactVersionResponse, error) { wireReq, err := resolveSoftwareArtifactVersionRequestToWire(&req) if err != nil { diff --git a/softwareartifacts/v1/model.go b/softwareartifacts/v1/model.go index 9f5e0a39..7fe47e19 100644 --- a/softwareartifacts/v1/model.go +++ b/softwareartifacts/v1/model.go @@ -25,7 +25,8 @@ type GetSoftwareArtifactRequest struct { // Request to get a software artifact version.. type GetSoftwareArtifactVersionRequest struct { // Resource name of the version, of the form - // `software-artifacts/{catalog}.{schema}.{artifact}/versions/{version}`. + // `software-artifacts/{catalog}.{schema}.{artifact}/versions/{version}`, where + // `{version}` is the version's content digest. Name *string } @@ -82,9 +83,12 @@ type ListSoftwareArtifactsResponse struct { // Request to resolve a software artifact version from its tag alias.. type ResolveSoftwareArtifactVersionRequest struct { - // Resource name of the software artifact containing the alias. + // Resource name of the software artifact containing the alias, of the form + // `software-artifacts/{catalog}.{schema}.{artifact}`. Parent *string - // Image tag alias selecting the version. + // Image tag alias selecting the version. The alias must be a valid OCI image + // tag of at most 128 characters, made of letters, digits, underscores, periods, + // and hyphens, and must not start with a period or hyphen. Alias *string } @@ -94,18 +98,26 @@ type ResolveSoftwareArtifactVersionResponse struct { SoftwareArtifactVersion *SoftwareArtifactVersion } -// A software artifact in a Unity Catalog schema.. +// A software artifact is a named container image stored in a Unity Catalog +// schema and governed by Unity Catalog privileges. Images are pushed through +// Databricks Artifact Registry; each pushed image digest becomes an immutable +// version of the artifact, exposed as a SoftwareArtifactVersion. See +// https://docs.databricks.com/aws/en/artifact-registry/ for concepts, naming +// rules, and the privileges each operation requires.. type SoftwareArtifact struct { // Resource name of the artifact, of the form // `software-artifacts/{catalog}.{schema}.{artifact}`. Name *string `fieldmask:"name"` - // Principal name (a user, service principal, or group) that owns this artifact. + // Principal that owns the artifact, given as a user's email address, a service + // principal's application ID, or a group name. Owner *string `fieldmask:"owner"` - // Time at which this software artifact was created. + // Time at which the artifact was created, formatted as an RFC 3339 timestamp in + // JSON. CreateTime *types.Time `fieldmask:"create_time"` // Principal name of the software artifact creator. CreatedBy *string `fieldmask:"created_by"` - // Time at which this software artifact was last updated. + // Time at which the artifact was last updated, formatted as an RFC 3339 + // timestamp in JSON. UpdateTime *types.Time `fieldmask:"update_time"` // Principal name that last updated this software artifact. UpdatedBy *string `fieldmask:"updated_by"` @@ -114,10 +126,12 @@ type SoftwareArtifact struct { // An immutable version of a software artifact, identified by its content // digest.. type SoftwareArtifactVersion struct { - // Content digest of the version, for example `sha256:...`. + // Content digest of the version's image manifest, of the form + // `{algorithm}:{hex}`. Digest *string // Resource name of the version, of the form - // `software-artifacts/{catalog}.{schema}.{artifact}/versions/{version}`. + // `software-artifacts/{catalog}.{schema}.{artifact}/versions/{version}`, where + // `{version}` is the version's content digest. Name *string // Tags (aliases) that currently point at this version, for example `latest` or // `v1.2.3`. Bounded at 10 per version; a version pointed at by more tags @@ -125,11 +139,13 @@ type SoftwareArtifactVersion struct { Tags []string // Image acceleration information for this version. ImageAccelerationDetails *ImageAccelerationDetail - // Time at which this software artifact version was created. + // Time at which this software artifact version was created, formatted as an RFC + // 3339 timestamp in JSON. CreateTime *types.Time // Principal name of the software artifact version creator. CreatedBy *string - // Time at which this software artifact version was last updated. + // Time at which this software artifact version was last updated, formatted as + // an RFC 3339 timestamp in JSON. UpdateTime *types.Time // Principal name that last updated this software artifact version. UpdatedBy *string diff --git a/statementexecution/CHANGELOG.md b/statementexecution/CHANGELOG.md index 85e386a2..c10d8f34 100644 --- a/statementexecution/CHANGELOG.md +++ b/statementexecution/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.9.0 (2026-11-17) + ## Release v0.8.0 (2026-11-16) ## Release v0.7.0 (2026-11-12) diff --git a/statementexecution/internal/version.go b/statementexecution/internal/version.go index 8b7e45b3..100c2d7b 100644 --- a/statementexecution/internal/version.go +++ b/statementexecution/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-statementexecution" -const Version = "0.8.0" +const Version = "0.9.0" diff --git a/statementexecution/v1/model.go b/statementexecution/v1/model.go index 58ab59dc..32096f5e 100644 --- a/statementexecution/v1/model.go +++ b/statementexecution/v1/model.go @@ -367,7 +367,9 @@ type GetStatementResultRequest struct { // users to attribute the executions by key and optional value to filter by. // QueryTag is the user-facing representation.. type QueryTag struct { - Key *string + // The key of the tag. + Key *string + // The value of the tag. Key-only tags have no value. Value *string } diff --git a/statementexecution/v2/model.go b/statementexecution/v2/model.go index 058fb558..6151c32d 100644 --- a/statementexecution/v2/model.go +++ b/statementexecution/v2/model.go @@ -420,7 +420,9 @@ type GetStatementResultRequest struct { // users to attribute the executions by key and optional value to filter by. // QueryTag is the user-facing representation.. type QueryTag struct { - Key *string + // The key of the tag. + Key *string + // The value of the tag. Key-only tags have no value. Value *string } diff --git a/tagassignments/CHANGELOG.md b/tagassignments/CHANGELOG.md index 5692dc8b..570bb416 100644 --- a/tagassignments/CHANGELOG.md +++ b/tagassignments/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/tagassignments/internal/version.go b/tagassignments/internal/version.go index 8d7097f9..6e4001e0 100644 --- a/tagassignments/internal/version.go +++ b/tagassignments/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-tagassignments" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/tagassignments/v1/model.go b/tagassignments/v1/model.go index 9c262faa..d8bc4902 100644 --- a/tagassignments/v1/model.go +++ b/tagassignments/v1/model.go @@ -54,6 +54,12 @@ type ListTagAssignmentsResponse struct { NextPageToken *string } +// A tag assignment attaches a tag, which is a key with an optional value, to a +// workspace object such as a dashboard, notebook, Genie space or app. An object +// can have at most one assignment for each tag key. If the key belongs to a +// governed tag, the caller needs the ASSIGN permission on that governed tag, +// and the value must be one of its allowed values. See +// https://docs.databricks.com/aws/en/admin/governed-tags/.. type TagAssignment struct { // The type of entity to which the tag is assigned. Allowed values are apps, // dashboards, geniespaces, notebooks @@ -64,11 +70,15 @@ type TagAssignment struct { // The key of the tag. The characters , . : / - = and leading/trailing spaces // are not allowed TagKey *string `fieldmask:"tag_key"` - // The value of the tag + // The value of the tag. If the value is omitted, the tag is assigned with its + // key only. TagValue *string `fieldmask:"tag_value"` } type UpdateTagAssignmentRequest struct { TagAssignment *TagAssignment - UpdateMask *types.FieldMask[TagAssignment] + // The comma-separated list of fields to update. Only `tag_value` can be + // updated, because the entity type, entity ID and tag key identify the tag + // assignment. + UpdateMask *types.FieldMask[TagAssignment] } diff --git a/uc/connections/CHANGELOG.md b/uc/connections/CHANGELOG.md index d80ae434..1008e240 100644 --- a/uc/connections/CHANGELOG.md +++ b/uc/connections/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.9.0 (2026-11-17) + ## Release v0.8.0 (2026-11-12) ### New Features and Improvements diff --git a/uc/connections/internal/version.go b/uc/connections/internal/version.go index c9d69f74..2e46ef05 100644 --- a/uc/connections/internal/version.go +++ b/uc/connections/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-uc-connections" -const Version = "0.8.0" +const Version = "0.9.0" diff --git a/uc/connections/v1/model.go b/uc/connections/v1/model.go index 59bf13c8..5fecf6a8 100644 --- a/uc/connections/v1/model.go +++ b/uc/connections/v1/model.go @@ -30,14 +30,15 @@ const ( ConnectionType_Confluence ConnectionType = "CONFLUENCE" ConnectionType_Jdbc ConnectionType = "JDBC" ConnectionType_MetaMarketing ConnectionType = "META_MARKETING" - ConnectionType_TiktokAds ConnectionType = "TIKTOK_ADS" - ConnectionType_Hubspot ConnectionType = "HUBSPOT" - ConnectionType_Zendesk ConnectionType = "ZENDESK" - ConnectionType_AwsSecretsManager ConnectionType = "AWS_SECRETS_MANAGER" - ConnectionType_AzureKeyVault ConnectionType = "AZURE_KEY_VAULT" - ConnectionType_Github ConnectionType = "GITHUB" - ConnectionType_Outlook ConnectionType = "OUTLOOK" - ConnectionType_Smartsheet ConnectionType = "SMARTSHEET" + // Connection to TikTok Ads for accessing advertising data. + ConnectionType_TiktokAds ConnectionType = "TIKTOK_ADS" + ConnectionType_Hubspot ConnectionType = "HUBSPOT" + ConnectionType_Zendesk ConnectionType = "ZENDESK" + ConnectionType_AwsSecretsManager ConnectionType = "AWS_SECRETS_MANAGER" + ConnectionType_AzureKeyVault ConnectionType = "AZURE_KEY_VAULT" + ConnectionType_Github ConnectionType = "GITHUB" + ConnectionType_Outlook ConnectionType = "OUTLOOK" + ConnectionType_Smartsheet ConnectionType = "SMARTSHEET" ) type CredentialType string diff --git a/uc/schemas/CHANGELOG.md b/uc/schemas/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/uc/schemas/CHANGELOG.md +++ b/uc/schemas/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/uc/schemas/internal/version.go b/uc/schemas/internal/version.go index 26df14c3..c0bde813 100644 --- a/uc/schemas/internal/version.go +++ b/uc/schemas/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-uc-schemas" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/uc/schemas/v1/model.go b/uc/schemas/v1/model.go index 3cb89c76..c679096f 100644 --- a/uc/schemas/v1/model.go +++ b/uc/schemas/v1/model.go @@ -20,14 +20,23 @@ type CreateSchemaRequest struct { Name *string // Name of parent catalog. CatalogName *string - // Username of current owner of schema. + // The user or group that owns the schema. The creator becomes the owner, and + // the owner can be changed only through the update method. Owner *string // User-provided free-form text description. Comment *string - // Storage root URL for managed tables within schema. + // Managed tables and volumes in the schema are stored under this storage root + // URL. The path must be inside an external location on which the caller has the + // CREATE MANAGED STORAGE privilege. If omitted, managed data is stored under + // the parent catalog's storage root, or under the metastore's root storage + // location when the catalog has none. Schemas in Delta Sharing catalogs cannot + // have a storage root, and schemas in a federated Hive metastore catalog must + // set one. StorageRoot *string - // Whether predictive optimization should be enabled for this object and objects - // under it. + // Whether predictive optimization is enabled for this schema and the objects + // under it. Valid values are ENABLE, DISABLE, and INHERIT, where INHERIT + // applies the parent catalog's setting. New schemas start as INHERIT. It cannot + // be set when creating a schema; use the update method instead. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -37,22 +46,30 @@ type CreateSchemaRequest struct { CreatedAt *int64 // Username of schema creator. CreatedBy *string - // Time at which this schema was created, in epoch milliseconds. + // Time at which this schema was last modified, in epoch milliseconds. UpdatedAt *int64 // Username of user who last modified schema. UpdatedBy *string - // The type of the parent catalog. + // The type of the parent catalog, which the server copies from the catalog. CatalogType CatalogType - // Storage location for managed tables within schema. - StorageLocation *string + // The server derives this full storage location URL for managed tables and + // volumes within the schema from storage_root and the schema ID. It is absent + // when the schema has no storage root of its own. + StorageLocation *string + // Indicates the predictive optimization setting that is in effect for the + // schema once inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The unique identifier of the schema. + // The unique identifier (UUID) that the server assigns to the schema when it is + // created. SchemaId *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled // in the request. BrowseOnly *bool - // Custom maximum retention period in hours for the schema. + // Dropped objects in the schema remain recoverable for this number of hours + // before they are permanently purged. The value must be 0 or within the range + // allowed for the metastore. Schemas that do not set their own value inherit + // the parent catalog's value. CustomMaxRetentionHours *int64 // A map of key-value properties attached to the securable. Properties map[string]string @@ -61,7 +78,7 @@ type CreateSchemaRequest struct { } type DeleteSchemaRequest struct { - // Full name of the schema. + // Full name of the schema, in the form catalog_name.schema_name. FullNameArg *string // Force deletion even if the schema is not empty. Force *bool @@ -83,10 +100,11 @@ type EffectivePredictiveOptimizationFlag struct { } type GetSchemaRequest struct { - // Full name of the schema. + // Full name of the schema, in the form catalog_name.schema_name. FullNameArg *string - // Whether to include schemas in the response for which the principal can only - // access selective metadata for + // Whether to return the schema when the principal can only access selective + // metadata for it through the BROWSE privilege. The response then has + // browse_only set to true. IncludeBrowse *bool } @@ -101,8 +119,9 @@ type ListSchemasRequest struct { MaxResults *int // Opaque pagination token to go to next page based on previous query. PageToken *string - // Whether to include schemas in the response for which the principal can only - // access selective metadata for + // Whether to include schemas for which the principal can only access selective + // metadata through the BROWSE privilege. Such schemas are returned with + // browse_only set to true. IncludeBrowse *bool } @@ -115,19 +134,33 @@ type ListSchemasResponse struct { NextPageToken *string } +// A schema (also called a database) is the second layer of Unity Catalog's +// three-level namespace and groups the tables, views, volumes, models, and +// functions under its parent catalog. Managed tables and volumes in the schema +// are stored under the schema's storage root when it has one, and otherwise +// under the storage of the parent catalog or the metastore.. type SchemaInfo struct { // Name of schema, relative to parent catalog. Name *string // Name of parent catalog. CatalogName *string - // Username of current owner of schema. + // The user or group that owns the schema. The creator becomes the owner, and + // the owner can be changed only through the update method. Owner *string // User-provided free-form text description. Comment *string - // Storage root URL for managed tables within schema. + // Managed tables and volumes in the schema are stored under this storage root + // URL. The path must be inside an external location on which the caller has the + // CREATE MANAGED STORAGE privilege. If omitted, managed data is stored under + // the parent catalog's storage root, or under the metastore's root storage + // location when the catalog has none. Schemas in Delta Sharing catalogs cannot + // have a storage root, and schemas in a federated Hive metastore catalog must + // set one. StorageRoot *string - // Whether predictive optimization should be enabled for this object and objects - // under it. + // Whether predictive optimization is enabled for this schema and the objects + // under it. Valid values are ENABLE, DISABLE, and INHERIT, where INHERIT + // applies the parent catalog's setting. New schemas start as INHERIT. It cannot + // be set when creating a schema; use the update method instead. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -137,22 +170,30 @@ type SchemaInfo struct { CreatedAt *int64 // Username of schema creator. CreatedBy *string - // Time at which this schema was created, in epoch milliseconds. + // Time at which this schema was last modified, in epoch milliseconds. UpdatedAt *int64 // Username of user who last modified schema. UpdatedBy *string - // The type of the parent catalog. + // The type of the parent catalog, which the server copies from the catalog. CatalogType CatalogType - // Storage location for managed tables within schema. - StorageLocation *string + // The server derives this full storage location URL for managed tables and + // volumes within the schema from storage_root and the schema ID. It is absent + // when the schema has no storage root of its own. + StorageLocation *string + // Indicates the predictive optimization setting that is in effect for the + // schema once inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The unique identifier of the schema. + // The unique identifier (UUID) that the server assigns to the schema when it is + // created. SchemaId *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled // in the request. BrowseOnly *bool - // Custom maximum retention period in hours for the schema. + // Dropped objects in the schema remain recoverable for this number of hours + // before they are permanently purged. The value must be 0 or within the range + // allowed for the metastore. Schemas that do not set their own value inherit + // the parent catalog's value. CustomMaxRetentionHours *int64 // A map of key-value properties attached to the securable. Properties map[string]string @@ -161,7 +202,7 @@ type SchemaInfo struct { } type UpdateSchemaRequest struct { - // Full name of the schema. + // Full name of the schema, in the form catalog_name.schema_name. FullNameArg *string // New name for the schema. NewName *string @@ -169,14 +210,23 @@ type UpdateSchemaRequest struct { Name *string // Name of parent catalog. CatalogName *string - // Username of current owner of schema. + // The user or group that owns the schema. The creator becomes the owner, and + // the owner can be changed only through the update method. Owner *string // User-provided free-form text description. Comment *string - // Storage root URL for managed tables within schema. + // Managed tables and volumes in the schema are stored under this storage root + // URL. The path must be inside an external location on which the caller has the + // CREATE MANAGED STORAGE privilege. If omitted, managed data is stored under + // the parent catalog's storage root, or under the metastore's root storage + // location when the catalog has none. Schemas in Delta Sharing catalogs cannot + // have a storage root, and schemas in a federated Hive metastore catalog must + // set one. StorageRoot *string - // Whether predictive optimization should be enabled for this object and objects - // under it. + // Whether predictive optimization is enabled for this schema and the objects + // under it. Valid values are ENABLE, DISABLE, and INHERIT, where INHERIT + // applies the parent catalog's setting. New schemas start as INHERIT. It cannot + // be set when creating a schema; use the update method instead. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -186,22 +236,30 @@ type UpdateSchemaRequest struct { CreatedAt *int64 // Username of schema creator. CreatedBy *string - // Time at which this schema was created, in epoch milliseconds. + // Time at which this schema was last modified, in epoch milliseconds. UpdatedAt *int64 // Username of user who last modified schema. UpdatedBy *string - // The type of the parent catalog. + // The type of the parent catalog, which the server copies from the catalog. CatalogType CatalogType - // Storage location for managed tables within schema. - StorageLocation *string + // The server derives this full storage location URL for managed tables and + // volumes within the schema from storage_root and the schema ID. It is absent + // when the schema has no storage root of its own. + StorageLocation *string + // Indicates the predictive optimization setting that is in effect for the + // schema once inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The unique identifier of the schema. + // The unique identifier (UUID) that the server assigns to the schema when it is + // created. SchemaId *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled // in the request. BrowseOnly *bool - // Custom maximum retention period in hours for the schema. + // Dropped objects in the schema remain recoverable for this number of hours + // before they are permanently purged. The value must be 0 or within the range + // allowed for the metastore. Schemas that do not set their own value inherit + // the parent catalog's value. CustomMaxRetentionHours *int64 // A map of key-value properties attached to the securable. Properties map[string]string diff --git a/uc/systemschemas/CHANGELOG.md b/uc/systemschemas/CHANGELOG.md index 343b52e7..fbb6e4d6 100644 --- a/uc/systemschemas/CHANGELOG.md +++ b/uc/systemschemas/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.7.0 (2026-11-17) + ## Release v0.6.0 (2026-11-12) ### New Features and Improvements diff --git a/uc/systemschemas/internal/version.go b/uc/systemschemas/internal/version.go index da052317..2f34bf6e 100644 --- a/uc/systemschemas/internal/version.go +++ b/uc/systemschemas/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-uc-systemschemas" -const Version = "0.6.0" +const Version = "0.7.0" diff --git a/uc/systemschemas/v1/model.go b/uc/systemschemas/v1/model.go index c3dc3e30..b5de3e14 100644 --- a/uc/systemschemas/v1/model.go +++ b/uc/systemschemas/v1/model.go @@ -3,7 +3,7 @@ package systemschemas type DisableSystemSchemaRequest struct { - // Full name of the system schema. + // Name of the system schema, without the catalog prefix. Schema *string // The metastore ID under which the system schema lives. MetastoreId *string @@ -13,11 +13,13 @@ type DisableSystemSchemaResponse struct { } type EnableSystemSchemaRequest struct { - // Full name of the system schema. + // Name of the system schema, without the catalog prefix. Schema *string // The metastore ID under which the system schema lives. MetastoreId *string - // the catalog for which the system schema is to enabled in + // Name of the catalog in which to enable the system schema. The supported + // values are `system` and `samples`. If not set, the system schema is enabled + // in the `system` catalog. CatalogName *string } @@ -46,12 +48,19 @@ type ListSystemSchemasResponse struct { NextPageToken *string } +// A system schema of a metastore, with its current enablement state. System +// schemas hold Databricks-provided system tables, such as audit logs and +// billing usage. They live in the `system` catalog unless they are enabled in +// another supported catalog. See +// https://docs.databricks.com/aws/en/admin/system-tables/.. type SystemSchemaInfo struct { - // Name of the system schema. + // Name of the system schema, without the catalog prefix. Schema *string - // The current state of enablement for the system schema. An empty string means - // the system schema is available and ready for opt-in. Possible values: - // AVAILABLE | ENABLE_INITIALIZED | ENABLE_COMPLETED | DISABLE_INITIALIZED | - // UNAVAILABLE | MANAGED + // The current enablement state of the system schema. Possible values are + // `AVAILABLE` (the schema can be enabled), `ENABLE_INITIALIZED` (enablement is + // in progress), `ENABLE_COMPLETED` (the schema is enabled), + // `DISABLE_INITIALIZED` (disablement is in progress), `UNAVAILABLE` (the schema + // cannot be enabled at the moment) and `MANAGED` (Databricks enables the schema + // automatically). State *string } diff --git a/uc/tables/CHANGELOG.md b/uc/tables/CHANGELOG.md index 251e37ea..dc2db25a 100644 --- a/uc/tables/CHANGELOG.md +++ b/uc/tables/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.10.0 (2026-11-17) + ## Release v0.9.0 (2026-11-16) ## Release v0.8.0 (2026-11-12) diff --git a/uc/tables/internal/version.go b/uc/tables/internal/version.go index 009c3b0f..6481de90 100644 --- a/uc/tables/internal/version.go +++ b/uc/tables/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-uc-tables" -const Version = "0.9.0" +const Version = "0.10.0" diff --git a/uc/tables/v1/model.go b/uc/tables/v1/model.go index 811071e4..5f2b73e5 100644 --- a/uc/tables/v1/model.go +++ b/uc/tables/v1/model.go @@ -258,6 +258,9 @@ type ColumnInfo struct { Name *string // Full data type specification as SQL/catalogString text. TypeText *string + // The name of the column's data type, for example STRING, DECIMAL or STRUCT. + // The complete type, including parameters such as the precision of a decimal or + // the element type of an array, is in type_text. TypeName ColumnTypeName // Ordinal position of column (starting at position 0). Position *int @@ -275,7 +278,11 @@ type ColumnInfo struct { Nullable *bool // Partition index for column. PartitionIndex *int - Mask *ColumnMask + // The column mask function applied to this column, if any. A column mask is a + // SQL user-defined function that transforms the values a query returns for the + // column. See https://docs.databricks.com/aws/en/tables/row-and-column-filters + // for details. + Mask *ColumnMask } type ColumnMask struct { @@ -310,10 +317,24 @@ type CreateTableRequest struct { // Name of parent catalog. CatalogName *string // Name of parent schema relative to its parent catalog. - SchemaName *string - TableType TableType + SchemaName *string + // The type of the table. MANAGED tables store their data in storage that Unity + // Catalog manages, EXTERNAL tables store it in a registered external location, + // FOREIGN tables mirror objects of an external database through a connection, + // and the shallow clone types reference the data files of a base table. VIEW, + // MATERIALIZED_VIEW, STREAMING_TABLE and METRIC_VIEW are defined by a view + // definition. See https://docs.databricks.com/aws/en/tables/ for an overview of + // the table types. + TableType TableType + // The format of the data that backs the table, such as DELTA, PARQUET, CSV or + // JSON. Foreign tables report the external system that holds them, for example + // SNOWFLAKE_FORMAT or MYSQL_FORMAT. DataSourceFormat DataSourceFormat - // Storage root URL for table (for **MANAGED**, **EXTERNAL** tables). + // The cloud storage URL under which the data files of the table are stored. For + // an EXTERNAL table, it must be set on creation to a path within an existing + // external location. For a MANAGED table, the server assigns a path under the + // managed storage location of the parent schema, catalog or metastore and + // returns it in this field. Views have no storage location. StorageLocation *string // View definition SQL (when __table_type__ is **VIEW**, **MATERIALIZED_VIEW**, // or **STREAMING_TABLE**) @@ -327,7 +348,8 @@ type CreateTableRequest struct { ViewDependencies *DependencyList // List of schemes whose objects can be referenced without qualification. SqlPath *string - // Username of current owner of table. + // The user or group that owns the table. The principal that creates the table + // becomes its owner. Owner *string // User-provided free-form text description. Comment *string @@ -337,10 +359,18 @@ type CreateTableRequest struct { // List of table constraints. Note: this field is not set in the output of the // __listTables__ API. TableConstraints []TableConstraint - RowFilter *RowFilter + // The row filter function applied to the table, if any. A row filter is a SQL + // user-defined function that decides which rows a query returns. See + // https://docs.databricks.com/aws/en/tables/row-and-column-filters for details. + RowFilter *RowFilter // The pipeline ID of the table. Applicable for tables created by pipelines // (Materialized View, Streaming Table, etc.). - PipelineId *string + PipelineId *string + // Whether predictive optimization is enabled for this table. Valid values are + // ENABLE, DISABLE and INHERIT, where INHERIT applies the setting of the parent + // schema. It is ignored when creating a table. See + // https://docs.databricks.com/aws/en/optimizations/predictive-optimization for + // details. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -363,9 +393,12 @@ type CreateTableRequest struct { DeltaRuntimePropertiesKvpairs *DeltaRuntimePropertiesKvPairs // Time at which this table was deleted, in epoch milliseconds. Field is omitted // if table is not deleted. - DeletedAt *int64 + DeletedAt *int64 + // The predictive optimization setting that is in effect for the table once + // inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The AWS access point to use when accesing s3 for this external location. + // The AWS access point to use when accessing S3 for the storage location of + // this table. AccessPoint *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled @@ -759,16 +792,34 @@ type TableExistsResponse struct { TableExists *bool } +// A table resides in the third layer of Unity Catalog's three-level namespace +// and holds rows of data organized in columns. Besides managed and external +// tables, it covers views, materialized views, streaming tables, metric views, +// foreign tables and shallow clones, which table_type distinguishes.. type TableInfo struct { // Name of table, relative to parent schema. Name *string // Name of parent catalog. CatalogName *string // Name of parent schema relative to its parent catalog. - SchemaName *string - TableType TableType + SchemaName *string + // The type of the table. MANAGED tables store their data in storage that Unity + // Catalog manages, EXTERNAL tables store it in a registered external location, + // FOREIGN tables mirror objects of an external database through a connection, + // and the shallow clone types reference the data files of a base table. VIEW, + // MATERIALIZED_VIEW, STREAMING_TABLE and METRIC_VIEW are defined by a view + // definition. See https://docs.databricks.com/aws/en/tables/ for an overview of + // the table types. + TableType TableType + // The format of the data that backs the table, such as DELTA, PARQUET, CSV or + // JSON. Foreign tables report the external system that holds them, for example + // SNOWFLAKE_FORMAT or MYSQL_FORMAT. DataSourceFormat DataSourceFormat - // Storage root URL for table (for **MANAGED**, **EXTERNAL** tables). + // The cloud storage URL under which the data files of the table are stored. For + // an EXTERNAL table, it must be set on creation to a path within an existing + // external location. For a MANAGED table, the server assigns a path under the + // managed storage location of the parent schema, catalog or metastore and + // returns it in this field. Views have no storage location. StorageLocation *string // View definition SQL (when __table_type__ is **VIEW**, **MATERIALIZED_VIEW**, // or **STREAMING_TABLE**) @@ -782,7 +833,8 @@ type TableInfo struct { ViewDependencies *DependencyList // List of schemes whose objects can be referenced without qualification. SqlPath *string - // Username of current owner of table. + // The user or group that owns the table. The principal that creates the table + // becomes its owner. Owner *string // User-provided free-form text description. Comment *string @@ -792,10 +844,18 @@ type TableInfo struct { // List of table constraints. Note: this field is not set in the output of the // __listTables__ API. TableConstraints []TableConstraint - RowFilter *RowFilter + // The row filter function applied to the table, if any. A row filter is a SQL + // user-defined function that decides which rows a query returns. See + // https://docs.databricks.com/aws/en/tables/row-and-column-filters for details. + RowFilter *RowFilter // The pipeline ID of the table. Applicable for tables created by pipelines // (Materialized View, Streaming Table, etc.). - PipelineId *string + PipelineId *string + // Whether predictive optimization is enabled for this table. Valid values are + // ENABLE, DISABLE and INHERIT, where INHERIT applies the setting of the parent + // schema. It is ignored when creating a table. See + // https://docs.databricks.com/aws/en/optimizations/predictive-optimization for + // details. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -818,9 +878,12 @@ type TableInfo struct { DeltaRuntimePropertiesKvpairs *DeltaRuntimePropertiesKvPairs // Time at which this table was deleted, in epoch milliseconds. Field is omitted // if table is not deleted. - DeletedAt *int64 + DeletedAt *int64 + // The predictive optimization setting that is in effect for the table once + // inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The AWS access point to use when accesing s3 for this external location. + // The AWS access point to use when accessing S3 for the storage location of + // this table. AccessPoint *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled @@ -837,7 +900,8 @@ type TableInfo struct { type TableSummary struct { // The full name of the table. - FullName *string + FullName *string + // The type of the table, with the same meaning as table_type on TableInfo. TableType TableType // SecurableKindManifest of table, including capabilities the table has. SecurableKindManifest *SecurableKindManifest @@ -851,10 +915,24 @@ type UpdateTableRequest struct { // Name of parent catalog. CatalogName *string // Name of parent schema relative to its parent catalog. - SchemaName *string - TableType TableType + SchemaName *string + // The type of the table. MANAGED tables store their data in storage that Unity + // Catalog manages, EXTERNAL tables store it in a registered external location, + // FOREIGN tables mirror objects of an external database through a connection, + // and the shallow clone types reference the data files of a base table. VIEW, + // MATERIALIZED_VIEW, STREAMING_TABLE and METRIC_VIEW are defined by a view + // definition. See https://docs.databricks.com/aws/en/tables/ for an overview of + // the table types. + TableType TableType + // The format of the data that backs the table, such as DELTA, PARQUET, CSV or + // JSON. Foreign tables report the external system that holds them, for example + // SNOWFLAKE_FORMAT or MYSQL_FORMAT. DataSourceFormat DataSourceFormat - // Storage root URL for table (for **MANAGED**, **EXTERNAL** tables). + // The cloud storage URL under which the data files of the table are stored. For + // an EXTERNAL table, it must be set on creation to a path within an existing + // external location. For a MANAGED table, the server assigns a path under the + // managed storage location of the parent schema, catalog or metastore and + // returns it in this field. Views have no storage location. StorageLocation *string // View definition SQL (when __table_type__ is **VIEW**, **MATERIALIZED_VIEW**, // or **STREAMING_TABLE**) @@ -868,7 +946,8 @@ type UpdateTableRequest struct { ViewDependencies *DependencyList // List of schemes whose objects can be referenced without qualification. SqlPath *string - // Username of current owner of table. + // The user or group that owns the table. The principal that creates the table + // becomes its owner. Owner *string // User-provided free-form text description. Comment *string @@ -878,10 +957,18 @@ type UpdateTableRequest struct { // List of table constraints. Note: this field is not set in the output of the // __listTables__ API. TableConstraints []TableConstraint - RowFilter *RowFilter + // The row filter function applied to the table, if any. A row filter is a SQL + // user-defined function that decides which rows a query returns. See + // https://docs.databricks.com/aws/en/tables/row-and-column-filters for details. + RowFilter *RowFilter // The pipeline ID of the table. Applicable for tables created by pipelines // (Materialized View, Streaming Table, etc.). - PipelineId *string + PipelineId *string + // Whether predictive optimization is enabled for this table. Valid values are + // ENABLE, DISABLE and INHERIT, where INHERIT applies the setting of the parent + // schema. It is ignored when creating a table. See + // https://docs.databricks.com/aws/en/optimizations/predictive-optimization for + // details. EnablePredictiveOptimization *string // Unique identifier of parent metastore. MetastoreId *string @@ -904,9 +991,12 @@ type UpdateTableRequest struct { DeltaRuntimePropertiesKvpairs *DeltaRuntimePropertiesKvPairs // Time at which this table was deleted, in epoch milliseconds. Field is omitted // if table is not deleted. - DeletedAt *int64 + DeletedAt *int64 + // The predictive optimization setting that is in effect for the table once + // inherited values are resolved. EffectivePredictiveOptimizationFlag *EffectivePredictiveOptimizationFlag - // The AWS access point to use when accesing s3 for this external location. + // The AWS access point to use when accessing S3 for the storage location of + // this table. AccessPoint *string // Indicates whether the principal is limited to retrieving metadata for the // associated object through the BROWSE privilege when include_browse is enabled diff --git a/vectorsearch/CHANGELOG.md b/vectorsearch/CHANGELOG.md index 24bec23e..f40113c7 100644 --- a/vectorsearch/CHANGELOG.md +++ b/vectorsearch/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.11.0 (2026-11-17) + ## Release v0.10.0 (2026-11-16) ## Release v0.9.0 (2026-11-12) diff --git a/vectorsearch/internal/version.go b/vectorsearch/internal/version.go index 840095d8..2f2c53cb 100644 --- a/vectorsearch/internal/version.go +++ b/vectorsearch/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-vectorsearch" -const Version = "0.10.0" +const Version = "0.11.0" diff --git a/vectorsearch/v1/model.go b/vectorsearch/v1/model.go index 905470e6..bfabed79 100644 --- a/vectorsearch/v1/model.go +++ b/vectorsearch/v1/model.go @@ -17,6 +17,44 @@ const ( EndpointType_Standard EndpointType = "STANDARD" ) +// Lifecycle and ingestion state of an index. +type IndexStatus string + +const ( + IndexStatus_Unspecified IndexStatus = "" + // Provisioning the serving endpoint. + IndexStatus_ProvisioningEndpoint IndexStatus = "PROVISIONING_ENDPOINT" + // Setting up a direct access index. + IndexStatus_ProvisioningIndex IndexStatus = "PROVISIONING_INDEX" + // Acquiring compute resources for the ingestion pipeline. + IndexStatus_ProvisioningPipelineResources IndexStatus = "PROVISIONING_PIPELINE_RESOURCES" + // Syncing the initial snapshot of the source Delta table. + IndexStatus_ProvisioningInitialSnapshot IndexStatus = "PROVISIONING_INITIAL_SNAPSHOT" + // The index can serve queries; more specific ingestion status is unavailable. + IndexStatus_Online IndexStatus = "ONLINE" + // The index can serve queries while the continuous pipeline processes updates. + IndexStatus_OnlineContinuousUpdate IndexStatus = "ONLINE_CONTINUOUS_UPDATE" + // The index can serve queries while a triggered update is in progress. + IndexStatus_OnlineTriggeredUpdate IndexStatus = "ONLINE_TRIGGERED_UPDATE" + // The index can serve queries and no triggered update is currently reported as + // active. + IndexStatus_OnlineNoPendingUpdate IndexStatus = "ONLINE_NO_PENDING_UPDATE" + // The direct access index can serve queries. + IndexStatus_OnlineDirectAccess IndexStatus = "ONLINE_DIRECT_ACCESS" + // The index cannot currently serve queries. + IndexStatus_Offline IndexStatus = "OFFLINE" + // The index cannot serve queries because of a failure. + IndexStatus_OfflineFailed IndexStatus = "OFFLINE_FAILED" + // The latest pipeline update failed, but the index can still serve previously + // indexed data. + IndexStatus_OnlinePipelineFailed IndexStatus = "ONLINE_PIPELINE_FAILED" + // The index can serve queries while its pipeline restarts and acquires compute + // resources. + IndexStatus_OnlineUpdatingPipelineResources IndexStatus = "ONLINE_UPDATING_PIPELINE_RESOURCES" + // Downloading index data onto serving compute. + IndexStatus_Downloading IndexStatus = "DOWNLOADING" +) + // The subtype of the AI Search index, determining the indexing and retrieval // strategy. - `VECTOR`: Not supported. Use `HYBRID` instead. - `FULL_TEXT`: An // index that uses full-text search without vector embeddings. - `HYBRID`: An @@ -421,6 +459,21 @@ type FacetResultData struct { FacetArray [][]json.RawMessage } +// Failure details for an index or its latest ingestion update.. +type FailedStatus struct { + // Error message, when available. + ErrorMessage *string + // Last source Delta table version fully processed while the index is still + // queryable. + LastProcessedCommitVersion *int64 + // [Standard endpoint] timestamp when the last processed commit was committed to + // the source table. [Storage Optimized endpoint] timestamp when the last + // processed commit was processed by the pipeline. + LastProcessedCommitTimestamp *types.Time + // Error code associated with the error message, when available. + ErrorCode *string +} + type GetEndpointRequest struct { // Name of the endpoint Name *string @@ -861,12 +914,31 @@ type VectorIndex_IndexSpec_DeltaSyncIndexSpec struct { func (*VectorIndex_IndexSpec_DeltaSyncIndexSpec) isVectorIndex_IndexSpec() {} type VectorIndexStatus struct { + // Lifecycle and ingestion state. An online index can still have a failed + // update. + DetailedState IndexStatus // Message associated with the index status Message *string // Number of rows indexed IndexedRowCount *int64 - // Whether the index is ready for search + DetailedStatus isVectorIndexStatus_DetailedStatus + // Whether the index can serve queries. This does not mean the latest update + // succeeded or that all source-table changes have been indexed; check + // detailed_state for update failures. Ready *bool // Index API Url to be used to perform operations on the index IndexUrl *string } + +type isVectorIndexStatus_DetailedStatus interface { + isVectorIndexStatus_DetailedStatus() +} + +// VectorIndexStatus_DetailedStatus_FailedStatus selects FailedStatus for VectorIndexStatus.DetailedStatus. +// Failure details, when available, for OFFLINE_FAILED or +// ONLINE_PIPELINE_FAILED. +type VectorIndexStatus_DetailedStatus_FailedStatus struct { + FailedStatus FailedStatus +} + +func (*VectorIndexStatus_DetailedStatus_FailedStatus) isVectorIndexStatus_DetailedStatus() {} diff --git a/vectorsearch/v1/wire.go b/vectorsearch/v1/wire.go index 219fe7ac..4f158945 100644 --- a/vectorsearch/v1/wire.go +++ b/vectorsearch/v1/wire.go @@ -1274,6 +1274,69 @@ func (v *FacetResultData) UnmarshalJSON(data []byte) error { return nil } +type failedStatusWire struct { + ErrorMessage *string `json:"error_message,omitempty"` + LastProcessedCommitVersion *wireInt64 `json:"last_processed_commit_version,omitempty"` + LastProcessedCommitTimestamp *types.Time `json:"last_processed_commit_timestamp,omitempty"` + ErrorCode *string `json:"error_code,omitempty"` +} + +func failedStatusToWire(v *FailedStatus) (*failedStatusWire, error) { + if v == nil { + return nil, nil + } + lastProcessedCommitVersionWireValue, err := int64ToWire(v.LastProcessedCommitVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "FailedStatus.LastProcessedCommitVersion", err) + } + return &failedStatusWire{ + ErrorMessage: v.ErrorMessage, + LastProcessedCommitVersion: lastProcessedCommitVersionWireValue, + LastProcessedCommitTimestamp: v.LastProcessedCommitTimestamp, + ErrorCode: v.ErrorCode, + }, nil +} + +func failedStatusFromWire(w *failedStatusWire) (*FailedStatus, error) { + if w == nil { + return nil, nil + } + lastProcessedCommitVersionPublicValue, err := int64FromWire(w.LastProcessedCommitVersion) + if err != nil { + return nil, fmt.Errorf("%s: %w", "FailedStatus.LastProcessedCommitVersion", err) + } + return &FailedStatus{ + ErrorMessage: w.ErrorMessage, + LastProcessedCommitVersion: lastProcessedCommitVersionPublicValue, + LastProcessedCommitTimestamp: w.LastProcessedCommitTimestamp, + ErrorCode: w.ErrorCode, + }, nil +} + +// MarshalJSON returns the JSON form that the API uses for this message, with +// fields that a request sends in the URL path or query as regular fields. +func (v FailedStatus) MarshalJSON() ([]byte, error) { + w, err := failedStatusToWire(&v) + if err != nil { + return nil, err + } + return json.Marshal(w) +} + +// UnmarshalJSON sets the message from the JSON that MarshalJSON writes. +func (v *FailedStatus) UnmarshalJSON(data []byte) error { + var w failedStatusWire + if err := json.Unmarshal(data, &w); err != nil { + return err + } + decoded, err := failedStatusFromWire(&w) + if err != nil { + return err + } + *v = *decoded + return nil +} + type getEndpointRequestWire struct { Name *string `json:"name,omitempty"` } @@ -3653,10 +3716,12 @@ func (v *VectorIndex) UnmarshalJSON(data []byte) error { } type vectorIndexStatusWire struct { - Message *string `json:"message,omitempty"` - IndexedRowCount *wireInt64 `json:"indexed_row_count,omitempty"` - Ready *bool `json:"ready,omitempty"` - IndexUrl *string `json:"index_url,omitempty"` + DetailedState IndexStatus `json:"detailed_state,omitempty"` + Message *string `json:"message,omitempty"` + IndexedRowCount *wireInt64 `json:"indexed_row_count,omitempty"` + FailedStatus *failedStatusWire `json:"failed_status,omitempty"` + Ready *bool `json:"ready,omitempty"` + IndexUrl *string `json:"index_url,omitempty"` } func vectorIndexStatusToWire(v *VectorIndexStatus) (*vectorIndexStatusWire, error) { @@ -3667,9 +3732,25 @@ func vectorIndexStatusToWire(v *VectorIndexStatus) (*vectorIndexStatusWire, erro if err != nil { return nil, fmt.Errorf("%s: %w", "VectorIndexStatus.IndexedRowCount", err) } + var detailedStatusFailedStatusWire *failedStatusWire + switch value := v.DetailedStatus.(type) { + case nil: + case *VectorIndexStatus_DetailedStatus_FailedStatus: + if value != nil { + detailedStatusFailedStatusConverted, err := failedStatusToWire(&value.FailedStatus) + if err != nil { + return nil, fmt.Errorf("%s: %w", "VectorIndexStatus.DetailedStatus.FailedStatus", err) + } + detailedStatusFailedStatusWire = detailedStatusFailedStatusConverted + } + default: + return nil, fmt.Errorf("%s: unsupported oneof implementation %T", "VectorIndexStatus.DetailedStatus", value) + } return &vectorIndexStatusWire{ + DetailedState: v.DetailedState, Message: v.Message, IndexedRowCount: indexedRowCountWireValue, + FailedStatus: detailedStatusFailedStatusWire, Ready: v.Ready, IndexUrl: v.IndexUrl, }, nil @@ -3679,15 +3760,33 @@ func vectorIndexStatusFromWire(w *vectorIndexStatusWire) (*VectorIndexStatus, er if w == nil { return nil, nil } + detailedStatusMembers := 0 + if w.FailedStatus != nil { + detailedStatusMembers++ + } + if detailedStatusMembers > 1 { + return nil, fmt.Errorf("%s: multiple oneof members set", "VectorIndexStatus.DetailedStatus") + } indexedRowCountPublicValue, err := int64FromWire(w.IndexedRowCount) if err != nil { return nil, fmt.Errorf("%s: %w", "VectorIndexStatus.IndexedRowCount", err) } + var detailedStatusSelection isVectorIndexStatus_DetailedStatus + switch { + case w.FailedStatus != nil: + detailedStatusFailedStatusConverted, err := failedStatusFromWire(w.FailedStatus) + if err != nil { + return nil, fmt.Errorf("%s: %w", "VectorIndexStatus.DetailedStatus.FailedStatus", err) + } + detailedStatusSelection = &VectorIndexStatus_DetailedStatus_FailedStatus{FailedStatus: *detailedStatusFailedStatusConverted} + } return &VectorIndexStatus{ + DetailedState: w.DetailedState, Message: w.Message, IndexedRowCount: indexedRowCountPublicValue, Ready: w.Ready, IndexUrl: w.IndexUrl, + DetailedStatus: detailedStatusSelection, }, nil } diff --git a/warehouses/CHANGELOG.md b/warehouses/CHANGELOG.md index c8867401..917e4106 100644 --- a/warehouses/CHANGELOG.md +++ b/warehouses/CHANGELOG.md @@ -1,5 +1,7 @@ # Version changelog +## Release v0.11.0 (2026-11-17) + ## Release v0.10.0 (2026-11-12) ### New Features and Improvements diff --git a/warehouses/internal/version.go b/warehouses/internal/version.go index a095847d..2f3d0df7 100644 --- a/warehouses/internal/version.go +++ b/warehouses/internal/version.go @@ -2,4 +2,4 @@ package internal const ModuleName = "sdk-go-warehouses" -const Version = "0.10.0" +const Version = "0.11.0" diff --git a/warehouses/v1/model.go b/warehouses/v1/model.go index cdb78e9d..998125ab 100644 --- a/warehouses/v1/model.go +++ b/warehouses/v1/model.go @@ -683,7 +683,9 @@ type CreateWarehouseRequest struct { // The amount of time in minutes that a SQL warehouse must be idle (i.e., no // RUNNING queries) before it is automatically stopped. // - // Supported values: - Must be == 0 or >= 10 mins - 0 indicates no autostop. + // Supported values: - Must be == 0 or >= 1 min for serverless warehouses. - + // Must be == 0 or >= 10 mins for classic and pro warehouses. - 0 indicates no + // autostop. // // Defaults to 120 mins AutoStopMins *int @@ -801,7 +803,9 @@ type EditWarehouseRequest struct { // The amount of time in minutes that a SQL warehouse must be idle (i.e., no // RUNNING queries) before it is automatically stopped. // - // Supported values: - Must be == 0 or >= 10 mins - 0 indicates no autostop. + // Supported values: - Must be == 0 or >= 1 min for serverless warehouses. - + // Must be == 0 or >= 10 mins for classic and pro warehouses. - 0 indicates no + // autostop. // // Defaults to 120 mins AutoStopMins *int @@ -903,7 +907,9 @@ type EndpointInfo struct { // The amount of time in minutes that a SQL warehouse must be idle (i.e., no // RUNNING queries) before it is automatically stopped. // - // Supported values: - Must be == 0 or >= 10 mins - 0 indicates no autostop. + // Supported values: - Must be == 0 or >= 1 min for serverless warehouses. - + // Must be == 0 or >= 10 mins for classic and pro warehouses. - 0 indicates no + // autostop. // // Defaults to 120 mins AutoStopMins *int @@ -1020,7 +1026,9 @@ type GetWarehouseResponse struct { // The amount of time in minutes that a SQL warehouse must be idle (i.e., no // RUNNING queries) before it is automatically stopped. // - // Supported values: - Must be == 0 or >= 10 mins - 0 indicates no autostop. + // Supported values: - Must be == 0 or >= 1 min for serverless warehouses. - + // Must be == 0 or >= 10 mins for classic and pro warehouses. - 0 indicates no + // autostop. // // Defaults to 120 mins AutoStopMins *int