CXP-897 Incremental sync support

README.md

@@ -115,6 +115,36 @@ To instead exclude specific workspaces from the sync, pass them to the
 list. Each entry can be a workspace name, deployment name, or numeric workspace
 ID. Excluded workspaces and their roles are skipped entirely.
 
+## Incremental sync
+
+By default, `baton-databricks` does a full resync of every resource on every run.
+You can opt into an additional, cheap pathway that polls a Databricks audit log
+between full syncs to pick up access changes early, by setting
+`--enable-incremental-sync` (or `BATON_ENABLE_INCREMENTAL_SYNC`). Full syncs
+still run as the correctness backstop; incremental sync does not detect
+deletions, which are only caught by the next full sync.
+
+Incremental sync requires OAuth2 (service principal) authentication — it's not
+available with workspace tokens, since the Account API needed to resolve audit
+events back to synced resources is unreachable that way.
+
+Incremental sync also requires:
+
+- `--sql-warehouse-id` (or `BATON_SQL_WAREHOUSE_ID`), the ID of a Databricks SQL
+  warehouse the connector can use to query the `system.access.audit` table. A
+  small serverless warehouse is recommended to minimize cold-start latency.
+  The connector automatically discovers which workspace hosts it.
+- A one-time setup performed by a Databricks admin, which the connector cannot
+  do on its own:
+  - An account admin must [enable the `access` system
+    schema](https://docs.databricks.com/en/admin/system-tables/index.html) for
+    the account's Unity Catalog metastore.
+  - A metastore admin must grant `SELECT` on `system.access` to the service
+    principal or user the connector authenticates as.
+
+Once enabled, ongoing polling only needs that `SELECT` grant plus warehouse
+access; no further elevated privilege is required.
+
 ## Group provisioning limitations
 provisioning of account groups from a workspace token is not supported, if you need to provision groups you can only do it using the client-id and client-secret flow,
 this is due to the fact that the Databricks API does not allow provisioning of groups from a workspace token.
@@ -153,7 +183,8 @@ Flags:
       --client-secret string                             The client secret used to authenticate with ConductorOne ($BATON_CLIENT_SECRET)
       --databricks-client-id string                      required: The Databricks service principal's client ID used to connect to the Databricks Account and Workspace API ($BATON_DATABRICKS_CLIENT_ID)
       --databricks-client-secret string                  required: The Databricks service principal's client secret used to connect to the Databricks Account and Workspace API ($BATON_DATABRICKS_CLIENT_SECRET)
-      --databricks-exclude-workspaces strings            Workspaces to exclude from sync, identified by workspace name, deployment name, or numeric workspace ID ($BATON_DATABRICKS_EXCLUDE_WORKSPACES)
+      --databricks-exclude-workspaces strings            Workspaces to exclude from sync, identified by workspace name, deployment name, or numeric workspace ID. Mutually exclusive with workspaces. ($BATON_DATABRICKS_EXCLUDE_WORKSPACES)
+      --enable-incremental-sync                          Poll a Databricks audit-log event feed between full syncs to pick up access changes early. Deletions are still only caught by the next full sync. ($BATON_ENABLE_INCREMENTAL_SYNC)
       --external-resource-c1z string                     The path to the c1z file to sync external baton resources with ($BATON_EXTERNAL_RESOURCE_C1Z)
       --external-resource-entitlement-id-filter string   The entitlement that external users, groups must have access to sync external baton resources ($BATON_EXTERNAL_RESOURCE_ENTITLEMENT_ID_FILTER)
       --external-resource-traits strings                 Resource type traits (e.g. "user", "group", "app") to sync and match from the external resource c1z. When unset the matcher falls back to user and group; passing this flag replaces the full set rather than adding to it. ($BATON_EXTERNAL_RESOURCE_TRAITS)
@@ -173,6 +204,7 @@ Flags:
   -p, --provisioning                                     This must be set in order for provisioning actions to be enabled ($BATON_PROVISIONING)
       --skip-entitlements-and-grants                     This must be set to skip syncing of entitlements and grants ($BATON_SKIP_ENTITLEMENTS_AND_GRANTS)
       --skip-full-sync                                   This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC)
+      --sql-warehouse-id string                          ID of the Databricks SQL warehouse used to query system.access.audit. Required when incremental sync is enabled; the workspace hosting it is discovered automatically. ($BATON_SQL_WAREHOUSE_ID)
       --storage-engine string                            The storage engine to use when opening the sync c1z file: sqlite or pebble. Leave unset to use the baton-sdk default. ($BATON_STORAGE_ENGINE)
       --sync-resource-types strings                      The resource type IDs to sync ($BATON_SYNC_RESOURCE_TYPES)
       --sync-resources strings                           The resource IDs to sync ($BATON_SYNC_RESOURCES)
@@ -181,7 +213,7 @@ Flags:
   -v, --version                                          version for baton-databricks
       --workers int                                      The number of sync workers to use. -1 for auto-detect, 0 for sequential, >0 for parallel ($BATON_WORKERS)
       --workspace-tokens strings                         required: The Databricks personal access tokens scoped to specific workspaces used to connect to the Databricks Workspace API ($BATON_WORKSPACE_TOKENS)
-      --workspaces strings                               Limit syncing to the specified workspaces, by deployment name, not workspace ID. Required when using workspace tokens, in the same order as workspace-tokens. ($BATON_WORKSPACES)
+      --workspaces strings                               Limit syncing to the specified workspaces, by deployment name, not workspace ID. Required when using workspace tokens, in the same order as workspace-tokens. Mutually exclusive with databricks-exclude-workspaces. ($BATON_WORKSPACES)
 
 Use "baton-databricks [command] --help" for more information about a command.
 ```

baton_capabilities.json

@@ -8,6 +8,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_PROVISION"
       ],
       "permissions": {}
@@ -22,6 +23,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_PROVISION"
       ],
       "permissions": {}
@@ -36,6 +38,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_PROVISION"
       ],
       "permissions": {}
@@ -50,6 +53,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_PROVISION"
       ],
       "permissions": {}
@@ -69,6 +73,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_ACCOUNT_PROVISIONING",
         "CAPABILITY_RESOURCE_DELETE"
       ],
@@ -84,6 +89,7 @@
       },
       "capabilities": [
         "CAPABILITY_SYNC",
+        "CAPABILITY_TARGETED_SYNC",
         "CAPABILITY_PROVISION"
       ],
       "permissions": {}
@@ -93,7 +99,10 @@
     "CAPABILITY_PROVISION",
     "CAPABILITY_SYNC",
     "CAPABILITY_ACCOUNT_PROVISIONING",
-    "CAPABILITY_RESOURCE_DELETE"
+    "CAPABILITY_RESOURCE_DELETE",
+    "CAPABILITY_TARGETED_SYNC",
+    "CAPABILITY_EVENT_FEED_V2",
+    "CAPABILITY_SERVICE_MODE_TARGETED_SYNC"
   ],
   "credentialDetails": {
     "capabilityAccountProvisioning": {

config_schema.json

@@ -163,6 +163,18 @@
       "displayName": "Exclude Workspaces",
       "description": "Workspaces to exclude from sync, identified by workspace name, deployment name, or numeric workspace ID. Mutually exclusive with workspaces.",
       "stringSliceField": {}
+    },
+    {
+      "name": "enable-incremental-sync",
+      "displayName": "Enable Incremental Sync",
+      "description": "Poll a Databricks audit-log event feed between full syncs to pick up access changes early. Deletions are still only caught by the next full sync.",
+      "boolField": {}
+    },
+    {
+      "name": "sql-warehouse-id",
+      "displayName": "SQL Warehouse ID",
+      "description": "ID of the Databricks SQL warehouse used to query system.access.audit. Required when incremental sync is enabled; the workspace hosting it is discovered automatically.",
+      "stringField": {}
     }
   ],
   "constraints": [
@@ -198,7 +210,9 @@
         "hostname",
         "account-hostname",
         "workspaces",
-        "databricks-exclude-workspaces"
+        "databricks-exclude-workspaces",
+        "enable-incremental-sync",
+        "sql-warehouse-id"
       ],
       "default": true
     },

docs/connector.mdx

@@ -23,6 +23,14 @@ The Databricks connector supports [automatic account provisioning and deprovisio
 Provisioning **account groups** requires OAuth authentication. It is not available when authenticating with a workspace token, because the Databricks API does not allow provisioning account groups from a workspace token. Workspace-scoped groups can still be provisioned with a workspace token.
 </Note>
 
+### Optional: faster updates between syncs
+
+By default, the Databricks connector picks up access changes on its regular sync schedule. You can optionally turn on incremental sync, which checks Databricks' activity log between full syncs so that changes like new group members show up in C1 sooner. Full syncs still run as usual and remain the source of truth; removed access is only reflected after the next full sync.
+
+Turning this on requires a small Databricks SQL warehouse and a one-time setup step performed by a Databricks admin (granting the connector read access to Databricks' `system.access` activity log). Ask your connector operator or C1 support contact to enable it for you.
+
+Incremental sync requires OAuth2 (service principal) authentication — it's not available with workspace tokens, since the Account API needed to resolve audit events back to synced resources is unreachable that way.
+
 ## Gather Databricks credentials
 
 Configuring the connector requires you to pass in credentials generated in Databricks. Gather these credentials before you move on.

pkg/config/config.go

@@ -70,6 +70,20 @@ var (
 		field.WithDescription("Workspaces to exclude from sync, identified by workspace name, deployment name, or numeric workspace ID. Mutually exclusive with workspaces."),
 		field.WithDisplayName("Exclude Workspaces"),
 	)
+	EnableIncrementalSyncField = field.BoolField(
+		"enable-incremental-sync",
+		field.WithDescription("Poll a Databricks audit-log event feed between full syncs to pick up access changes early. Deletions are still only caught by the next full sync."),
+		field.WithDisplayName("Enable Incremental Sync"),
+		field.WithDefaultValue(false),
+	)
+	SQLWarehouseIDField = field.StringField(
+		"sql-warehouse-id",
+		field.WithDescription(
+			"ID of the Databricks SQL warehouse used to query system.access.audit. Required when incremental "+
+				"sync is enabled; the workspace hosting it is discovered automatically.",
+		),
+		field.WithDisplayName("SQL Warehouse ID"),
+	)
 	configFields = []field.SchemaField{
 		AccountHostnameField,
 		AccountIdField,
@@ -80,6 +94,8 @@ var (
 		WorkspaceTokensField,
 		BaseURLField,
 		ExcludeWorkspacesField,
+		EnableIncrementalSyncField,
+		SQLWarehouseIDField,
 	}
 )
 
@@ -101,6 +117,7 @@ var Config = field.NewConfiguration(
 			Fields: []field.SchemaField{
 				AccountIdField, DatabricksClientIdField, DatabricksClientSecretField,
 				HostnameField, AccountHostnameField, WorkspacesField, ExcludeWorkspacesField,
+				EnableIncrementalSyncField, SQLWarehouseIDField,
 			},
 			Default: true,
 		},
@@ -110,6 +127,8 @@ var Config = field.NewConfiguration(
 			HelpText: "Authenticate with a personal access token scoped to each workspace. " +
 				"Does not sync account-level data (account entitlements and grants, and " +
 				"workspace-membership entitlements); use OAuth for full account coverage.",
+			// Incremental sync requires the Account API, which workspace tokens can't reach
+			// (see Validate) — omitted here so the UI doesn't offer an option that can never work.
 			Fields:  []field.SchemaField{AccountIdField, WorkspacesField, WorkspaceTokensField, HostnameField, AccountHostnameField},
 			Default: false,
 		},

pkg/connector/account.go

@@ -216,6 +216,16 @@ func (a *accountBuilder) Grant(ctx context.Context, principal *v2.Resource, enti
 	return nil, nil
 }
 
+// Get returns the singleton account resource, used to re-sync it after a RESOURCE_CHANGE event.
+func (a *accountBuilder) Get(ctx context.Context, resourceId *v2.ResourceId, parentResourceId *v2.ResourceId) (*v2.Resource, annotations.Annotations, error) {
+	resource, err := a.accountResource(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return resource, nil, nil
+}
+
 func (a *accountBuilder) Revoke(ctx context.Context, grant *v2.Grant) (annotations.Annotations, error) {
 	l := ctxzap.Extract(ctx)