From 7b8fd216f7a23a1753443be7967d0c6beb35b491 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:38:19 +0200 Subject: [PATCH 1/8] CFTL-589 Add GitHub OAuth as a third git authentication method Cloning a private repository so far required the customer to create and rotate a credential by hand, and a classic PAT grants read and write across every private repository the user can reach. A GitHub App user access token is limited to Contents: Read-only on the repositories selected when the app is installed, so nothing has to be created or rotated manually. The token arrives outside "parameters", in the authorization section of the configuration, and is handed to git through a GIT_ASKPASS helper. That keeps it out of the command line, out of the cloned repository's .git/config and out of the environment of the executed user script. The authorization section is also stripped from the config.json written for that script. Besides the user's own token it carries the shared application secret of the Keboola GitHub App, which no user code may be able to read. The existing none/pat/ssh branches are untouched. Co-Authored-By: Claude Opus 5 --- component_config/configSchema.json | 7 +- src/component.py | 26 +++++- src/configuration.py | 1 + src/source_git.py | 57 +++++++++++-- tests/test_component.py | 132 ++++++++++++++++++++++++++++- 5 files changed, 212 insertions(+), 11 deletions(-) diff --git a/component_config/configSchema.json b/component_config/configSchema.json index 75639a5..9beb2d8 100644 --- a/component_config/configSchema.json +++ b/component_config/configSchema.json @@ -126,13 +126,16 @@ "enum": [ "none", "pat", - "ssh" + "ssh", + "oauth" ], "options": { + "tooltip": "The **GitHub (OAuth)** option is authorized in the **Authorization** section of this configuration. The repositories the component may read are selected on GitHub when the Keboola GitHub App is installed.", "enum_titles": [ "Public – None", "Private – Personal Access Token", - "Private – SSH Key" + "Private – SSH Key", + "Private – GitHub (OAuth)" ] }, "default": "none" diff --git a/src/component.py b/src/component.py index 9daff32..b5268c0 100644 --- a/src/component.py +++ b/src/component.py @@ -64,13 +64,29 @@ def __init__(self): f"in the configuration. Detail: {err}" ) from err + self.oauth_token = self._get_oauth_token() + + def _get_oauth_token(self) -> str | None: + """Access token issued by the OAuth broker, delivered outside "parameters" in the authorization + section. Returns None for configurations that do not use OAuth.""" + try: + credentials = self.configuration.oauth_credentials + except json.JSONDecodeError as err: + # unreadable broker credentials are a configuration problem, not an internal one + raise UserException( + "The stored GitHub authorization could not be read. Please authorize the component again " + "in the Authorization section of the configuration." + ) from err + + return credentials.data.get("access_token") if credentials else None + def run(self): if self.parameters.source == SourceEnum.CODE: base_path = Path(self.data_folder_path) script_filename = FileHandler.prepare_script_file(self.data_folder_path, self.parameters.code) else: base_path = Path(GitHandler.REPO_PATH).absolute() - git_handler = GitHandler(self.parameters.git) + git_handler = GitHandler(self.parameters.git, self.oauth_token) script_filename = git_handler.clone_repository() if self.parameters.venv == VenvEnum.BASE: @@ -147,6 +163,10 @@ def _merge_user_parameters(self): # remove code config_data = self.configuration.config_data.copy() + # the authorization section carries the decrypted OAuth access token and the shared application + # secret, neither of which may reach the executed user script + config_data.pop("authorization", None) + # build config data and overwrite for the user script config_data["parameters"] = self.parameters.user_properties with open(Path(self.data_folder_path) / "config.json", "w+") as inp: @@ -158,7 +178,7 @@ def get_repository_branches(self): Returns a list of branches in the git repository. This method is used to populate the branches dropdown in the UI. """ - git_handler = GitHandler(self.parameters.git) + git_handler = GitHandler(self.parameters.git, self.oauth_token) return git_handler.get_repository_branches() @sync_action("listFiles") @@ -167,7 +187,7 @@ def get_repository_files(self): Returns a list of branches in the git repository. This method is used to populate the branches dropdown in the UI. """ - git_handler = GitHandler(self.parameters.git) + git_handler = GitHandler(self.parameters.git, self.oauth_token) return git_handler.get_repository_files() diff --git a/src/configuration.py b/src/configuration.py index b3f9e3c..f5602c4 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -25,6 +25,7 @@ class AuthEnum(Enum): NONE = "none" PAT = "pat" SSH = "ssh" + OAUTH = "oauth" # the ssh_keys.keys.[#private,public] structure is based on Keboola's standard SSH keys UI element output structure diff --git a/src/source_git.py b/src/source_git.py index 5fd5fba..55b38c9 100644 --- a/src/source_git.py +++ b/src/source_git.py @@ -9,11 +9,18 @@ from configuration import AuthEnum, GitConfiguration +GITHUB_HOSTS = ("github.com", "www.github.com") +OAUTH_GIT_USERNAME = "x-access-token" +OAUTH_TOKEN_ENV = "GIT_OAUTH_TOKEN" +# git runs this helper whenever it needs a password. Reading the token from the environment keeps it out +# of the command line, out of the cloned repository's .git/config and out of the user script's environment. +ASKPASS_SCRIPT = f'#!/bin/sh\nprintf "%s" "${OAUTH_TOKEN_ENV}"\n' + class GitHandler: REPO_PATH = "repo_clone" - def __init__(self, git_cfg: GitConfiguration): + def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): # add path for absolute imports to start at the cloned repository root level sys.path.append(str(Path(__file__).parent.parent / GitHandler.REPO_PATH)) @@ -26,6 +33,8 @@ def __init__(self, git_cfg: GitConfiguration): if self.git_cfg.auth == AuthEnum.PAT: self._set_up_token_auth() + elif self.git_cfg.auth == AuthEnum.OAUTH: + self._set_up_oauth_auth(oauth_token) repo_url = self.git_cfg.url if repo_url.startswith("git@") or repo_url.startswith("ssh://"): @@ -58,6 +67,30 @@ def _set_up_netrc(repo_url: str, token: str) -> None: f.write(entry) os.chmod(netrc_path, 0o600) + def _set_up_oauth_auth(self, oauth_token: str | None) -> None: + if not oauth_token: + raise UserException( + "GitHub authorization is missing. Please authorize the component in the Authorization " + "section of the configuration." + ) + + parsed = urlparse(self.git_cfg.url) + if parsed.scheme != "https" or parsed.hostname not in GITHUB_HOSTS: + raise UserException("GitHub authorization is only supported for https://github.com repository URLs") + + # only the username goes into the URL, the token itself is supplied by the askpass helper + self.repo_auth_url = self.git_cfg.url.replace("https://", f"https://{OAUTH_GIT_USERNAME}@") + self.env[OAUTH_TOKEN_ENV] = oauth_token + self.env["GIT_ASKPASS"] = str(self._write_askpass_helper()) + logging.info("Git OAuth authentication set up for GitHub URL.") + + @staticmethod + def _write_askpass_helper() -> Path: + askpass_path = Path("~/.git_askpass.sh").expanduser() + askpass_path.write_text(ASKPASS_SCRIPT) + os.chmod(askpass_path, 0o700) + return askpass_path + def _set_up_ssh_command(self) -> None: if not self.git_cfg.ssh_keys.keys.encrypted_private: if self.git_cfg.auth == AuthEnum.SSH: @@ -90,6 +123,22 @@ def _set_up_ssh_command(self) -> None: self.env["GIT_SSH_COMMAND"] = " ".join(ssh_command) + def _explain_error(self, error_msg: str) -> str: + """Append an actionable hint to git errors whose raw wording does not point at the actual cause.""" + if "Permission denied" in error_msg or "publickey" in error_msg: + return f"{error_msg}. Please check SSH key configuration or use HTTPS URL." + + # GitHub answers with "not found" for repositories the app cannot see, so that it does not + # disclose their existence. The usual cause is a missing or incomplete app installation. + if self.git_cfg.auth == AuthEnum.OAUTH and "not found" in error_msg.lower(): + return ( + f"{error_msg}. The repository is not available to the Keboola GitHub App. Make sure the app " + "is installed on the account owning the repository and that this repository is included in " + "the app's repository selection." + ) + + return error_msg + def clone_repository(self, sync_action=False) -> Path: """ Clone a git repository and return the path to the cloned code. @@ -119,9 +168,7 @@ def clone_repository(self, sync_action=False) -> Path: if process.returncode != 0: error_msg = stderr.decode() if stderr else "Unknown git clone error" - if "Permission denied" in error_msg or "publickey" in error_msg: - error_msg += ". Please check SSH key configuration or use HTTPS URL." - raise UserException(f"Failed to clone git repository: {error_msg}") + raise UserException(f"Failed to clone git repository: {self._explain_error(error_msg)}") logging.info("Successfully cloned repository") @@ -161,7 +208,7 @@ def get_repository_branches(self): stdout, stderr = process.communicate() if process.returncode != 0: - raise UserException(f"Failed to get branches: {stderr.decode()}") + raise UserException(f"Failed to get branches: {self._explain_error(stderr.decode())}") branches = [line.strip().split("refs/heads/")[-1] for line in stdout.decode().splitlines() if line.strip()] return [{"value": b, "label": b} for b in branches] diff --git a/tests/test_component.py b/tests/test_component.py index cbe119e..5fe3094 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -2,13 +2,15 @@ import os import tempfile import unittest +from pathlib import Path import mock from freezegun import freeze_time from keboola.component.exceptions import UserException from component import Component -from configuration import Configuration, SourceEnum, VenvEnum +from configuration import AuthEnum, Configuration, GitConfiguration, SourceEnum, VenvEnum +from source_git import GitHandler class TestComponent(unittest.TestCase): @@ -128,6 +130,134 @@ def test_valid_configuration_is_parsed_unchanged(self): self.assertEqual(component.parameters.code, "print('hello')") +class TestOAuthAuthentication(unittest.TestCase): + """The OAuth access token must reach git without ever appearing in the command line. + + The component executes arbitrary user code, so the token is passed through an askpass helper that + reads it from the environment of the git subprocess only. + """ + + def setUp(self): + home = tempfile.TemporaryDirectory() + self.addCleanup(home.cleanup) + home_patch = mock.patch.dict(os.environ, {"HOME": home.name}) + home_patch.start() + self.addCleanup(home_patch.stop) + + @staticmethod + def _git_cfg(url: str = "https://github.com/keboola/example.git") -> GitConfiguration: + return GitConfiguration(url=url, auth=AuthEnum.OAUTH) + + def test_missing_token_raises_user_exception(self): + """An unauthorized configuration must fail with an actionable message, not with a git error.""" + with self.assertRaises(UserException) as context: + GitHandler(self._git_cfg(), None) + self.assertIn("GitHub authorization is missing", str(context.exception)) + + def test_non_github_url_raises_user_exception(self): + """The token is only valid for github.com, other hosts must be rejected up front.""" + with self.assertRaises(UserException) as context: + GitHandler(self._git_cfg("https://gitlab.com/keboola/example.git"), "secret-token") + self.assertIn("github.com", str(context.exception)) + + def test_token_is_not_part_of_the_clone_url(self): + """The clone URL carries the username only, so the token cannot leak via argv or .git/config.""" + handler = GitHandler(self._git_cfg(), "secret-token") + self.assertEqual(handler.repo_auth_url, "https://x-access-token@github.com/keboola/example.git") + + def test_token_is_passed_through_the_askpass_helper(self): + """The helper is executable and reads the token from the environment instead of embedding it.""" + handler = GitHandler(self._git_cfg(), "secret-token") + self.assertEqual(handler.env["GIT_OAUTH_TOKEN"], "secret-token") + + askpass_path = Path(handler.env["GIT_ASKPASS"]) + self.assertTrue(os.access(askpass_path, os.X_OK)) + self.assertNotIn("secret-token", askpass_path.read_text()) + + def test_missing_installation_is_explained(self): + """GitHub reports an unreachable repository as "not found", which hides the real cause.""" + handler = GitHandler(self._git_cfg(), "secret-token") + explained = handler._explain_error("remote: Repository not found.") + self.assertIn("Keboola GitHub App", explained) + self.assertIn("repository selection", explained) + + def test_unrelated_errors_are_not_annotated(self): + handler = GitHandler(self._git_cfg(), "secret-token") + self.assertEqual(handler._explain_error("fatal: could not read from remote"), "fatal: could not read from remote") + + def test_other_auth_methods_are_untouched(self): + """A configuration that does not use OAuth must not gain any OAuth environment.""" + handler = GitHandler(GitConfiguration(url="https://github.com/keboola/example.git")) + self.assertIsNone(handler.repo_auth_url) + self.assertNotIn("GIT_ASKPASS", handler.env) + self.assertNotIn("GIT_OAUTH_TOKEN", handler.env) + + def test_ssh_hint_is_preserved(self): + """The pre-existing hint for SSH failures must keep working for non-OAuth configurations.""" + handler = GitHandler(GitConfiguration(url="git@github.com:keboola/example.git", auth=AuthEnum.NONE)) + self.assertIn("SSH key configuration", handler._explain_error("Permission denied (publickey).")) + + +class TestAuthorizationSectionIsNotExposed(unittest.TestCase): + """The config.json handed to the user script must not contain the decrypted OAuth credentials. + + Besides the user's own access token, the authorization section also carries the shared application + secret of the Keboola GitHub App, which must never be readable by the executed script. + """ + + CONFIG_DATA = { + "parameters": {"source": "code", "venv": "base", "user_properties": {"debug": True}}, + "authorization": { + "oauth_api": { + "credentials": { + "id": "main", + "#data": '{"access_token": "secret-token"}', + "appKey": "client-id", + "#appSecret": "app-secret", + } + } + }, + } + + def setUp(self): + datadir = tempfile.TemporaryDirectory() + self.addCleanup(datadir.cleanup) + self.config_path = Path(datadir.name) / "config.json" + self.config_path.write_text(json.dumps(self.CONFIG_DATA)) + with mock.patch.dict(os.environ, {"KBC_DATADIR": datadir.name}): + self.component = Component() + + def test_access_token_is_read_from_the_authorization_section(self): + self.assertEqual(self.component.oauth_token, "secret-token") + + def test_unreadable_credentials_raise_user_exception(self): + """Broker credentials that are not valid JSON must not surface as an internal error.""" + datadir = tempfile.TemporaryDirectory() + self.addCleanup(datadir.cleanup) + config_data = dict(self.CONFIG_DATA) + config_data["authorization"] = {"oauth_api": {"credentials": {"id": "main", "#data": "access_token=abc"}}} + (Path(datadir.name) / "config.json").write_text(json.dumps(config_data)) + + with mock.patch.dict(os.environ, {"KBC_DATADIR": datadir.name}): + with self.assertRaises(UserException) as context: + Component() + self.assertIn("could not be read", str(context.exception)) + + def test_authorization_is_stripped_from_the_script_config(self): + self.component._merge_user_parameters() + + written = self.config_path.read_text() + self.assertNotIn("authorization", json.loads(written)) + self.assertNotIn("secret-token", written) + self.assertNotIn("app-secret", written) + + def test_user_properties_are_still_written(self): + """Stripping the credentials must not disturb what the script actually needs.""" + self.component._merge_user_parameters() + + self.assertEqual(json.loads(self.config_path.read_text())["parameters"], {"debug": True}) + + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] unittest.main() From 5e0df2ff0fdb9309f096d961db80f237752b2b17 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:38:19 +0200 Subject: [PATCH 2/8] CFTL-589 Document the two-step GitHub OAuth setup Repository selection happens when the GitHub App is installed, not when the component is authorized, and the two are independent flows on GitHub's side. Authorizing without installing first yields a valid token that can see no repositories, and the job then fails with a bare "repository not found". Spelling the order out makes that failure avoidable rather than merely explainable after the fact. Co-Authored-By: Claude Opus 5 --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index bad495e..1761526 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ - [Configuration](#configuration) - [Git configuration](#git-configuration) - [SSH configuration](#ssh-configuration) + - [GitHub OAuth configuration](#github-oauth-configuration) - [Example: Running code saved in custom repository + template 🧩](#example-running-code-saved-in-custom-repository--template-) - [Example: Listing preinstalled packages](#example-listing-preinstalled-packages) - [Example: Accessing custom configuration parameters](#example-accessing-custom-configuration-parameters) @@ -86,6 +87,8 @@ The git configuration object supports the following parameters: - `none`: Public repository, no authentication (default). - `pat`: Private repository, Personal Access Token. - `ssh`: Private repository, SSH key. + - `oauth`: Private GitHub repository authorized via the **Authorization** section of the configuration. + Requires the Keboola GitHub App to be installed first – see [GitHub OAuth configuration](#github-oauth-configuration). - `#token`: Personal Access Token (`"auth": "pat"` only). This value will be encrypted in Keboola Storage. The same token also authenticates private git dependencies declared in `[tool.uv.sources]` in your `pyproject.toml`, so there is no need to embed tokens directly in the source file. @@ -99,6 +102,30 @@ The git configuration object supports the following parameters: - `#private`: Private key used for authentication. This value will be encrypted in Keboola Storage. +### GitHub OAuth configuration + +With `"auth": "oauth"` no credential is entered into the configuration at all – the access token is issued by +the Keboola OAuth broker. Only `https://github.com` URLs are supported. + +Setting this up takes two separate steps on GitHub, **in this order**: + +1. **Install** the Keboola GitHub App on your account or organisation + (`https://github.com/apps//installations/new`) and pick the repositories it may read. + Repository selection happens here and nowhere else. +2. **Authorize** the component in the **Authorization** section of the configuration in Keboola. + +The two steps are independent. Authorizing does not install the app, and the authorization screen offers no +repository selection at all – so if you authorize without installing first, you receive a valid token that can +see no repositories and the job fails with `repository not found`. To change which repositories are available +later, reconfigure the installation on GitHub; re-authorizing in Keboola will not change it. + +Some organisations require an owner to approve the installation before it takes effect. + +The app requests `Contents: Read-only` and `Metadata: Read-only`, and the token is limited to the intersection +of those permissions and your own access. It does not authenticate private git dependencies declared in +`[tool.uv.sources]` – keep using `pat` if you rely on those. + + ### Example: Running code saved in custom repository + template 🧩 As this might become a preferred way of running custom Python code in Keboola for many, we prepared a [simple example project](https://github.com/keboola/component-custom-python-example-repo-1), which help you with your first steps (and can also server you as a template for any of your future projects). From 7cbb71d4b40febcc2f74ec8d8dd2216f6a24915a Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:23:53 +0200 Subject: [PATCH 3/8] CFTL-589 Add a repository picker for OAuth configurations Typing the repository URL by hand works, but it defers every mistake to the first job run: a repository the app cannot see fails as "not found" only once the clone is attempted. Listing what the installation actually exposes moves that feedback into the configuration form, and an empty list is precisely the signal that the app was authorized but never installed. The list comes from /user/installations followed by /user/installations/{id}/repositories, because a user access token carries no installation id of its own. Both endpoints are paginated and one user may see several installations, so the results are aggregated. The picker writes a clone URL into git.repository. The free-text git.url stays in place for the other authentication methods, because a schema field cannot be a dropdown and a text input at the same time; GitConfiguration.repository_url resolves which of the two applies. Calls go through urllib to avoid adding an HTTP dependency for two endpoints. Co-Authored-By: Claude Opus 5 --- README.md | 14 ++-- component_config/configSchema.json | 32 +++++++- src/component.py | 23 ++++++ src/configuration.py | 6 ++ src/github_api.py | 78 ++++++++++++++++++++ src/source_git.py | 24 +++--- tests/test_component.py | 113 ++++++++++++++++++++++++++++- 7 files changed, 269 insertions(+), 21 deletions(-) create mode 100644 src/github_api.py diff --git a/README.md b/README.md index 1761526..f0cfffd 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,9 @@ configured and executed directly in Keboola. This eliminates the need to build a The git configuration object supports the following parameters: -- `url`: Repository URL – supports both HTTPS and SSH formats. +- `url`: Repository URL – supports both HTTPS and SSH formats (all `auth` methods except `oauth`). +- `repository`: Clone URL of the repository, picked from a list of what the GitHub App may read + (`"auth": "oauth"` only). Replaces `url` for OAuth configurations. - `branch`: Branch name to checkout – UI provides branch selection. - `filename`: Python script filename to execute – UI lists available files. - `auth`: Repository visibility & authentication method. @@ -113,11 +115,13 @@ Setting this up takes two separate steps on GitHub, **in this order**: (`https://github.com/apps//installations/new`) and pick the repositories it may read. Repository selection happens here and nowhere else. 2. **Authorize** the component in the **Authorization** section of the configuration in Keboola. +3. Pick the repository from the **Repository** dropdown, which lists what the installation makes available. -The two steps are independent. Authorizing does not install the app, and the authorization screen offers no -repository selection at all – so if you authorize without installing first, you receive a valid token that can -see no repositories and the job fails with `repository not found`. To change which repositories are available -later, reconfigure the installation on GitHub; re-authorizing in Keboola will not change it. +Installing and authorizing are independent. Authorizing does not install the app, and the authorization +screen offers no repository selection at all – so if you authorize without installing first, you receive a +valid token that can see no repositories and the **Repository** dropdown reports that none are available. To +change which repositories are available later, reconfigure the installation on GitHub; re-authorizing in +Keboola will not change it. Some organisations require an owner to approve the installation before it takes effect. diff --git a/component_config/configSchema.json b/component_config/configSchema.json index 9beb2d8..06574f5 100644 --- a/component_config/configSchema.json +++ b/component_config/configSchema.json @@ -116,7 +116,16 @@ "url": { "type": "string", "title": "Repository URL", - "propertyOrder": 70 + "propertyOrder": 70, + "options": { + "dependencies": { + "auth": [ + "none", + "pat", + "ssh" + ] + } + } }, "auth": { "type": "string", @@ -161,6 +170,24 @@ } } }, + "repository": { + "type": "string", + "enum": [], + "format": "select", + "title": "Repository", + "propertyOrder": 105, + "options": { + "dependencies": { + "auth": "oauth" + }, + "tooltip": "Only repositories included in the Keboola GitHub App installation are listed. To make more of them available, change the repository selection of the installation on GitHub.", + "async": { + "label": "List Repositories", + "action": "listRepositories", + "cache": false + } + } + }, "branch": { "type": "string", "enum": [], @@ -172,7 +199,8 @@ "label": "List Branches", "action": "listBranches", "autoload": [ - "git.url" + "git.url", + "git.repository" ], "cache": false } diff --git a/src/component.py b/src/component.py index b5268c0..029ca08 100644 --- a/src/component.py +++ b/src/component.py @@ -15,6 +15,7 @@ from keboola.component.exceptions import UserException from configuration import AuthEnum, Configuration, SourceEnum, VenvEnum, encrypted_keys +from github_api import GitHubApi from package_installer import PackageInstaller from source_file import FileHandler from source_git import GitHandler @@ -172,6 +173,28 @@ def _merge_user_parameters(self): with open(Path(self.data_folder_path) / "config.json", "w+") as inp: json.dump(config_data, inp) + @sync_action("listRepositories") + def get_oauth_repositories(self): + """ + Returns the repositories the Keboola GitHub App is allowed to read. + This method is used to populate the repository dropdown in the UI. + """ + if not self.oauth_token: + raise UserException( + "GitHub authorization is missing. Please authorize the component in the Authorization " + "section of the configuration." + ) + + repositories = GitHubApi(self.oauth_token).list_installation_repositories() + if not repositories: + # authorizing does not install the app, so this is the expected state after authorizing alone + raise UserException( + "No repositories are available to the Keboola GitHub App. Install the app on the account " + "owning the repository and include that repository in the app's repository selection." + ) + + return repositories + @sync_action("listBranches") def get_repository_branches(self): """ diff --git a/src/configuration.py b/src/configuration.py index f5602c4..8ec6831 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -43,12 +43,18 @@ class SSHKeysConfiguration: @dataclass class GitConfiguration: url: str = "" + repository: str = "" branch: str = "main" filename: str = "main.py" auth: AuthEnum = AuthEnum.NONE encrypted_token: str | None = None ssh_keys: SSHKeysConfiguration = field(default_factory=SSHKeysConfiguration) + @property + def repository_url(self) -> str: + """Repository to clone. OAuth configurations select it from a list instead of typing in a URL.""" + return self.repository if self.auth == AuthEnum.OAUTH else self.url + @dataclass class Configuration: diff --git a/src/github_api.py b/src/github_api.py new file mode 100644 index 0000000..3d3fb34 --- /dev/null +++ b/src/github_api.py @@ -0,0 +1,78 @@ +import json +import urllib.error +import urllib.request + +from keboola.component.exceptions import UserException + +API_BASE_URL = "https://api.github.com" +API_VERSION = "2022-11-28" +USER_AGENT = "keboola-custom-python-component" +PAGE_SIZE = 100 +REQUEST_TIMEOUT = 30 +# a user access token never reaches anywhere near this many pages, it is a runaway guard only +MAX_PAGES = 50 + + +class GitHubApi: + """Read-only client for the endpoints needed to list the repositories the app may read.""" + + def __init__(self, token: str, base_url: str = API_BASE_URL): + self.token = token + self.base_url = base_url.rstrip("/") + + def list_installation_repositories(self) -> list[dict]: + """Repositories of every app installation visible to the authorizing user, as dropdown options. + + The value is the clone URL, so that selecting a repository fills in the same thing the other + authentication methods expect to be typed in by hand. + """ + options = [] + for installation in self._get_all("/user/installations", "installations"): + path = f"/user/installations/{installation['id']}/repositories" + for repository in self._get_all(path, "repositories"): + options.append({"value": repository["clone_url"], "label": repository["full_name"]}) + + return options + + def _get_all(self, path: str, items_key: str) -> list[dict]: + items: list[dict] = [] + for page in range(1, MAX_PAGES + 1): + payload = self._get(f"{path}?per_page={PAGE_SIZE}&page={page}") + page_items = payload.get(items_key, []) + items.extend(page_items) + if not page_items or len(items) >= payload.get("total_count", 0): + break + + return items + + def _get(self, path: str) -> dict: + request = urllib.request.Request( + f"{self.base_url}{path}", + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": API_VERSION, + # GitHub rejects requests without a User-Agent + "User-Agent": USER_AGENT, + }, + ) + + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + return json.load(response) + except urllib.error.HTTPError as err: + raise UserException(self._explain_http_error(err.code, err.reason)) from err + except urllib.error.URLError as err: + raise UserException(f"Could not reach the GitHub API: {err.reason}") from err + + @staticmethod + def _explain_http_error(code: int, reason: str) -> str: + if code == 401: + return ( + "The GitHub authorization is no longer valid. Please authorize the component again in the " + "Authorization section of the configuration." + ) + if code == 403: + return "The GitHub API refused the request. The authorization may not have the required permissions." + + return f"The GitHub API returned an unexpected error: HTTP {code} {reason}" diff --git a/src/source_git.py b/src/source_git.py index 55b38c9..6a375b4 100644 --- a/src/source_git.py +++ b/src/source_git.py @@ -26,9 +26,10 @@ def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): self.env = os.environ.copy() self.git_cfg = git_cfg + self.repo_url = git_cfg.repository_url self.repo_auth_url = None # ‼️ NEVER EVER INCLUDE THIS VARIABLE IN LOGGING OUTPUT ‼️ - if not self.git_cfg.url: + if not self.repo_url: raise UserException("Git repository URL is required") if self.git_cfg.auth == AuthEnum.PAT: @@ -36,8 +37,7 @@ def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): elif self.git_cfg.auth == AuthEnum.OAUTH: self._set_up_oauth_auth(oauth_token) - repo_url = self.git_cfg.url - if repo_url.startswith("git@") or repo_url.startswith("ssh://"): + if self.repo_url.startswith("git@") or self.repo_url.startswith("ssh://"): self._set_up_ssh_command() # do not ask for credentials when git authentication fails @@ -47,13 +47,11 @@ def _set_up_token_auth(self) -> None: if not self.git_cfg.encrypted_token: raise UserException("No personal access token provided") - if not self.git_cfg.url.startswith("https://"): + if not self.repo_url.startswith("https://"): raise UserException("PAT authentication is only supported for HTTPS URLs") - self.repo_auth_url = self.git_cfg.url.replace( - "https://", f"https://x-token-auth:{self.git_cfg.encrypted_token}@" - ) - self._set_up_netrc(self.git_cfg.url, self.git_cfg.encrypted_token) + self.repo_auth_url = self.repo_url.replace("https://", f"https://x-token-auth:{self.git_cfg.encrypted_token}@") + self._set_up_netrc(self.repo_url, self.git_cfg.encrypted_token) logging.info("Git token authentication set up for HTTPS URL.") @staticmethod @@ -74,12 +72,12 @@ def _set_up_oauth_auth(self, oauth_token: str | None) -> None: "section of the configuration." ) - parsed = urlparse(self.git_cfg.url) + parsed = urlparse(self.repo_url) if parsed.scheme != "https" or parsed.hostname not in GITHUB_HOSTS: raise UserException("GitHub authorization is only supported for https://github.com repository URLs") # only the username goes into the URL, the token itself is supplied by the askpass helper - self.repo_auth_url = self.git_cfg.url.replace("https://", f"https://{OAUTH_GIT_USERNAME}@") + self.repo_auth_url = self.repo_url.replace("https://", f"https://{OAUTH_GIT_USERNAME}@") self.env[OAUTH_TOKEN_ENV] = oauth_token self.env["GIT_ASKPASS"] = str(self._write_askpass_helper()) logging.info("Git OAuth authentication set up for GitHub URL.") @@ -148,7 +146,7 @@ def clone_repository(self, sync_action=False) -> Path: """ branch = self.git_cfg.branch or "main" - logging.info("Cloning git repository: %s", self.git_cfg.url) + logging.info("Cloning git repository: %s", self.repo_url) try: clone_args = ["git", "clone"] @@ -156,7 +154,7 @@ def clone_repository(self, sync_action=False) -> Path: if branch: clone_args.extend(["--branch", branch]) - clone_args.extend([self.repo_auth_url or self.git_cfg.url, GitHandler.REPO_PATH]) + clone_args.extend([self.repo_auth_url or self.repo_url, GitHandler.REPO_PATH]) process = subprocess.Popen( clone_args, @@ -197,7 +195,7 @@ def get_repository_branches(self): try: branches_args = ["git", "ls-remote", "--heads"] - branches_args.append(self.repo_auth_url or self.git_cfg.url) + branches_args.append(self.repo_auth_url or self.repo_url) process = subprocess.Popen( branches_args, diff --git a/tests/test_component.py b/tests/test_component.py index 5fe3094..391e08b 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -1,7 +1,9 @@ +import io import json import os import tempfile import unittest +import urllib.error from pathlib import Path import mock @@ -10,9 +12,17 @@ from component import Component from configuration import AuthEnum, Configuration, GitConfiguration, SourceEnum, VenvEnum +from github_api import GitHubApi from source_git import GitHandler +def api_response(payload: dict): + """A urlopen context manager yielding the given payload as a JSON body.""" + response = mock.MagicMock() + response.__enter__.return_value = io.BytesIO(json.dumps(payload).encode()) + return response + + class TestComponent(unittest.TestCase): # set global time to 2010-10-10 - affects functions like datetime.now() @@ -146,7 +156,7 @@ def setUp(self): @staticmethod def _git_cfg(url: str = "https://github.com/keboola/example.git") -> GitConfiguration: - return GitConfiguration(url=url, auth=AuthEnum.OAUTH) + return GitConfiguration(repository=url, auth=AuthEnum.OAUTH) def test_missing_token_raises_user_exception(self): """An unauthorized configuration must fail with an actionable message, not with a git error.""" @@ -258,6 +268,107 @@ def test_user_properties_are_still_written(self): self.assertEqual(json.loads(self.config_path.read_text())["parameters"], {"debug": True}) +class TestGitHubApi(unittest.TestCase): + """Listing repositories has to work across several installations and several pages.""" + + def test_repositories_from_all_installations_are_listed(self): + responses = [ + api_response({"total_count": 2, "installations": [{"id": 1}, {"id": 2}]}), + api_response( + {"total_count": 1, "repositories": [{"full_name": "acme/first", "clone_url": "https://gh/first.git"}]} + ), + api_response( + {"total_count": 1, "repositories": [{"full_name": "acme/second", "clone_url": "https://gh/second.git"}]} + ), + ] + with mock.patch("github_api.urllib.request.urlopen", side_effect=responses): + options = GitHubApi("secret-token").list_installation_repositories() + + self.assertEqual( + options, + [ + {"value": "https://gh/first.git", "label": "acme/first"}, + {"value": "https://gh/second.git", "label": "acme/second"}, + ], + ) + + def test_paginated_results_are_collected(self): + first_page = [{"full_name": f"acme/repo-{i}", "clone_url": f"https://gh/repo-{i}.git"} for i in range(100)] + responses = [ + api_response({"total_count": 1, "installations": [{"id": 1}]}), + api_response({"total_count": 101, "repositories": first_page}), + api_response({"total_count": 101, "repositories": [{"full_name": "acme/last", "clone_url": "https://gh/l"}]}), + ] + with mock.patch("github_api.urllib.request.urlopen", side_effect=responses): + options = GitHubApi("secret-token").list_installation_repositories() + + self.assertEqual(len(options), 101) + self.assertEqual(options[-1]["label"], "acme/last") + + def test_request_is_authenticated(self): + response = api_response({"total_count": 0, "installations": []}) + with mock.patch("github_api.urllib.request.urlopen", side_effect=[response]) as urlopen: + GitHubApi("secret-token").list_installation_repositories() + + headers = {key.lower(): value for key, value in urlopen.call_args.args[0].header_items()} + self.assertEqual(headers["authorization"], "Bearer secret-token") + self.assertIn("user-agent", headers) + + def test_revoked_authorization_is_explained(self): + """A revoked authorization must tell the user to re-authorize, not show a bare HTTP 401.""" + error = urllib.error.HTTPError("https://api.github.com/user/installations", 401, "Unauthorized", {}, None) + with mock.patch("github_api.urllib.request.urlopen", side_effect=error): + with self.assertRaises(UserException) as context: + GitHubApi("secret-token").list_installation_repositories() + + self.assertIn("authorize the component again", str(context.exception)) + + +class TestListRepositoriesAction(unittest.TestCase): + """The dropdown is where a missing app installation shows up before a job is ever run.""" + + def _component(self, authorized: bool) -> Component: + datadir = tempfile.TemporaryDirectory() + self.addCleanup(datadir.cleanup) + # "run" keeps the sync_action decorator from swallowing exceptions into exit(1) + config_data = {"action": "run", "parameters": {"source": "git", "venv": "base", "user_properties": {}}} + if authorized: + credentials = {"id": "main", "#data": '{"access_token": "secret-token"}'} + config_data["authorization"] = {"oauth_api": {"credentials": credentials}} + (Path(datadir.name) / "config.json").write_text(json.dumps(config_data)) + + with mock.patch.dict(os.environ, {"KBC_DATADIR": datadir.name}): + return Component() + + def test_unauthorized_configuration_is_reported(self): + with self.assertRaises(UserException) as context: + self._component(authorized=False).get_oauth_repositories() + self.assertIn("GitHub authorization is missing", str(context.exception)) + + def test_missing_installation_is_reported(self): + component = self._component(authorized=True) + response = api_response({"total_count": 0, "installations": []}) + with mock.patch("github_api.urllib.request.urlopen", side_effect=[response]): + with self.assertRaises(UserException) as context: + component.get_oauth_repositories() + + self.assertIn("No repositories are available", str(context.exception)) + + +class TestRepositoryUrlResolution(unittest.TestCase): + """OAuth configurations carry the repository in "repository", the other methods in "url".""" + + def test_oauth_uses_the_selected_repository(self): + cfg = GitConfiguration(url="https://github.com/acme/typed.git", auth=AuthEnum.OAUTH, + repository="https://github.com/acme/picked.git") + self.assertEqual(cfg.repository_url, "https://github.com/acme/picked.git") + + def test_other_methods_use_the_typed_url(self): + cfg = GitConfiguration(url="https://github.com/acme/typed.git", auth=AuthEnum.PAT, + repository="https://github.com/acme/picked.git") + self.assertEqual(cfg.repository_url, "https://github.com/acme/typed.git") + + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] unittest.main() From 45a585c05e63a91428ae6207e8eafa4aa3eb9423 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:45:45 +0200 Subject: [PATCH 4/8] CFTL-589 Stop requiring a repository URL that OAuth configurations cannot fill in The free-text url field is hidden for auth: oauth, where the repository is picked from a dropdown into git.repository instead, but it stayed listed in git.required. That leaves an OAuth configuration demanding a field the form gives no way to fill in. Only auth stays required. An empty repository is still caught at runtime, now with wording that fits a dropdown rather than asking for a URL. Co-Authored-By: Claude Opus 5 --- component_config/configSchema.json | 1 - src/source_git.py | 4 +++- tests/test_component.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/component_config/configSchema.json b/component_config/configSchema.json index 06574f5..a4bf614 100644 --- a/component_config/configSchema.json +++ b/component_config/configSchema.json @@ -109,7 +109,6 @@ } }, "required": [ - "url", "auth" ], "properties": { diff --git a/src/source_git.py b/src/source_git.py index 6a375b4..24d11db 100644 --- a/src/source_git.py +++ b/src/source_git.py @@ -30,7 +30,9 @@ def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): self.repo_auth_url = None # ‼️ NEVER EVER INCLUDE THIS VARIABLE IN LOGGING OUTPUT ‼️ if not self.repo_url: - raise UserException("Git repository URL is required") + raise UserException( + "Please select a repository" if git_cfg.auth == AuthEnum.OAUTH else "Git repository URL is required" + ) if self.git_cfg.auth == AuthEnum.PAT: self._set_up_token_auth() diff --git a/tests/test_component.py b/tests/test_component.py index 391e08b..63915b8 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -158,6 +158,18 @@ def setUp(self): def _git_cfg(url: str = "https://github.com/keboola/example.git") -> GitConfiguration: return GitConfiguration(repository=url, auth=AuthEnum.OAUTH) + def test_missing_repository_asks_for_a_selection(self): + """The repository is picked from a dropdown here, so asking for a URL would be confusing.""" + with self.assertRaises(UserException) as context: + GitHandler(GitConfiguration(auth=AuthEnum.OAUTH), "secret-token") + self.assertIn("select a repository", str(context.exception)) + + def test_missing_url_still_asks_for_a_url(self): + """The wording for the other authentication methods is unchanged.""" + with self.assertRaises(UserException) as context: + GitHandler(GitConfiguration(auth=AuthEnum.PAT)) + self.assertIn("URL is required", str(context.exception)) + def test_missing_token_raises_user_exception(self): """An unauthorized configuration must fail with an actionable message, not with a git error.""" with self.assertRaises(UserException) as context: From 95d9b6b3c833b02ac6dec14bf5e04e690d538aa4 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:03:29 +0200 Subject: [PATCH 5/8] CFTL-589 Tell users to install the app with a narrow repository selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installation dialog defaults to a choice, not to a safe default, and the onboarding text so far only said that repositories are picked during installation. Picking "All repositories" grants Contents: Read-only across the whole account or organisation with a token that does not expire — the same over-scoped long-lived credential that motivated moving away from personal access tokens. Nothing on the Keboola side can narrow it afterwards, so the guidance has to arrive before the user clicks. Also records the app's client ID and the page where a user can review or revoke the access they granted. Co-Authored-By: Claude Opus 5 --- README.md | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f0cfffd..905f1e3 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ - [Git configuration](#git-configuration) - [SSH configuration](#ssh-configuration) - [GitHub OAuth configuration](#github-oauth-configuration) + - [Choose "Only select repositories"](#choose-only-select-repositories) - [Example: Running code saved in custom repository + template 🧩](#example-running-code-saved-in-custom-repository--template-) - [Example: Listing preinstalled packages](#example-listing-preinstalled-packages) - [Example: Accessing custom configuration parameters](#example-accessing-custom-configuration-parameters) @@ -109,10 +110,14 @@ The git configuration object supports the following parameters: With `"auth": "oauth"` no credential is entered into the configuration at all – the access token is issued by the Keboola OAuth broker. Only `https://github.com` URLs are supported. +The Keboola GitHub App has the client ID `Iv23liWeMeCpr1xBOVsj`. You can review the access you granted it, and +revoke it, at +[github.com/settings/connections/applications/Iv23liWeMeCpr1xBOVsj](https://github.com/settings/connections/applications/Iv23liWeMeCpr1xBOVsj). + Setting this up takes two separate steps on GitHub, **in this order**: -1. **Install** the Keboola GitHub App on your account or organisation - (`https://github.com/apps//installations/new`) and pick the repositories it may read. +1. **Install** the app on your account or organisation + (`https://github.com/apps//installations/new`) and choose which repositories it may read. Repository selection happens here and nowhere else. 2. **Authorize** the component in the **Authorization** section of the configuration in Keboola. 3. Pick the repository from the **Repository** dropdown, which lists what the installation makes available. @@ -125,10 +130,29 @@ Keboola will not change it. Some organisations require an owner to approve the installation before it takes effect. -The app requests `Contents: Read-only` and `Metadata: Read-only`, and the token is limited to the intersection -of those permissions and your own access. It does not authenticate private git dependencies declared in + +#### Choose "Only select repositories" + +The installation dialog offers **All repositories** or **Only select repositories**. Choose the second one and +list only the repositories this component needs. + +That dialog is the only place where the reach of the access token is decided, and narrowing it is the entire +reason to use OAuth rather than a personal access token. **All repositories** grants `Contents: Read-only` +across every repository in the account or organisation, including ones created later, and the token that +results does not expire – which is the same over-scoped, long-lived credential that a personal access token +was criticised for. Keboola cannot narrow this from its side; only the installation can. + +On a large organisation, **All repositories** also makes the **Repository** dropdown slow to load or unable to +load at all. A narrow selection avoids that. + +The token is always the intersection of what the app may read and what you can read yourself, so it never +reaches anything you could not already reach. The app requests `Contents: Read-only` and `Metadata: +Read-only`, and nothing else. It does not authenticate private git dependencies declared in `[tool.uv.sources]` – keep using `pat` if you rely on those. +To change the selection later, reconfigure the installation on GitHub. Re-authorizing in Keboola does not +change it. + ### Example: Running code saved in custom repository + template 🧩 From bb516b46229bc3c01b405139606e9e753cee4de3 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:13:39 +0200 Subject: [PATCH 6/8] CFTL-589 Fill in the real installation URL for the GitHub App The app slug is keboola-custom-python-read; it is derived from the app name and is not the client ID, so it could not be inferred from what the OAuth registration already recorded. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 905f1e3..ab5b237 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ revoke it, at Setting this up takes two separate steps on GitHub, **in this order**: 1. **Install** the app on your account or organisation - (`https://github.com/apps//installations/new`) and choose which repositories it may read. + (`https://github.com/apps/keboola-custom-python-read/installations/new`) and choose which repositories it may read. Repository selection happens here and nowhere else. 2. **Authorize** the component in the **Authorization** section of the configuration in Keboola. 3. Pick the repository from the **Repository** dropdown, which lists what the installation makes available. From aba710aec1f72d06b3ed62c0fd768386c7760719 Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:27:42 +0200 Subject: [PATCH 7/8] CFTL-589 Authenticate private git dependencies on the OAuth path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Job 53340777 cloned its repository over OAuth and then failed in `uv sync` with "could not read Username for https://github.com": uv shells out to git to fetch a private dependency, and that git had no credentials at all. The PAT path is unaffected because it leaves a ~/.netrc behind, which uv picks up on its own — but a file in the home directory is also readable by the executed user script, which is why the OAuth path does not write one. The credentials now travel to the dependency installation explicitly, through a new optional env argument on SubprocessRunner.run. The executed script keeps inheriting the untouched process environment, so the token still does not reach it. GitHandler now holds only the git-specific overrides and resolves the full environment when a subprocess is started, rather than snapshotting os.environ in the constructor. The snapshot predates the virtual environment selection, so handing it to `uv sync` would have pointed the install at the wrong environment. Co-Authored-By: Claude Opus 5 --- README.md | 7 +++-- src/component.py | 2 +- src/package_installer.py | 6 ++-- src/source_git.py | 24 +++++++++----- src/subprocess_runner.py | 2 ++ tests/test_component.py | 68 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 93 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ab5b237..ba4f502 100644 --- a/README.md +++ b/README.md @@ -147,8 +147,11 @@ load at all. A narrow selection avoids that. The token is always the intersection of what the app may read and what you can read yourself, so it never reaches anything you could not already reach. The app requests `Contents: Read-only` and `Metadata: -Read-only`, and nothing else. It does not authenticate private git dependencies declared in -`[tool.uv.sources]` – keep using `pat` if you rely on those. +Read-only`, and nothing else. + +Private git dependencies declared in `[tool.uv.sources]` authenticate with the same token and need no +credentials of their own. The repositories they live in have to be part of the installation's repository +selection as well, not just the repository holding the code. To change the selection later, reconfigure the installation on GitHub. Re-authorizing in Keboola does not change it. diff --git a/src/component.py b/src/component.py index 029ca08..02dfa25 100644 --- a/src/component.py +++ b/src/component.py @@ -104,7 +104,7 @@ def run(self): self.parameters.packages.insert(0, "keboola.component") PackageInstaller.install_packages(self.parameters.packages) else: - PackageInstaller.install_packages_for_repository(base_path) + PackageInstaller.install_packages_for_repository(base_path, git_handler.subprocess_env()) self._merge_user_parameters() diff --git a/src/package_installer.py b/src/package_installer.py index 9e2b116..933248b 100644 --- a/src/package_installer.py +++ b/src/package_installer.py @@ -17,7 +17,7 @@ def install_packages(packages: list[str]): SubprocessRunner.run(args, MSG_OK, MSG_ERR) @staticmethod - def install_packages_for_repository(repository_path: Path): + def install_packages_for_repository(repository_path: Path, env: dict[str, str] | None = None): """ Install packages based on the given repository path. - If there is a pyproject.toml and a uv.lock file, run uv sync. @@ -25,6 +25,8 @@ def install_packages_for_repository(repository_path: Path): Args: repository_path (str): Path to the repository containing requirements.txt. + env: Environment for the installation. Carries the git credentials, without which + private git dependencies cannot be fetched. """ pyproject_file = repository_path / "pyproject.toml" uv_lock_file = repository_path / "uv.lock" @@ -46,4 +48,4 @@ def install_packages_for_repository(repository_path: Path): logging.info("No dependencies file found") return - SubprocessRunner.run(args, MSG_OK, MSG_ERR) + SubprocessRunner.run(args, MSG_OK, MSG_ERR, env) diff --git a/src/source_git.py b/src/source_git.py index 24d11db..ee5faac 100644 --- a/src/source_git.py +++ b/src/source_git.py @@ -24,7 +24,9 @@ def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): # add path for absolute imports to start at the cloned repository root level sys.path.append(str(Path(__file__).parent.parent / GitHandler.REPO_PATH)) - self.env = os.environ.copy() + # only the git-specific overrides; the full environment is resolved at call time so that + # changes made after the clone (the virtual environment selection) are not lost + self.git_env: dict[str, str] = {} self.git_cfg = git_cfg self.repo_url = git_cfg.repository_url self.repo_auth_url = None # ‼️ NEVER EVER INCLUDE THIS VARIABLE IN LOGGING OUTPUT ‼️ @@ -43,7 +45,7 @@ def __init__(self, git_cfg: GitConfiguration, oauth_token: str | None = None): self._set_up_ssh_command() # do not ask for credentials when git authentication fails - self.env["GIT_TERMINAL_PROMPT"] = "0" + self.git_env["GIT_TERMINAL_PROMPT"] = "0" def _set_up_token_auth(self) -> None: if not self.git_cfg.encrypted_token: @@ -80,8 +82,8 @@ def _set_up_oauth_auth(self, oauth_token: str | None) -> None: # only the username goes into the URL, the token itself is supplied by the askpass helper self.repo_auth_url = self.repo_url.replace("https://", f"https://{OAUTH_GIT_USERNAME}@") - self.env[OAUTH_TOKEN_ENV] = oauth_token - self.env["GIT_ASKPASS"] = str(self._write_askpass_helper()) + self.git_env[OAUTH_TOKEN_ENV] = oauth_token + self.git_env["GIT_ASKPASS"] = str(self._write_askpass_helper()) logging.info("Git OAuth authentication set up for GitHub URL.") @staticmethod @@ -121,7 +123,7 @@ def _set_up_ssh_command(self) -> None: os.chmod(ssh_key_path, 0o600) ssh_command.extend(["-i", str(ssh_key_path)]) - self.env["GIT_SSH_COMMAND"] = " ".join(ssh_command) + self.git_env["GIT_SSH_COMMAND"] = " ".join(ssh_command) def _explain_error(self, error_msg: str) -> str: """Append an actionable hint to git errors whose raw wording does not point at the actual cause.""" @@ -139,6 +141,14 @@ def _explain_error(self, error_msg: str) -> str: return error_msg + def subprocess_env(self) -> dict[str, str]: + """Environment for a subprocess that needs to reach the repository, credentials included. + + Also used for the dependency installation, so that private git dependencies declared in the + repository authenticate with the same credentials as the clone itself. + """ + return {**os.environ, **self.git_env} + def clone_repository(self, sync_action=False) -> Path: """ Clone a git repository and return the path to the cloned code. @@ -162,7 +172,7 @@ def clone_repository(self, sync_action=False) -> Path: clone_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.env, + env=self.subprocess_env(), ) _, stderr = process.communicate() @@ -203,7 +213,7 @@ def get_repository_branches(self): branches_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.env, + env=self.subprocess_env(), ) stdout, stderr = process.communicate() diff --git a/src/subprocess_runner.py b/src/subprocess_runner.py index f6e68f2..322231d 100644 --- a/src/subprocess_runner.py +++ b/src/subprocess_runner.py @@ -63,6 +63,7 @@ def run( args: list[str], ok_message: str = "Command finished successfully.", err_message: str = "Command failed.", + env: dict[str, str] | None = None, ): logging.debug("Running command: %s", " ".join(args)) process = subprocess.Popen( @@ -70,6 +71,7 @@ def run( stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=env, ) stderr_output: deque[str] = deque(maxlen=MAX_STDERR_LINES) diff --git a/tests/test_component.py b/tests/test_component.py index 63915b8..ca19cbb 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -13,6 +13,7 @@ from component import Component from configuration import AuthEnum, Configuration, GitConfiguration, SourceEnum, VenvEnum from github_api import GitHubApi +from package_installer import PackageInstaller from source_git import GitHandler @@ -190,9 +191,9 @@ def test_token_is_not_part_of_the_clone_url(self): def test_token_is_passed_through_the_askpass_helper(self): """The helper is executable and reads the token from the environment instead of embedding it.""" handler = GitHandler(self._git_cfg(), "secret-token") - self.assertEqual(handler.env["GIT_OAUTH_TOKEN"], "secret-token") + self.assertEqual(handler.git_env["GIT_OAUTH_TOKEN"], "secret-token") - askpass_path = Path(handler.env["GIT_ASKPASS"]) + askpass_path = Path(handler.git_env["GIT_ASKPASS"]) self.assertTrue(os.access(askpass_path, os.X_OK)) self.assertNotIn("secret-token", askpass_path.read_text()) @@ -207,12 +208,26 @@ def test_unrelated_errors_are_not_annotated(self): handler = GitHandler(self._git_cfg(), "secret-token") self.assertEqual(handler._explain_error("fatal: could not read from remote"), "fatal: could not read from remote") + def test_token_stays_out_of_the_process_environment(self): + """The executed user script inherits os.environ, so the token must never be put there.""" + handler = GitHandler(self._git_cfg(), "secret-token") + self.assertNotIn("GIT_OAUTH_TOKEN", os.environ) + self.assertEqual(handler.subprocess_env()["GIT_OAUTH_TOKEN"], "secret-token") + + def test_subprocess_env_reflects_later_environment_changes(self): + """The virtual environment is chosen after the clone, so the env cannot be a stale snapshot.""" + handler = GitHandler(self._git_cfg(), "secret-token") + with mock.patch.dict(os.environ, {"UV_PROJECT_ENVIRONMENT": "/code/repo_clone/.venv"}): + env = handler.subprocess_env() + self.assertEqual(env["UV_PROJECT_ENVIRONMENT"], "/code/repo_clone/.venv") + self.assertEqual(env["GIT_OAUTH_TOKEN"], "secret-token") + def test_other_auth_methods_are_untouched(self): """A configuration that does not use OAuth must not gain any OAuth environment.""" handler = GitHandler(GitConfiguration(url="https://github.com/keboola/example.git")) self.assertIsNone(handler.repo_auth_url) - self.assertNotIn("GIT_ASKPASS", handler.env) - self.assertNotIn("GIT_OAUTH_TOKEN", handler.env) + self.assertNotIn("GIT_ASKPASS", handler.git_env) + self.assertNotIn("GIT_OAUTH_TOKEN", handler.git_env) def test_ssh_hint_is_preserved(self): """The pre-existing hint for SSH failures must keep working for non-OAuth configurations.""" @@ -381,6 +396,51 @@ def test_other_methods_use_the_typed_url(self): self.assertEqual(cfg.repository_url, "https://github.com/acme/typed.git") +class TestDependencyInstallationCredentials(unittest.TestCase): + """A repository's private git dependencies must authenticate with the credentials of the clone. + + `uv sync` shells out to git, which offers no credentials of its own, so without an explicit + environment the fetch fails with "could not read Username for https://github.com". + """ + + def _repository(self, *files: str) -> Path: + repo = tempfile.TemporaryDirectory() + self.addCleanup(repo.cleanup) + self.addCleanup(os.chdir, os.getcwd()) + repo_path = Path(repo.name) + for name in files: + (repo_path / name).write_text("") + return repo_path + + def test_environment_is_forwarded_to_uv_sync(self): + repo_path = self._repository("pyproject.toml", "uv.lock") + + with mock.patch("package_installer.SubprocessRunner.run") as run: + PackageInstaller.install_packages_for_repository(repo_path, {"GIT_OAUTH_TOKEN": "secret-token"}) + + args = run.call_args.args + self.assertEqual(args[0], ["uv", "sync", "--inexact"]) + self.assertEqual(args[3], {"GIT_OAUTH_TOKEN": "secret-token"}) + + def test_environment_is_forwarded_to_requirements_install(self): + """requirements.txt can reference private git URLs just as pyproject.toml can.""" + repo_path = self._repository("requirements.txt") + + with mock.patch("package_installer.SubprocessRunner.run") as run: + PackageInstaller.install_packages_for_repository(repo_path, {"GIT_OAUTH_TOKEN": "secret-token"}) + + self.assertEqual(run.call_args.args[3], {"GIT_OAUTH_TOKEN": "secret-token"}) + + def test_installation_without_credentials_still_works(self): + """Configurations that need no credentials must keep inheriting the process environment.""" + repo_path = self._repository("pyproject.toml", "uv.lock") + + with mock.patch("package_installer.SubprocessRunner.run") as run: + PackageInstaller.install_packages_for_repository(repo_path) + + self.assertIsNone(run.call_args.args[3]) + + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] unittest.main() From 0c1a6d86e3bc2d8216a33881efae91c5ba69ce6b Mon Sep 17 00:00:00 2001 From: Oscar-XXII <305386918+Oscar-XXII@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:51:51 +0200 Subject: [PATCH 8/8] CFTL-589 Update the installation URL after the app was renamed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The app is now "Keboola Custom Python", which moves its slug from keboola-custom-python-read to keboola-custom-python. The old slug returns 404 — GitHub does not redirect a renamed app — so the previous link would simply have been dead. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba4f502..badbc6f 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ revoke it, at Setting this up takes two separate steps on GitHub, **in this order**: 1. **Install** the app on your account or organisation - (`https://github.com/apps/keboola-custom-python-read/installations/new`) and choose which repositories it may read. + (`https://github.com/apps/keboola-custom-python/installations/new`) and choose which repositories it may read. Repository selection happens here and nowhere else. 2. **Authorize** the component in the **Authorization** section of the configuration in Keboola. 3. Pick the repository from the **Repository** dropdown, which lists what the installation makes available.