[ci_runner] Support using git credentials pre-configured on self-host…

…ed executors (#7231)

cli/remotebazel/remotebazel.go

@@ -63,14 +63,15 @@ const (
 var (
 	RemoteFlagset = flag.NewFlagSet("remote", flag.ContinueOnError)
 
-	execOs         = RemoteFlagset.String("os", "", "If set, requests execution on a specific OS.")
-	execArch       = RemoteFlagset.String("arch", "", "If set, requests execution on a specific CPU architecture.")
-	containerImage = RemoteFlagset.String("container_image", "", "If set, requests execution on a specific runner image. Otherwise uses the default hosted runner version. A `docker://` prefix is required.")
-	envInput       = bbflag.New(RemoteFlagset, "env", []string{}, "Environment variables to set in the runner environment. Key-value pairs can either be separated by '=' (Ex. --env=k1=val1), or if only a key is specified, the value will be taken from the invocation environment (Ex. --env=k2). To apply multiple env vars, pass the env flag multiple times (Ex. --env=k1=v1 --env=k2). If the same key is given twice, the latest will apply.")
-	remoteRunner   = RemoteFlagset.String("remote_runner", defaultRemoteExecutionURL, "The Buildbuddy grpc target the remote runner should run on.")
-	timeout        = RemoteFlagset.Duration("timeout", 0, "If set, requests that have exceeded this timeout will be canceled automatically. (Ex. --timeout=15m; --timeout=2h)")
-	execPropsFlag  = bbflag.New(RemoteFlagset, "runner_exec_properties", []string{}, "Exec properties that will apply to the *ci runner execution*. Key-value pairs should be separated by '=' (Ex. --runner_exec_properties=NAME=VALUE). Can be specified more than once. NOTE: If you want to apply an exec property to the bazel command that's run on the runner, just pass at the end of the command (Ex. bb remote build //... --remote_default_exec_properties=OSFamily=linux).")
-	runRemotely    = RemoteFlagset.Bool("run_remotely", true, "For `run` commands, whether the target should be run remotely. If false, the target will be built remotely, and then fetched and run locally.")
+	execOs                  = RemoteFlagset.String("os", "", "If set, requests execution on a specific OS.")
+	execArch                = RemoteFlagset.String("arch", "", "If set, requests execution on a specific CPU architecture.")
+	containerImage          = RemoteFlagset.String("container_image", "", "If set, requests execution on a specific runner image. Otherwise uses the default hosted runner version. A `docker://` prefix is required.")
+	envInput                = bbflag.New(RemoteFlagset, "env", []string{}, "Environment variables to set in the runner environment. Key-value pairs can either be separated by '=' (Ex. --env=k1=val1), or if only a key is specified, the value will be taken from the invocation environment (Ex. --env=k2). To apply multiple env vars, pass the env flag multiple times (Ex. --env=k1=v1 --env=k2). If the same key is given twice, the latest will apply.")
+	remoteRunner            = RemoteFlagset.String("remote_runner", defaultRemoteExecutionURL, "The Buildbuddy grpc target the remote runner should run on.")
+	timeout                 = RemoteFlagset.Duration("timeout", 0, "If set, requests that have exceeded this timeout will be canceled automatically. (Ex. --timeout=15m; --timeout=2h)")
+	execPropsFlag           = bbflag.New(RemoteFlagset, "runner_exec_properties", []string{}, "Exec properties that will apply to the *ci runner execution*. Key-value pairs should be separated by '=' (Ex. --runner_exec_properties=NAME=VALUE). Can be specified more than once. NOTE: If you want to apply an exec property to the bazel command that's run on the runner, just pass at the end of the command (Ex. bb remote build //... --remote_default_exec_properties=OSFamily=linux).")
+	runRemotely             = RemoteFlagset.Bool("run_remotely", true, "For `run` commands, whether the target should be run remotely. If false, the target will be built remotely, and then fetched and run locally.")
+	useSystemGitCredentials = RemoteFlagset.Bool("use_system_git_credentials", false, "Whether to use github auth pre-configured on the remote runner. If false, require https and an access token for git access.")
 
 	defaultBranchRefs = []string{"refs/heads/main", "refs/heads/master"}
 )
@@ -746,14 +747,19 @@ func Run(ctx context.Context, opts RunOpts, repoConfig *RepoConfig) (int, error)
 		envVars["GIT_REPO_DEFAULT_BRANCH"] = defaultBranch
 	}
 
+	if *useSystemGitCredentials {
+		envVars["USE_SYSTEM_GIT_CREDENTIALS"] = "1"
+	}
+
 	platform, err := rexec.MakePlatform(*execPropsFlag...)
 	if err != nil {
 		return 0, status.InvalidArgumentErrorf("invalid exec properties - key value pairs must be separated by '=': %s", err)
 	}
 
 	req := &rnpb.RunRequest{
 		GitRepo: &gitpb.GitRepo{
-			RepoUrl: repoConfig.URL,
+			RepoUrl:                 repoConfig.URL,
+			UseSystemGitCredentials: *useSystemGitCredentials,
 		},
 		RepoState: &gitpb.RepoState{
 			CommitSha: repoConfig.CommitSHA,

enterprise/server/cmd/ci_runner/main.go

@@ -2042,7 +2042,8 @@ func (ws *workspace) config(ctx context.Context) error {
 	// Set up global config (~/.gitconfig) but only on Linux for now since Linux
 	// workflows are isolated.
 	// TODO(bduffany): find a solution that works for Mac workflows too.
-	if runtime.GOOS == "linux" {
+	useSystemGitCredentials := os.Getenv("USE_SYSTEM_GIT_CREDENTIALS") == "1"
+	if !useSystemGitCredentials && runtime.GOOS == "linux" {
 		// SSH URL rewrites and git credential helper are used for external git
 		// deps fetched by bazel, so these need to be in the global config.
 		if err := configureGlobalURLRewrites(ctx); err != nil {
@@ -2067,20 +2068,28 @@ func (ws *workspace) fetch(ctx context.Context, remoteURL string, refs []string,
 	if len(refs) == 0 {
 		return nil
 	}
-	authURL, err := gitutil.AuthRepoURL(remoteURL, os.Getenv(repoUserEnvVarName), os.Getenv(repoTokenEnvVarName))
-	if err != nil {
-		return err
+
+	fetchURL := remoteURL
+	useSystemGitCredentials := os.Getenv("USE_SYSTEM_GIT_CREDENTIALS") == "1"
+	if !useSystemGitCredentials {
+		authURL, err := gitutil.AuthRepoURL(remoteURL, os.Getenv(repoUserEnvVarName), os.Getenv(repoTokenEnvVarName))
+		if err != nil {
+			return err
+		}
+		fetchURL = authURL
 	}
+
 	remoteName := gitRemoteName(remoteURL)
 	writeCommandSummary(ws.log, "Configuring remote %q...", remoteName)
+
 	// Don't show `git remote add` command or the error message since the URL may
 	// contain the repo access token.
-	if _, err := git(ctx, io.Discard, "remote", "add", remoteName, authURL); err != nil {
+	if _, err := git(ctx, io.Discard, "remote", "add", remoteName, fetchURL); err != nil {
 		// Rename the existing remote. Removing then re-adding would be simpler,
 		// but unfortunately that drops the "partialclonefilter" options on the
 		// existing remote.
 		if isRemoteAlreadyExists(err) {
-			if _, err := git(ctx, io.Discard, "remote", "set-url", remoteName, authURL); err != nil {
+			if _, err := git(ctx, io.Discard, "remote", "set-url", remoteName, fetchURL); err != nil {
 				return err
 			}
 		} else {

enterprise/server/hostedrunner/hostedrunner.go

@@ -101,10 +101,14 @@ func (r *runnerService) createAction(ctx context.Context, req *rnpb.RunRequest,
 		patchURIs = append(patchURIs, uri)
 	}
 
-	// Use https for git operations.
-	repoURL, err := git.NormalizeRepoURL(req.GetGitRepo().GetRepoUrl())
-	if err != nil {
-		return nil, status.WrapError(err, "normalize git repo")
+	repoURL := req.GetGitRepo().GetRepoUrl()
+	if !req.GetGitRepo().GetUseSystemGitCredentials() {
+		// Use https for git operations.
+		u, err := git.NormalizeRepoURL(req.GetGitRepo().GetRepoUrl())
+		if err != nil {
+			return nil, status.WrapError(err, "normalize git repo")
+		}
+		repoURL = u.String()
 	}
 
 	// TODO(Maggie) - Remove bazel_sub_command and do this unconditionally
@@ -128,8 +132,8 @@ func (r *runnerService) createAction(ctx context.Context, req *rnpb.RunRequest,
 		"--cache_backend=" + cache_api_url.String(),
 		"--rbe_backend=" + remote_exec_api_url.String(),
 		"--bes_results_url=" + build_buddy_url.WithPath("/invocation/").String(),
-		"--target_repo_url=" + repoURL.String(),
-		"--pushed_repo_url=" + repoURL.String(),
+		"--target_repo_url=" + repoURL,
+		"--pushed_repo_url=" + repoURL,
 		"--pushed_branch=" + req.GetRepoState().GetBranch(),
 		"--bazel_sub_command=" + req.GetBazelCommand(),
 		"--invocation_id=" + invocationID,
@@ -151,7 +155,7 @@ func (r *runnerService) createAction(ctx context.Context, req *rnpb.RunRequest,
 
 	affinityKey := req.GetSessionAffinityKey()
 	if affinityKey == "" {
-		affinityKey = repoURL.String()
+		affinityKey = repoURL
 	}
 
 	// By default, use the non-root user as the operating user on the runner.

proto/git.proto

@@ -20,13 +20,23 @@ message GitRepo {
   // GitHub account, if the repo URL is a GitHub URL. Otherwise,
   // an error is returned.
   //
+  // This is ignored if `use_system_git_credentials` if set.
+  //
   // Ex. "ABASDBASDBASBD"
   string access_token = 2;
 
   // The username to use when cloning this repository.
   // This is required for Bitbucket, whose "app passwords" require an
   // associated username.
   string username = 3;
+
+  // Whether to use github credentials configured on the system.
+  //
+  // By default, we require https for git operations and generate short-lived
+  // access tokens using our github app installation.
+  // If GitHub SSH access is already configured on the runners, set this
+  // to true to skip doing that and use the configured auth.
+  bool use_system_git_credentials = 4;
 }
 
 message RepoState {