feat: support auth for git+https repos

Use the credentials configured within repositories.yaml also for git+https urls.
To avoid reloading the config, move repository-related code into a separate package.

Author: mgoltzsche

cmd/khelm/common.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/mgoltzsche/khelm/v2/pkg/config"
 	"github.com/mgoltzsche/khelm/v2/pkg/helm"
+	"github.com/mgoltzsche/khelm/v2/pkg/repositories"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -23,7 +24,7 @@ func render(h *helm.Helm, req *config.ChartConfig) ([]*yaml.RNode, error) {
 	}()
 
 	rendered, err := h.Render(ctx, req)
-	if helm.IsUntrustedRepository(err) {
+	if repositories.IsUntrustedRepository(err) {
 		log.Printf("HINT: access to untrusted repositories can be enabled using env var %s=true or option --%s", envTrustAnyRepo, flagTrustAnyRepo)
 	}
 	return rendered, err

example/git-getter/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 description: example chart using a git url as dependency
-name: git-getter-example-chart
+name: git-getter
 version: 0.1.0
 dependencies:
 - name: cert-manager

example/localrefref-with-git/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: intermediate-chart
+  repository: file://../localref/intermediate-chart
+  version: 0.1.1
+- name: git-getter
+  repository: git+https://github.com/mgoltzsche/khelm@example?ref=afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62
+  version: 0.1.0
+digest: sha256:83407caeeb8ff5cf80c07d046efe3ee2961dda5d9dce64ee95d162a124263dfd
+generated: "2022-10-17T01:20:17.281584028Z"

example/localrefref-with-git/Chart.yaml

@@ -0,0 +1,13 @@
+apiVersion: v2
+description: Chart that refers another one that has dependencies itself
+name: myumbrella-chart
+version: 0.2.0
+dependencies:
+- name: intermediate-chart
+  version: "~0.1.0"
+  repository: "file://../localref/intermediate-chart"
+- name: "git-getter"
+  version: "x.x.x"
+  repository: "git+https://github.com/mgoltzsche/khelm@example?ref=afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62"
+#afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62
+#3857e76b956c4652d3cacc2fd71e7dcff5843302

example/localrefref-with-git/generator.yaml

@@ -0,0 +1,6 @@
+apiVersion: khelm.mgoltzsche.github.com/v2
+kind: ChartRenderer
+metadata:
+  name: mychart
+  namespace: myotherns
+chart: .

pkg/getter/git/checkout.go

@@ -2,78 +2,84 @@ package git
 
 import (
 	"context"
+	"encoding/hex"
 	"fmt"
+	"log"
+	"strings"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"helm.sh/helm/v3/pkg/repo"
 )
 
-func gitCheckoutImpl(ctx context.Context, repoURL, ref, destDir string) error {
-	/*err := runCmds(destDir, [][]string{
-		{"git", "init", "--quiet"},
-		{"git", "remote", "add", "origin", repoURL},
-
-		{"git", "config", "core.sparseCheckout", "true"},
-		{"git", "sparse-checkout", "set", path},
-		{"git", "pull", "--quiet", "--depth", "1", "origin", ref},
-
-		//{"git", "fetch", "--quiet", "--tags", "origin"},
-		//{"git", "checkout", "--quiet", ref},
-	})*/
-	r, err := git.PlainCloneContext(ctx, destDir, false, &git.CloneOptions{
-		URL: repoURL,
-		// TODO: Auth: ...
-		RemoteName:   "origin",
-		SingleBranch: true,
-		Depth:        1,
-		NoCheckout:   true,
-	})
+func gitCheckoutImpl(ctx context.Context, repoURL, ref string, repo *repo.Entry, destDir string) error {
+	cloneOpts := git.CloneOptions{
+		URL:        repoURL,
+		RemoteName: "origin",
+		NoCheckout: true,
+	}
+	isCommitRef := isCommitSHA(ref)
+	if !isCommitRef {
+		cloneOpts.SingleBranch = true
+		cloneOpts.Depth = 1
+	}
+	scheme := strings.SplitN(repoURL, ":", 2)[0]
+	switch scheme {
+	case "https":
+		cloneOpts.Auth = &http.BasicAuth{
+			Username: repo.Username,
+			Password: repo.Password,
+		}
+	default:
+		if repo.Username != "" || repo.Password != "" {
+			log.Printf("WARNING: ignoring auth config for %s since authentication is not supported for url scheme %q", repoURL, scheme)
+		}
+	}
+	r, err := git.PlainCloneContext(ctx, destDir, false, &cloneOpts)
 	if err != nil {
-		return err
+		return fmt.Errorf("git clone: %w", err)
 	}
 	tree, err := r.Worktree()
 	if err != nil {
 		return fmt.Errorf("git worktree: %w", err)
 	}
 	// TODO: support sparse checkout, see https://github.com/go-git/go-git/issues/90
+	refType := "without ref"
 	opts := git.CheckoutOptions{}
 	if ref != "" {
-		opts.Branch = plumbing.ReferenceName("refs/tags/" + ref)
+		// TODO: consider making this more usable: Currently resolving a commit as shown below doesn't work when the commit is not within the main branch (object not found).
+		if isCommitRef {
+			opts.Hash = plumbing.NewHash(ref)
+			refType = fmt.Sprintf("commit %s", ref)
+		} else {
+			opts.Branch = plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", ref))
+			refType = fmt.Sprintf("tag %s", ref)
+		}
 	}
 	err = tree.Checkout(&opts)
 	if err != nil {
-		return err
+		return fmt.Errorf("git checkout %s: %w", refType, err)
 	}
-	return nil
-}
+	/*err := runCmds(destDir, [][]string{
+		{"git", "init", "--quiet"},
+		{"git", "remote", "add", "origin", repoURL},
 
-/*
-func runCmds(dir string, cmds [][]string) error {
-	for _, c := range cmds {
-		err := runCmd(dir, c[0], c[1:]...)
-		if err != nil {
-			return err
-		}
-	}
+		{"git", "config", "core.sparseCheckout", "true"},
+		{"git", "sparse-checkout", "set", path},
+		{"git", "pull", "--quiet", "--depth", "1", "origin", ref},
+
+		//{"git", "fetch", "--quiet", "--tags", "origin"},
+		//{"git", "checkout", "--quiet", ref},
+	})*/
 	return nil
 }
 
-func runCmd(dir, cmd string, args ...string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-	c := exec.CommandContext(ctx, cmd, args...)
-	var stderr bytes.Buffer
-	c.Stderr = &stderr
-	c.Dir = dir
-	err := c.Run()
-	if err != nil {
-		msg := strings.TrimSpace(stderr.String())
-		if msg == "" {
-			msg = err.Error()
+func isCommitSHA(s string) bool {
+	if len(s) == 40 {
+		if _, err := hex.DecodeString(s); err == nil {
+			return true
 		}
-		cmds := append([]string{cmd}, args...)
-		return fmt.Errorf("%s: %s", strings.Join(cmds, " "), msg)
 	}
-	return err
+	return false
 }
-*/

pkg/getter/git/gitgetter.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/mgoltzsche/khelm/v2/pkg/repositories"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/cli"
 	helmgetter "helm.sh/helm/v3/pkg/getter"
@@ -23,16 +24,24 @@ var gitCheckout = gitCheckoutImpl
 
 type HelmPackageFunc func(ctx context.Context, path, repoDir string) (string, error)
 
-func New(settings *cli.EnvSettings, packageFn HelmPackageFunc) helmgetter.Constructor {
+type RepositoriesFunc func() (repositories.Interface, error)
+
+func New(settings *cli.EnvSettings, reposFn RepositoriesFunc, packageFn HelmPackageFunc) helmgetter.Constructor {
 	return func(o ...helmgetter.Option) (helmgetter.Getter, error) {
+		repos, err := reposFn()
+		if err != nil {
+			return nil, err
+		}
 		return &gitIndexGetter{
 			settings:  settings,
+			repos:     repos,
 			packageFn: packageFn,
 		}, nil
 	}
 }
 
 type gitIndexGetter struct {
+	repos     repositories.Interface
 	settings  *cli.EnvSettings
 	Getters   helmgetter.Providers
 	packageFn HelmPackageFunc
@@ -52,7 +61,7 @@ func (g *gitIndexGetter) Get(location string, options ...helmgetter.Option) (*by
 	if isRepoIndex {
 		// Generate repo index from directory
 		ref = ref.Dir()
-		repoDir, err := download(ctx, ref, g.settings.RepositoryCache)
+		repoDir, err := download(ctx, ref, g.settings.RepositoryCache, g.repos)
 		if err != nil {
 			return nil, err
 		}
@@ -67,8 +76,9 @@ func (g *gitIndexGetter) Get(location string, options ...helmgetter.Option) (*by
 		}
 	} else {
 		// Build and package chart
-		chartPath := filepath.FromSlash(strings.TrimSuffix(ref.Path, ".tgz"))
-		repoDir, err := download(ctx, ref, g.settings.RepositoryCache)
+		ref.Path = strings.TrimSuffix(ref.Path, ".tgz")
+		chartPath := filepath.FromSlash(ref.Path)
+		repoDir, err := download(ctx, ref, g.settings.RepositoryCache, g.repos)
 		if err != nil {
 			return nil, err
 		}
@@ -136,15 +146,19 @@ func generateRepoIndex(dir, cacheDir string, u *gitURL) (*repo.IndexFile, error)
 	return idx, nil
 }
 
-func download(ctx context.Context, ref *gitURL, cacheDir string) (string, error) {
+func download(ctx context.Context, ref *gitURL, cacheDir string, repos repositories.Interface) (string, error) {
 	repoRef := *ref
 	repoRef.Path = ""
 	cacheKey := fmt.Sprintf("sha256-%x", sha256.Sum256([]byte(repoRef.String())))
 	cacheDir = filepath.Join(cacheDir, "git")
 	destDir := filepath.Join(cacheDir, cacheKey)
 
 	if _, e := os.Stat(destDir); os.IsNotExist(e) {
-		err := os.MkdirAll(cacheDir, 0755)
+		auth, _, err := repos.Get("git+" + ref.String())
+		if err != nil {
+			return "", err
+		}
+		err = os.MkdirAll(cacheDir, 0755)
 		if err != nil {
 			return "", err
 		}
@@ -155,7 +169,7 @@ func download(ctx context.Context, ref *gitURL, cacheDir string) (string, error)
 		defer os.RemoveAll(tmpDir)
 
 		tmpRepoDir := tmpDir
-		err = gitCheckout(ctx, ref.Repo, ref.Ref, tmpRepoDir)
+		err = gitCheckout(ctx, ref.Repo, ref.Ref, auth, tmpRepoDir)
 		if err != nil {
 			return "", err
 		}

pkg/getter/git/gitgetter_test.go

@@ -7,9 +7,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/mgoltzsche/khelm/v2/pkg/repositories"
 	"github.com/stretchr/testify/require"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/repo"
 	"sigs.k8s.io/yaml"
 )
@@ -18,7 +20,7 @@ func TestGitGetter(t *testing.T) {
 	tmpDir, err := os.MkdirTemp("", "khelm-git-getter-test-")
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpDir)
-	gitCheckout = func(_ context.Context, repoURL, ref, destDir string) error {
+	gitCheckout = func(_ context.Context, repoURL, ref string, auth *repo.Entry, destDir string) error {
 		err := os.MkdirAll(filepath.Join(destDir, "mypath", "fakechart"), 0755)
 		require.NoError(t, err)
 		err = os.WriteFile(filepath.Join(destDir, "mypath", "fakechart", "Chart.yaml"), []byte(`
@@ -38,7 +40,11 @@ version: 0.1.0`), 0600)
 		require.NoError(t, err)
 		return file, nil
 	}
-	testee := New(settings, fakePackageFn)
+	reposFn := func() (repositories.Interface, error) {
+		trust := true
+		return repositories.New(*settings, getter.All(settings), &trust)
+	}
+	testee := New(settings, reposFn, fakePackageFn)
 	getter, err := testee()
 	require.NoError(t, err)
 

pkg/helm/helm.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mgoltzsche/khelm/v2/pkg/repositories"
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/helmpath"
@@ -14,6 +15,7 @@ type Helm struct {
 	TrustAnyRepository *bool
 	Settings           cli.EnvSettings
 	Getters            getter.Providers
+	repos              repositories.Interface
 }
 
 // NewHelm creates a new helm environment
@@ -24,6 +26,18 @@ func NewHelm() *Helm {
 		settings.RepositoryConfig = filepath.Join(helmHome, "repository", "repositories.yaml")
 	}
 	h := &Helm{Settings: *settings}
-	h.Getters = getters(settings, &h.TrustAnyRepository)
+	h.Getters = getters(settings, h.repositories)
 	return h
 }
+
+func (h *Helm) repositories() (repositories.Interface, error) {
+	if h.repos != nil {
+		return h.repos, nil
+	}
+	repos, err := repositories.New(h.Settings, h.Getters, h.TrustAnyRepository)
+	if err != nil {
+		return nil, err
+	}
+	h.repos = repos
+	return repos, nil
+}