Allow specifying git-host-to-provider-type mapping for on-prem gitlab…

… support (#1219)

* Allow specifying git-provider (github/gitlab) as an env var
  - Rather than deriving it from the URL, which doesn't work for self hosted gitlab instances
* Adds support for a host=>provider mapping

cmd/gitops/root/cmd.go

@@ -3,10 +3,12 @@ package root
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/go-resty/resty/v2"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	wego "github.com/weaveworks/weave-gitops/api/v1alpha1"
 	"github.com/weaveworks/weave-gitops/cmd/gitops/add"
 	beta "github.com/weaveworks/weave-gitops/cmd/gitops/beta/cmd"
@@ -33,6 +35,18 @@ var options struct {
 	endpoint          string
 	overrideInCluster bool
 	verbose           bool
+	gitHostTypes      map[string]string
+}
+
+// Only want AutomaticEnv to be called once!
+func init() {
+	// Setup flag to env mapping:
+	//   config-repo => GITOPS_CONFIG_REPO
+	replacer := strings.NewReplacer("-", "_")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetEnvPrefix("GITOPS")
+
+	viper.AutomaticEnv()
 }
 
 func RootCmd(client *resty.Client) *cobra.Command {
@@ -86,6 +100,12 @@ func RootCmd(client *resty.Client) *cobra.Command {
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			configureLogger()
 
+			// Sync flag values and env vars.
+			err := viper.BindPFlags(cmd.Flags())
+			if err != nil {
+				log.Fatalf("Error binding viper to flags: %v", err)
+			}
+
 			ns, _ := cmd.Flags().GetString("namespace")
 
 			if ns == "" {
@@ -106,7 +126,9 @@ func RootCmd(client *resty.Client) *cobra.Command {
 	rootCmd.PersistentFlags().String("namespace", wego.DefaultNamespace, "The namespace scope for this operation")
 	rootCmd.PersistentFlags().StringVarP(&options.endpoint, "endpoint", "e", os.Getenv("WEAVE_GITOPS_ENTERPRISE_API_URL"), "The Weave GitOps Enterprise HTTP API endpoint")
 	rootCmd.PersistentFlags().BoolVar(&options.overrideInCluster, "override-in-cluster", false, "override running in cluster check")
+	rootCmd.PersistentFlags().StringToStringVar(&options.gitHostTypes, "git-host-types", map[string]string{}, "Specify which custom domains are running what (github or gitlab)")
 	cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("override-in-cluster"))
+	cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("git-host-types"))
 
 	rootCmd.AddCommand(install.Cmd)
 	rootCmd.AddCommand(beta.Cmd)

cmd/internal/provider.go

@@ -36,7 +36,11 @@ func (c *gitProviderClient) GetProvider(repoUrl gitproviders.RepoURL, getAccount
 		return nil, err
 	}
 
-	provider, err := gitproviders.New(gitproviders.Config{Provider: repoUrl.Provider(), Token: token}, repoUrl.Owner(), getAccountType)
+	provider, err := gitproviders.New(gitproviders.Config{
+		Provider: repoUrl.Provider(),
+		Token:    token,
+		Hostname: repoUrl.URL().Host,
+	}, repoUrl.Owner(), getAccountType)
 	if err != nil {
 		return nil, fmt.Errorf("error creating git provider client: %w", err)
 	}

pkg/gitproviders/factory.go

@@ -41,28 +41,39 @@ func buildGitProvider(config Config) (gitprovider.Client, string, error) {
 		opts := []gitprovider.ClientOption{
 			gitprovider.WithOAuth2Token(config.Token),
 		}
-		if config.Hostname != "" {
-			opts = append(opts, gitprovider.WithDomain(config.Hostname))
+
+		// Quirk of ggp, if using github.com or gitlab.com and you prepend
+		// that with https:// you end up with https://https//github.com !!!
+		hostname := github.DefaultDomain
+		if config.Hostname != "" && config.Hostname != github.DefaultDomain {
+			// Quirk of ggp, have to specify scheme with custom domain
+			hostname = "https://" + config.Hostname
+			opts = append(opts, gitprovider.WithDomain(hostname))
 		}
 
 		if client, err := github.NewClient(opts...); err != nil {
 			return nil, "", err
 		} else {
-			return client, github.DefaultDomain, nil
+			return client, hostname, nil
 		}
 	case GitProviderGitLab:
 		opts := []gitprovider.ClientOption{
 			gitprovider.WithOAuth2Token(config.Token),
 			gitprovider.WithConditionalRequests(true),
 		}
-		if config.Hostname != "" {
-			opts = append(opts, gitprovider.WithDomain(config.Hostname))
+
+		// Quirk, see above
+		hostname := gitlab.DefaultDomain
+		if config.Hostname != "" && config.Hostname != gitlab.DefaultDomain {
+			// Quirk, see above
+			hostname = "https://" + config.Hostname
+			opts = append(opts, gitprovider.WithDomain(hostname))
 		}
 
 		if client, err := gitlab.NewClient(config.Token, tokenTypeOauth, opts...); err != nil {
 			return nil, "", err
 		} else {
-			return client, gitlab.DefaultDomain, nil
+			return client, hostname, nil
 		}
 	default:
 		return nil, "", fmt.Errorf("unsupported Git provider '%s'", config.Provider)

pkg/gitproviders/factory_test.go

@@ -0,0 +1,44 @@
+package gitproviders
+
+import (
+	"github.com/fluxcd/go-git-providers/gitprovider"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+)
+
+type expectedGitProvider struct {
+	clientDomain     string
+	clientProviderID string
+	hostname         string
+}
+
+var _ = DescribeTable("buildGitProvider", func(input Config, expected expectedGitProvider) {
+	c, h, err := buildGitProvider(input)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(c.ProviderID()).To(Equal(gitprovider.ProviderID(expected.clientProviderID)), "ProviderID")
+	Expect(c.SupportedDomain()).To(Equal(expected.clientDomain), "SupportedDomain")
+	Expect(h).To(Equal(expected.hostname), "hostname")
+
+},
+	Entry("github.com", Config{Provider: "github", Hostname: "github.com", Token: "abc"}, expectedGitProvider{
+		clientDomain:     "github.com",
+		clientProviderID: "github",
+		hostname:         "github.com",
+	}),
+	Entry("gitlab.com", Config{Provider: "gitlab", Hostname: "gitlab.com", Token: "abc"}, expectedGitProvider{
+		// QUIRK..
+		clientDomain:     "https://gitlab.com",
+		clientProviderID: "gitlab",
+		hostname:         "gitlab.com",
+	}),
+	Entry("github.acme.com", Config{Provider: "github", Hostname: "github.acme.com", Token: "abc"}, expectedGitProvider{
+		clientDomain:     "https://github.acme.com",
+		clientProviderID: "github",
+		hostname:         "https://github.acme.com",
+	}),
+	Entry("gitlab.acme.com", Config{Provider: "gitlab", Hostname: "gitlab.acme.com", Token: "abc"}, expectedGitProvider{
+		clientDomain:     "https://gitlab.acme.com",
+		clientProviderID: "gitlab",
+		hostname:         "https://gitlab.acme.com",
+	}),
+)

pkg/gitproviders/repo_url.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/fluxcd/go-git-providers/github"
 	"github.com/fluxcd/go-git-providers/gitlab"
+	"github.com/spf13/viper"
 	"github.com/weaveworks/weave-gitops/pkg/utils"
 )
 
@@ -25,12 +26,15 @@ type RepoURL struct {
 }
 
 func NewRepoURL(uri string) (RepoURL, error) {
-	providerName, err := detectGitProviderFromUrl(uri)
+	providerName, err := detectGitProviderFromUrl(uri, ViperGetStringMapString("git-host-types"))
 	if err != nil {
 		return RepoURL{}, fmt.Errorf("could not get provider name from URL %s: %w", uri, err)
 	}
 
-	normalized := normalizeRepoURLString(uri, providerName)
+	normalized, err := normalizeRepoURLString(uri)
+	if err != nil {
+		return RepoURL{}, fmt.Errorf("could not normalize repo URL %s: %w", uri, err)
+	}
 
 	u, err := url.Parse(normalized)
 	if err != nil {
@@ -104,33 +108,40 @@ func getOwnerFromUrl(url url.URL, providerName GitProviderName) (string, error)
 
 // detectGitProviderFromUrl accepts a url related to a git repo and
 // returns the name of the provider associated.
-func detectGitProviderFromUrl(raw string) (GitProviderName, error) {
-	if strings.HasPrefix(raw, "git@") {
-		raw = "ssh://" + raw
-		raw = strings.Replace(raw, ".com:", ".com/", 1)
-	}
-
-	u, err := url.Parse(raw)
+func detectGitProviderFromUrl(raw string, gitHostTypes map[string]string) (GitProviderName, error) {
+	u, err := parseGitURL(raw)
 	if err != nil {
 		return "", fmt.Errorf("could not parse git repo url %q: %w", raw, err)
 	}
 
-	switch u.Hostname() {
-	case github.DefaultDomain:
-		return GitProviderGitHub, nil
-	case gitlab.DefaultDomain:
-		return GitProviderGitLab, nil
+	// defaults for github and gitlab
+	gitHostTypes[github.DefaultDomain] = string(GitProviderGitHub)
+	gitHostTypes[gitlab.DefaultDomain] = string(GitProviderGitLab)
+
+	provider := gitHostTypes[u.Host]
+	if provider == "" {
+		return "", fmt.Errorf("no git providers found for %q", raw)
+	}
+
+	return GitProviderName(provider), nil
+}
+
+// Hacks around "scp" formatted urls ($user@$host:$path)
+// the `:` delimiter between host and path throws off the std. url parser
+func parseGitURL(raw string) (*url.URL, error) {
+	if strings.HasPrefix(raw, "git@") {
+		// The first occurance of `:` should be the host:path delimiter.
+		raw = strings.Replace(raw, ":", "/", 1)
+		raw = "ssh://" + raw
 	}
 
-	return "", fmt.Errorf("no git providers found for %q", raw)
+	return url.Parse(raw)
 }
 
 // normalizeRepoURLString accepts a url like [email protected]:someuser/podinfo.git and converts it into
 // a string like ssh://[email protected]/someuser/podinfo.git. This helps standardize the different
 // user inputs that might be provided.
-func normalizeRepoURLString(url string, providerName GitProviderName) string {
-	trimmed := ""
-
+func normalizeRepoURLString(url string) (string, error) {
 	// https://github.com/weaveworks/weave-gitops/issues/878
 	// A trailing slash causes problems when naming secrets.
 	url = strings.TrimSuffix(url, "/")
@@ -139,18 +150,33 @@ func normalizeRepoURLString(url string, providerName GitProviderName) string {
 		url = url + ".git"
 	}
 
-	sshPrefix := fmt.Sprintf("git@%s.com:", providerName)
-	httpsPrefix := fmt.Sprintf("https://%s.com/", providerName)
+	u, err := parseGitURL(url)
+	if err != nil {
+		return "", fmt.Errorf("could not parse git repo url while normalizing %q: %w", url, err)
+	}
+
+	return fmt.Sprintf("ssh://git@%s%s", u.Host, u.Path), nil
+}
 
-	if strings.HasPrefix(url, sshPrefix) {
-		trimmed = strings.TrimPrefix(url, sshPrefix)
-	} else if strings.HasPrefix(url, httpsPrefix) {
-		trimmed = strings.TrimPrefix(url, httpsPrefix)
+// ViperGetStringMapString looks up a command line flag or env var in the format "foo=1,bar=2"
+// GetStringMapString tries to JSON decode the env var
+// If that fails (silently), try and decode the classic "foo=1,bar=2" form.
+// https://github.com/spf13/viper/issues/911
+func ViperGetStringMapString(key string) map[string]string {
+	sms := viper.GetStringMapString(key)
+	if len(sms) > 0 {
+		return sms
 	}
 
-	if trimmed != "" {
-		return fmt.Sprintf("ssh://git@%s.com/%s", providerName, trimmed)
+	ss := viper.GetStringSlice(key)
+	out := map[string]string{}
+
+	for _, pair := range ss {
+		kv := strings.SplitN(pair, "=", 2)
+		if len(kv) == 2 {
+			out[kv[0]] = kv[1]
+		}
 	}
 
-	return url
+	return out
 }

pkg/gitproviders/repo_url_test.go

@@ -6,10 +6,11 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
+	"github.com/spf13/viper"
 )
 
 var _ = DescribeTable("detectGitProviderFromUrl", func(input string, expected GitProviderName) {
-	result, err := detectGitProviderFromUrl(input)
+	result, err := detectGitProviderFromUrl(input, map[string]string{})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(result).To(Equal(expected))
 },
@@ -66,7 +67,10 @@ type expectedRepoURL struct {
 	protocol RepositoryURLProtocol
 }
 
-var _ = DescribeTable("NewRepoURL", func(input string, expected expectedRepoURL) {
+var _ = DescribeTable("NewRepoURL", func(input, gitProviderEnv string, expected expectedRepoURL) {
+	if gitProviderEnv != "" {
+		viper.Set("git-host-types", gitProviderEnv)
+	}
 	result, err := NewRepoURL(input)
 	Expect(err).NotTo(HaveOccurred())
 
@@ -78,46 +82,57 @@ var _ = DescribeTable("NewRepoURL", func(input string, expected expectedRepoURL)
 	Expect(result.Provider()).To(Equal(expected.provider))
 	Expect(result.Protocol()).To(Equal(expected.protocol))
 },
-	Entry("github git clone style", "[email protected]:someuser/podinfo.git", expectedRepoURL{
+	Entry("github git clone style", "[email protected]:someuser/podinfo.git", "", expectedRepoURL{
 		s:        "ssh://[email protected]/someuser/podinfo.git",
 		owner:    "someuser",
 		name:     "podinfo",
 		provider: GitProviderGitHub,
 		protocol: RepositoryURLProtocolSSH,
 	}),
-	Entry("github url style", "ssh://[email protected]/someuser/podinfo.git", expectedRepoURL{
+	Entry("github url style", "ssh://[email protected]/someuser/podinfo.git", "", expectedRepoURL{
 		s:        "ssh://[email protected]/someuser/podinfo.git",
 		owner:    "someuser",
 		name:     "podinfo",
 		provider: GitProviderGitHub,
 		protocol: RepositoryURLProtocolSSH,
 	}),
-	Entry("github https", "https://github.com/someuser/podinfo.git", expectedRepoURL{
+	Entry("github https", "https://github.com/someuser/podinfo.git", "", expectedRepoURL{
 		s:        "ssh://[email protected]/someuser/podinfo.git",
 		owner:    "someuser",
 		name:     "podinfo",
 		provider: GitProviderGitHub,
 		protocol: RepositoryURLProtocolSSH,
 	}),
-	Entry("gitlab git clone style", "[email protected]:someuser/podinfo.git", expectedRepoURL{
+	Entry("gitlab git clone style", "[email protected]:someuser/podinfo.git", "", expectedRepoURL{
 		s:        "ssh://[email protected]/someuser/podinfo.git",
 		owner:    "someuser",
 		name:     "podinfo",
 		provider: GitProviderGitLab,
 		protocol: RepositoryURLProtocolSSH,
 	}),
-	Entry("gitlab https", "https://gitlab.com/someuser/podinfo.git", expectedRepoURL{
+	Entry("gitlab https", "https://gitlab.com/someuser/podinfo.git", "", expectedRepoURL{
 		s:        "ssh://[email protected]/someuser/podinfo.git",
 		owner:    "someuser",
 		name:     "podinfo",
 		provider: GitProviderGitLab,
 		protocol: RepositoryURLProtocolSSH,
 	}),
-	Entry("trailing slash in url", "https://github.com/sympatheticmoose/podinfo-deploy/", expectedRepoURL{
+	Entry("trailing slash in url", "https://github.com/sympatheticmoose/podinfo-deploy/", "", expectedRepoURL{
 		s:        "ssh://[email protected]/sympatheticmoose/podinfo-deploy.git",
 		owner:    "sympatheticmoose",
 		name:     "podinfo-deploy",
 		provider: GitProviderGitHub,
 		protocol: RepositoryURLProtocolSSH,
 	}),
+	Entry(
+		"custom domain",
+		"[email protected]/sympatheticmoose/podinfo-deploy/",
+		"gitlab.acme.org=gitlab",
+		expectedRepoURL{
+			s:        "ssh://[email protected]/sympatheticmoose/podinfo-deploy.git",
+			owner:    "sympatheticmoose",
+			name:     "podinfo-deploy",
+			provider: "gitlab",
+			protocol: RepositoryURLProtocolSSH,
+		}),
 )

pkg/server/internal/provider.go

@@ -18,7 +18,11 @@ func NewGitProviderClient(token string) gitproviders.Client {
 
 // GetProvider returns a GitProvider passing the auth token into the implementation
 func (c *gitProviderClient) GetProvider(repoUrl gitproviders.RepoURL, getAccountType gitproviders.AccountTypeGetter) (gitproviders.GitProvider, error) {
-	provider, err := gitproviders.New(gitproviders.Config{Provider: repoUrl.Provider(), Token: c.token}, repoUrl.Owner(), getAccountType)
+	provider, err := gitproviders.New(gitproviders.Config{
+		Provider: repoUrl.Provider(),
+		Token:    c.token,
+		Hostname: repoUrl.URL().Host,
+	}, repoUrl.Owner(), getAccountType)
 	if err != nil {
 		return nil, fmt.Errorf("error creating git provider client: %w", err)
 	}