From 259c8910b180de52f6944a3688db9726ae6e8a5c Mon Sep 17 00:00:00 2001 From: monalisa Date: Thu, 14 Sep 2023 18:31:57 +0200 Subject: [PATCH 1/4] Patch support for git repos hosted on HTTP --- libs/git/clone.go | 22 +++++++++++++++++++--- libs/git/clone_test.go | 15 +++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/libs/git/clone.go b/libs/git/clone.go index 8b075cdeac3..d1a74319795 100644 --- a/libs/git/clone.go +++ b/libs/git/clone.go @@ -27,11 +27,14 @@ type cloneOptions struct { TargetPath string } -func (opts cloneOptions) args() []string { - args := []string{"clone", opts.RepositoryUrl, opts.TargetPath, "--depth=1", "--no-tags"} +func (opts cloneOptions) args(shallow bool) []string { + args := []string{"clone", opts.RepositoryUrl, opts.TargetPath, "--no-tags"} if opts.Reference != "" { args = append(args, "--branch", opts.Reference) } + if shallow { + args = append(args, "--depth=1") + } return args } @@ -50,7 +53,7 @@ func Clone(ctx context.Context, url, reference, targetPath string) error { TargetPath: targetPath, } - cmd := exec.CommandContext(ctx, "git", opts.args()...) + cmd := exec.CommandContext(ctx, "git", opts.args(true)...) var cmdErr bytes.Buffer cmd.Stderr = &cmdErr @@ -65,6 +68,19 @@ func Clone(ctx context.Context, url, reference, targetPath string) error { // wait for git clone to complete err = cmd.Wait() + // Git repos hosted via HTTP do not support shallow cloning. We try with + // a deep clone this time + if err != nil && strings.Contains(cmdErr.String(), "dumb http transport does not support shallow capabilities") { + retryCmd := exec.CommandContext(ctx, "git", opts.args(false)...) + var retryCmdErr bytes.Buffer + cmd.Stderr = &retryCmdErr + retryErr := retryCmd.Run() + if retryErr != nil { + return fmt.Errorf("git clone failed: %w. %s", retryErr, retryCmdErr.String()) + } else { + return nil + } + } if err != nil { return fmt.Errorf("git clone failed: %w. %s", err, cmdErr.String()) } diff --git a/libs/git/clone_test.go b/libs/git/clone_test.go index 8101178fb67..e4e6bc8f8c4 100644 --- a/libs/git/clone_test.go +++ b/libs/git/clone_test.go @@ -10,18 +10,25 @@ import ( func TestGitCloneArgs(t *testing.T) { // case: No branch / tag specified. In this case git clones the default branch - assert.Equal(t, []string{"clone", "abc", "/def", "--depth=1", "--no-tags"}, cloneOptions{ + assert.Equal(t, []string{"clone", "abc", "/def", "--no-tags", "--depth=1"}, cloneOptions{ Reference: "", RepositoryUrl: "abc", TargetPath: "/def", - }.args()) + }.args(true)) // case: A branch is specified. - assert.Equal(t, []string{"clone", "abc", "/def", "--depth=1", "--no-tags", "--branch", "my-branch"}, cloneOptions{ + assert.Equal(t, []string{"clone", "abc", "/def", "--no-tags", "--branch", "my-branch", "--depth=1"}, cloneOptions{ Reference: "my-branch", RepositoryUrl: "abc", TargetPath: "/def", - }.args()) + }.args(true)) + + // case: deep cloning + assert.Equal(t, []string{"clone", "abc", "/def", "--no-tags"}, cloneOptions{ + Reference: "", + RepositoryUrl: "abc", + TargetPath: "/def", + }.args(false)) } func TestGitCloneWithGitNotFound(t *testing.T) { From 56666b4d776cc16c050f45fbecbb072a7cf0bc61 Mon Sep 17 00:00:00 2001 From: monalisa Date: Thu, 14 Sep 2023 18:41:11 +0200 Subject: [PATCH 2/4] - --- libs/git/clone.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/git/clone.go b/libs/git/clone.go index d1a74319795..a2eb00ff48d 100644 --- a/libs/git/clone.go +++ b/libs/git/clone.go @@ -73,7 +73,7 @@ func Clone(ctx context.Context, url, reference, targetPath string) error { if err != nil && strings.Contains(cmdErr.String(), "dumb http transport does not support shallow capabilities") { retryCmd := exec.CommandContext(ctx, "git", opts.args(false)...) var retryCmdErr bytes.Buffer - cmd.Stderr = &retryCmdErr + retryCmd.Stderr = &retryCmdErr retryErr := retryCmd.Run() if retryErr != nil { return fmt.Errorf("git clone failed: %w. %s", retryErr, retryCmdErr.String()) From 313aef1fb08cf1b2a1e26831db73f59196f3fe9a Mon Sep 17 00:00:00 2001 From: monalisa Date: Thu, 14 Sep 2023 23:54:31 +0200 Subject: [PATCH 3/4] refactor --- libs/git/clone.go | 64 ++++++++++++++++++++++-------------------- libs/git/clone_test.go | 9 ++++-- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/libs/git/clone.go b/libs/git/clone.go index a2eb00ff48d..1c7123c6be2 100644 --- a/libs/git/clone.go +++ b/libs/git/clone.go @@ -25,35 +25,24 @@ type cloneOptions struct { // Local path to clone repository at TargetPath string + + // If true, the repository is shallow cloned + Shallow bool } -func (opts cloneOptions) args(shallow bool) []string { +func (opts cloneOptions) args() []string { args := []string{"clone", opts.RepositoryUrl, opts.TargetPath, "--no-tags"} if opts.Reference != "" { args = append(args, "--branch", opts.Reference) } - if shallow { + if opts.Shallow { args = append(args, "--depth=1") } return args } -func Clone(ctx context.Context, url, reference, targetPath string) error { - // We assume only the repository name has been if input does not contain any - // `/` characters and the url is only made up of alphanumeric characters and - // ".", "_" and "-". This repository is resolved again databricks github account. - fullUrl := url - if githubRepoRegex.MatchString(url) { - fullUrl = strings.Join([]string{githubUrl, databricksOrg, url}, "/") - } - - opts := cloneOptions{ - Reference: reference, - RepositoryUrl: fullUrl, - TargetPath: targetPath, - } - - cmd := exec.CommandContext(ctx, "git", opts.args(true)...) +func (opts cloneOptions) clone(ctx context.Context) error { + cmd := exec.CommandContext(ctx, "git", opts.args()...) var cmdErr bytes.Buffer cmd.Stderr = &cmdErr @@ -68,21 +57,34 @@ func Clone(ctx context.Context, url, reference, targetPath string) error { // wait for git clone to complete err = cmd.Wait() - // Git repos hosted via HTTP do not support shallow cloning. We try with - // a deep clone this time - if err != nil && strings.Contains(cmdErr.String(), "dumb http transport does not support shallow capabilities") { - retryCmd := exec.CommandContext(ctx, "git", opts.args(false)...) - var retryCmdErr bytes.Buffer - retryCmd.Stderr = &retryCmdErr - retryErr := retryCmd.Run() - if retryErr != nil { - return fmt.Errorf("git clone failed: %w. %s", retryErr, retryCmdErr.String()) - } else { - return nil - } - } if err != nil { return fmt.Errorf("git clone failed: %w. %s", err, cmdErr.String()) } return nil } + +func Clone(ctx context.Context, url, reference, targetPath string) error { + // We assume only the repository name has been if input does not contain any + // `/` characters and the url is only made up of alphanumeric characters and + // ".", "_" and "-". This repository is resolved again databricks github account. + fullUrl := url + if githubRepoRegex.MatchString(url) { + fullUrl = strings.Join([]string{githubUrl, databricksOrg, url}, "/") + } + + opts := cloneOptions{ + Reference: reference, + RepositoryUrl: fullUrl, + TargetPath: targetPath, + Shallow: true, + } + + err := opts.clone(ctx) + // Git repos hosted via HTTP do not support shallow cloning. We try with + // a deep clone this time + if err != nil && strings.Contains(err.Error(), "dumb http transport does not support shallow capabilities") { + opts.Shallow = false + err = opts.clone(ctx) + } + return err +} diff --git a/libs/git/clone_test.go b/libs/git/clone_test.go index e4e6bc8f8c4..bed5fa54edd 100644 --- a/libs/git/clone_test.go +++ b/libs/git/clone_test.go @@ -14,21 +14,24 @@ func TestGitCloneArgs(t *testing.T) { Reference: "", RepositoryUrl: "abc", TargetPath: "/def", - }.args(true)) + Shallow: true, + }.args()) // case: A branch is specified. assert.Equal(t, []string{"clone", "abc", "/def", "--no-tags", "--branch", "my-branch", "--depth=1"}, cloneOptions{ Reference: "my-branch", RepositoryUrl: "abc", TargetPath: "/def", - }.args(true)) + Shallow: true, + }.args()) // case: deep cloning assert.Equal(t, []string{"clone", "abc", "/def", "--no-tags"}, cloneOptions{ Reference: "", RepositoryUrl: "abc", TargetPath: "/def", - }.args(false)) + Shallow: false, + }.args()) } func TestGitCloneWithGitNotFound(t *testing.T) { From e324592902ffa697d22eb4e8c3ce413660dbce08 Mon Sep 17 00:00:00 2001 From: monalisa Date: Fri, 15 Sep 2023 11:05:27 +0200 Subject: [PATCH 4/4] - --- libs/git/clone.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/git/clone.go b/libs/git/clone.go index 1c7123c6be2..af7ffa4bbf2 100644 --- a/libs/git/clone.go +++ b/libs/git/clone.go @@ -84,7 +84,7 @@ func Clone(ctx context.Context, url, reference, targetPath string) error { // a deep clone this time if err != nil && strings.Contains(err.Error(), "dumb http transport does not support shallow capabilities") { opts.Shallow = false - err = opts.clone(ctx) + return opts.clone(ctx) } return err }