Skip to content

Commit

Permalink
update tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@gabrie30
gabrie30 committed May 24, 2024
1 parent 1cd9ea9 commit a6bedfb
Show file tree
Hide file tree
Showing 6 changed files with 285 additions and 58 deletions.
53 changes: 40 additions & 13 deletions cmd/clone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,10 @@ func hasRepoNameCollisions(repos []scm.Repo) (map[string]bool, bool) {
continue
}

if repo.IsWiki {
continue
}

if _, ok := repoNameWithCollisions[repo.Name]; ok {
repoNameWithCollisions[repo.Name] = true
hasCollisions = true
Expand Down Expand Up @@ -686,7 +690,10 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}

totalResourcesToClone, reposToCloneCount, snippetToCloneCount, wikisToCloneCount := getCloneableInventory(cloneTargets)
if os.Getenv("GHORG_CLONE_WIKI") == "true" {
if os.Getenv("GHORG_CLONE_WIKI") == "true" && os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
m := fmt.Sprintf("%v resources to clone found in %v, %v repos, %v snippets, and %v wikis\n", totalResourcesToClone, targetCloneSource, snippetToCloneCount, reposToCloneCount, wikisToCloneCount)
colorlog.PrintInfo(m)
} else if os.Getenv("GHORG_CLONE_WIKI") == "true" {
m := fmt.Sprintf("%v resources to clone found in %v, %v repos and %v wikis\n", totalResourcesToClone, targetCloneSource, reposToCloneCount, wikisToCloneCount)
colorlog.PrintInfo(m)
} else if os.Getenv("GHORG_CLONE_SNIPPETS") == "true" {
Expand Down Expand Up @@ -721,7 +728,9 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
for i := range cloneTargets {
repo := cloneTargets[i]

// we use this because we dont want spaces in the final directory, using the web address makes it more file friendly
// We use this because we dont want spaces in the final directory, using the web address makes it more file friendly
// In the case of root level snippets we use the title which will have spaces in it, the url uses an ID so its not possible to use name from url
// With snippets that originate on repos, we use that repo name
repoSlug := getAppNameFromURL(repo.URL)

if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
Expand All @@ -736,13 +745,28 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}

mutex.Lock()
inHash := repoNameWithCollisions[repo.Name]
var inHash bool
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
inHash = repoNameWithCollisions[repo.GitLabSnippetInfo.NameOfRepo]
} else {
inHash = repoNameWithCollisions[repo.Name]
}

mutex.Unlock()
// Only GitLab repos can have collisions due to groups and subgroups
// If there are collisions and this is a repo with a naming collision change name to avoid collisions
if hasCollisions && inHash {
repoSlug = trimCollisionFilename(strings.Replace(repo.Path, "/", "_", -1))

repoSlug = trimCollisionFilename(strings.Replace(repo.Path, string(os.PathSeparator), "_", -1))
if repo.IsWiki {
if !strings.HasSuffix(repoSlug, ".wiki") {
repoSlug = repoSlug + ".wiki"
}
}
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
if !strings.HasSuffix(repoSlug, ".snippets") {
repoSlug = repoSlug + ".snippets"
}
}
mutex.Lock()
slugCollision := repoNameWithCollisions[repoSlug]
mutex.Unlock()
Expand All @@ -756,18 +780,21 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
}
}

if repo.IsWiki {
if !strings.HasSuffix(repoSlug, ".wiki") {
repoSlug = repoSlug + ".wiki"
}
}
if repo.IsGitLabSnippet && !repo.IsGitLabRootLevelSnippet {
if !strings.HasSuffix(repoSlug, ".snippets") {
repoSlug = repoSlug + ".snippets"
}
}

repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug)

if repo.IsGitLabRootLevelSnippet {
repo.HostPath = filepath.Join(outputDirAbsolutePath, "_ghorg_root_level_snippets", repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID)
} else if repo.IsGitLabSnippet {
repo.HostPath = filepath.Join(outputDirAbsolutePath, repoSlug+".snippets", repo.GitLabSnippetInfo.Title+"-"+repo.GitLabSnippetInfo.ID)
}

if repo.IsWiki {
if !strings.HasSuffix(repo.HostPath, ".wiki") {
repo.HostPath = repo.HostPath + ".wiki"
}
}

action := "cloning"
Expand Down
3 changes: 3 additions & 0 deletions scm/gitlab.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ func (c Gitlab) GetSnippets(cloneData []Repo) ([]Repo, error) {
s.GitLabSnippetInfo.ID = snippetID
s.GitLabSnippetInfo.Title = snippetTitle
s.GitLabSnippetInfo.URLOfRepo = cloneTarget.URL
s.GitLabSnippetInfo.NameOfRepo = cloneTarget.Name
cloneData = append(cloneData, s)
}
}
Expand Down Expand Up @@ -505,6 +506,8 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {

if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" {
wiki := Repo{}
// wiki needs name for gitlab name collisions
wiki.Name = p.Name
wiki.IsWiki = true
wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
Expand Down
9 changes: 7 additions & 2 deletions scm/structs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ type Repo struct {
}

type GitLabSnippet struct {
ID string
Title string
// GitLab ID of the snippet
ID string
// Title of the snippet
Title string
// URL of the repo that snippet was made on
URLOfRepo string
// Name of the repo that the snippet was made on
NameOfRepo string
}
2 changes: 1 addition & 1 deletion scripts/local-gitlab/get_credentials.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ done

set -x

docker exec -it gitlab gitlab-rails runner "token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api, :read_api, :sudo], name: 'CI Test Token'); token.set_token('password'); token.save!"
docker exec -it gitlab gitlab-rails runner "token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api, :read_api, :sudo], name: 'CI Test Token', expires_at: 365.days.from_now); token.set_token('password'); token.save!"

API_TOKEN="password"

Expand Down
110 changes: 92 additions & 18 deletions scripts/local-gitlab/integration-tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,21 @@

set -ex

TOKEN=${1:-'password'}
GITLAB_URL=${2:-'http://gitlab.example.com'}
LOCAL_GITLAB_GHORG_DIR=${3:-"${HOME}/ghorg"}
LOCAL_GITLAB_GHORG_DIR=${1:-"${HOME}/ghorg"}
TOKEN=${2:-'password'}
GITLAB_URL=${3:-'http://gitlab.example.com'}


# Delete all folders that start with local-gitlab-v15- in the LOCAL_GITLAB_GHORG_DIR
for dir in "${LOCAL_GITLAB_GHORG_DIR}"/local-gitlab-*; do
if [ -d "$dir" ]; then
rm -rf "$dir"
fi
done


export GHORG_INSECURE_GITLAB_CLIENT=true
# export GHORG_DEBUG=true

# NOTE run all clones twice to test once for clone then pull

Expand Down Expand Up @@ -36,7 +46,7 @@ EOF

if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED"
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED local-gitlab-group1"
exit 1
fi

Expand All @@ -51,7 +61,7 @@ EOF

if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED"
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED local-gitlab-group2"
exit 1
fi

Expand All @@ -67,7 +77,7 @@ EOF

if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED"
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR TEST FAILED local-gitlab-group3/subgroup-a"
exit 1
fi

Expand All @@ -77,7 +87,6 @@ ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}

GOT=$( ghorg ls local-gitlab-v15-repos-flat | grep -o 'local-gitlab-v15-repos-flat.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-flat/Monitoring
local-gitlab-v15-repos-flat/local-gitlab-group1_baz0
local-gitlab-v15-repos-flat/local-gitlab-group1_baz1
local-gitlab-v15-repos-flat/local-gitlab-group1_baz2
Expand All @@ -103,6 +112,71 @@ echo "CLONE AND TEST ALL-GROUPS, OUTPUT DIR"
exit 1
fi

########### CLONE AND TEST ALL-GROUPS, OUTPUT DIR, WIKI ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --clone-wiki --output-dir=local-gitlab-v15-repos-flat-wiki
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --clone-wiki --output-dir=local-gitlab-v15-repos-flat-wiki

GOT=$( ghorg ls local-gitlab-v15-repos-flat-wiki | grep -o 'local-gitlab-v15-repos-flat-wiki.*')
WANT=$(cat <<EOF
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz0
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz0.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz1
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz1.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz2
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz2.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz3
local-gitlab-v15-repos-flat-wiki/local-gitlab-group1_baz3.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz0
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz0.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz1
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz1.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz2
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz2.wiki
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz3
local-gitlab-v15-repos-flat-wiki/local-gitlab-group2_baz3.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_0
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_0.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_1
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_1.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_2
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_2.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_3
local-gitlab-v15-repos-flat-wiki/subgroup_a_repo_3.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_0
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_0.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_1
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_1.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_2
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_2.wiki
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_3
local-gitlab-v15-repos-flat-wiki/subgroup_b_repo_3.wiki
EOF
)

if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, OUTPUT DIR"
exit 1
fi

############ CLONE AND TEST ALL-GROUPS, OUTPUT DIR, SNIPPETS, ROOT LEVEL ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --clone-snippets --output-dir=local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --clone-snippets --output-dir=local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups

# Test root level snippets
GOT=$( ghorg ls local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups/_ghorg_root_level_snippets | grep -o 'local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups.*')
WANT=$(cat <<EOF
local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups/_ghorg_root_level_snippets/snippet1-1
local-gitlab-v15-snippets-preserve-dir-output-dir-all-groups/_ghorg_root_level_snippets/snippet2-2
EOF
)

if [ "${WANT}" != "${GOT}" ]
then
echo "CLONE AND TEST ALL-GROUPS, OUTPUT DIR, SNIPPETS, ROOT LEVEL FAILED"
exit 1
fi

############ CLONE ALL-GROUPS, BACKUP, CLONE WIKI, OUTPUT DIR ############
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --backup --clone-wiki --output-dir=local-gitlab-v15-backup
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --backup --clone-wiki --output-dir=local-gitlab-v15-backup
Expand Down Expand Up @@ -138,19 +212,19 @@ ghorg clone local-gitlab-group1 --scm=gitlab --base-url="${GITLAB_URL}" --token=
ghorg clone local-gitlab-group1 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-group1

############ CLONE AND TEST TOP LEVEL GROUP ############
ghorg clone local-gitlab-group3 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}"
ghorg clone local-gitlab-group3 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}"
ghorg clone local-gitlab-group3 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-top-level-group
ghorg clone local-gitlab-group3 --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-top-level-group

GOT=$(ghorg ls local-gitlab-group3 | grep -o 'local-gitlab-group3.*')
GOT=$(ghorg ls local-gitlab-v15-top-level-group | grep -o 'local-gitlab-v15-top-level-group.*')
WANT=$(cat <<EOF
local-gitlab-group3/subgroup_a_repo_0
local-gitlab-group3/subgroup_a_repo_1
local-gitlab-group3/subgroup_a_repo_2
local-gitlab-group3/subgroup_a_repo_3
local-gitlab-group3/subgroup_b_repo_0
local-gitlab-group3/subgroup_b_repo_1
local-gitlab-group3/subgroup_b_repo_2
local-gitlab-group3/subgroup_b_repo_3
local-gitlab-v15-top-level-group/subgroup_a_repo_0
local-gitlab-v15-top-level-group/subgroup_a_repo_1
local-gitlab-v15-top-level-group/subgroup_a_repo_2
local-gitlab-v15-top-level-group/subgroup_a_repo_3
local-gitlab-v15-top-level-group/subgroup_b_repo_0
local-gitlab-v15-top-level-group/subgroup_b_repo_1
local-gitlab-v15-top-level-group/subgroup_b_repo_2
local-gitlab-v15-top-level-group/subgroup_b_repo_3
EOF
)

Expand Down
Loading

0 comments on commit a6bedfb

Please sign in to comment.