test more contained, gitlab partially tested

app/crawler/gitlab.sh

@@ -2,7 +2,7 @@
 
 gitlab() (
    is() {
-      echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wiki/'
+      echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wikis/'
    }
 
    get() {
@@ -33,7 +33,8 @@ gitlab() (
       local url="$1"
       local url_path="${url#http*://gitlab*.net/}"
       local project="${url_path%%/-/*}"
-      local root="${url_path#*/-/}"
+      local root="${url_path#*$project}"
+      local root="${root#*/-/}"
       local root="${root#tree/}"
       local root="${root#blob/}"
       local root="${root#*/}"

app/crawler/gitlab_test.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+test_get() {
+   gitlab eval "$(cat <<EOF
+      _gcurl() {
+         echo "\$*" | grep -q api.v4.projects.my%20project.repository.files.my%20file.raw
+      }
+      get "my project" "my file"
+EOF
+)"
+}
+
+test_url_to_project_root() {
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/blob/master/Alerts/rems/README.md | grep -q 'data-store/orchestration/runbooks'
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/blob/master/Alerts/rems/README.md | grep -q 'Alerts/rems/README.md'
+
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/tree/master/Alerts | grep -q 'data-store/orchestration/runbooks'
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/tree/master/Alerts | grep -q 'Alerts'
+
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks | grep -q 'data-store/orchestration/runbooks'
+   gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks | grep -q '^$'
+}
+
+test_is() {
+   gitlab is https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/blob/master/Alerts/rems/README.md
+   gitlab is https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/-/tree/master/Alerts
+   gitlab is https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks
+   ! gitlab is https://gitlab-app.eng.qops.net/surveys/marauders-map/wikis/Customer-impact-of-an-outage
+}

app/crawler/test.sh

@@ -15,15 +15,10 @@ main() {
 }
 
 one_main() (
-   echo testing $f >&2
    local f="$1"
    local ret=0
    for t in $(grep ^test_ "$f" | sed 's/(.*//'); do
-      echo testing $f:$t >&2
-      each
-      source "${f%_test.sh}.sh"
-      source "$f"
-      if ! eval "$t"; then
+      if ! one_test "$f" "$t"; then
          echo failed $f:$t >&2
          ret=$((ret+1))
       fi
@@ -31,9 +26,20 @@ one_main() (
    return $ret
 )
 
+one_test() (
+   local f="$1"
+   local t="$2"
+   each
+   source "${f%_test.sh}.sh"
+   source "$f"
+   eval "$t"
+)
+
 each() {
    export CACHE=$(mktemp -d)
    source ./cache.sh
+   set -e
+   set -o pipefail
 }
 
 if [ "$0" == "$BASH_SOURCE" ]; then