add easier

spike/crawl/gitlab-wiki/expand.sh

@@ -1,64 +0,0 @@
-#! /bin/bash
-
-urlencode() {
-    # urlencode <string>
-
-    old_lc_collate=$LC_COLLATE
-    LC_COLLATE=C
-
-    local length="${#1}"
-    for (( i = 0; i < length; i++ )); do
-        local c="${1:$i:1}"
-        case $c in
-            [a-zA-Z0-9.~_-]) printf '%s' "$c" ;;
-            *) printf '%%%02X' "'$c" ;;
-        esac
-    done
-
-    LC_COLLATE=$old_lc_collate
-}
-
-pat="${GITLAB_PAT:-"$(security find-generic-password -a "${USER}" -s GITLAB_PAT -w 2> /dev/null )"}"
-project="$(urlencode ${PROJECT:-"data-store/orchestration/runbooks"})"
-path="/api/v4/projects/$project/repository/tree"
-list() {
-   local query="recursive=true&path=$(test -n "$GITLAB_PATH" && echo "$GITLAB_PATH")"
-   curl -sS \
-      -H "Authorization: Bearer $pat" \
-      "https://gitlab-app.eng.qops.net/$path?$query"
-}
-list_wiki() {
-   curl -sS \
-      -H "Authorization: Bearer $pat" \
-      "https://gitlab-app.eng.qops.net/api/v4/projects/$project/wikis?with_content=0"
-}
-files=()
-trees=("${GITLAB_PATH:-"/"}")
-i=0
-while [ "$i" -lt "${#trees[@]}" ]; do
-   gitlab_path="${trees[i]}"
-   echo gitlab_path=$gitlab_path, i=$i, trees=${#trees[@]}, files=${#files[@]}... >&2
-   got="$(GITLAB_PATH=$gitlab_path list)"
-   for nested_path in $(echo "$got" | jq -c .[] | grep '"type":"tree"' | jq -r .path | while read -r line; do echo "$line" | base64; done | grep .); do
-      nested_path="$(echo "$nested_path" | base64 --decode)"
-      if echo "${trees[@]}" | grep -q "[ ^]$nested_path[ $]"; then
-         continue
-      fi
-      trees+=("$nested_path")
-   done
-   for nested_path in $(echo "$got" | jq -c .[] | grep '"type":"blob"' | jq -r .path | while read -r line; do echo "$line" | base64; done | grep .); do
-      nested_path="$(echo "$nested_path" | base64 --decode)"
-      if echo "${files[@]}" | grep -q "[ ^]$nested_path[ $]"; then
-         continue
-      fi
-      files+=("$nested_path")
-   done
-   i=$((i+1))
-   if ((i>5)); then
-      break
-   fi
-done
-for file in "${files[@]}"; do
-   echo "gitlab://$file"
-done | sort
-list_wiki | jq -c .[] | jq -r .title | sed 's/^/wiki:\/\//'

spike/crawl/gitlab-wiki/expand.sh

@@ -0,0 +1 @@
+../gitlab/expand.sh

spike/crawl/gitlab/expand.sh

@@ -1,5 +1,9 @@
 #! /bin/bash
 
+GITLAB_PAT="$GITLAB_PAT"
+PROJECT="$PROJECT"
+GITLAB_PATH="$GITLAB_PATH"
+
 urlencode() {
     # urlencode <string>