no submod

work/notea/crawler/gitlab.sh

@@ -0,0 +1,156 @@
+#! /bin/bash
+
+gitlab() (
+	_is_gitlab() {
+      echo "$*" | grep -q gitlab.app
+	}
+
+	_is_wiki() {
+		echo "$*" | grep -q '/wikis'
+	}
+
+   is() {
+      _is_gitlab "$@" && ! _is_wiki "$@"
+   }
+
+	human_url() {
+		_url "$@" | sed 's/api.v4.projects.//' | sed 's/%2F/\//g' | sed 's/.raw$//' | sed 's/repository\/files/-\/tree\/master/'
+	}
+
+	_url() {
+      local base_url="$1"
+      local blob="$(echo "$2" | base64 --decode)"
+
+      local project="$(_url_to_project_root "$base_url" | head -n 1)"
+      project="$(urlencode "$project")"
+      local root="$(_url_to_project_root "$base_url" | tail -n 1)"
+		if [ -n "$root" ]; then
+      	blob="${root%/}/${blob#/}"
+      	blob="${blob#/}"
+      	blob="${blob%/}"
+		fi
+      blob="$(urlencode "$blob")"
+
+      local path="api/v4/projects/$project/repository/files/$blob/raw"
+      echo "https://gitlab-app.eng.qops.net/$path"
+	}
+
+   get() {
+		_gcurl "$(_url "$@")"
+   }
+
+   expand() {
+      local cache_key="gitlab expand $*"
+      if cache get "$cache_key"; then
+         return 0
+      fi
+      _expand "$@" | sort | cache put "$cache_key"
+   }
+
+   _expand() {
+      local url="$1"
+      local project="$(_url_to_project_root "$url" | head -n 1)"
+      local root="$(_url_to_project_root "$url" | tail -n 1)"
+      __expand "$project" "$root"
+   }
+
+   _url_to_project_root() {
+      local url="$1"
+      local url_path="${url#http*://gitlab*.net/}"
+		local project=""
+		if [[ "$url_path" == *"/-/"* ]]; then
+			project="${url_path%%/-/*}"
+		elif [[ "$url_path" == *"/tree/"* ]]; then
+			project="${url_path%%/tree/*}"
+		else
+			project="$url_path"
+		fi
+      local root="${url_path#*"$project"}"
+      root="${root#*/-/}"
+		root="${root#/}"
+      root="${root#blob/}"
+      root="${root#tree/}"
+		root="$(echo "$root" | sed 's/^[^\/]*//')"
+		root="${root#/}"
+      echo "$project"
+      echo "$root"
+   }
+
+   __expand() {
+      local project="$1"
+      local root="${2:-"/"}"
+
+      local b64_files=()
+      local b64_trees=("$(echo "$root" | base64)")
+      local i=0
+
+      find_each() {
+         local type="$1"
+         shift
+         echo "$*" \
+            | jq -c .[] \
+            | grep "\"type\":\"$type\"" \
+            | jq -r .path \
+            | while read -r line; do echo "$line" | base64; done \
+            | grep .
+      }
+      while [ "$i" -lt "${#b64_trees[@]}" ]; do
+         got="$(_list_tree "$project" "$(echo "${b64_trees[i]}" | base64 --decode)")"
+         for b64_tree in $(find_each "tree" "$got"); do
+            if ! echo "${b64_trees[@]}" | grep -q "[ ^]$b64_tree[ $]"; then
+               b64_trees+=("$b64_tree")
+            fi
+         done
+         for b64_file in $(find_each "blob" "$got"); do
+            if ! echo "${b64_files[@]}" | grep -q "[ ^]$b64_file[ $]"; then
+               b64_files+=("$b64_file")
+            fi
+         done
+         i=$((i+1))
+      done
+      for b64_file in "${b64_files[@]}"; do
+         local file="$(echo "$b64_file" | base64 --decode)"
+         file="${file#$root}"
+         file="${file#/}"
+         case "${file##*.}" in
+            md|txt )
+               echo "$file" | base64
+            ;;
+         esac
+      done
+   }
+
+   _list_tree() {
+      local project="$(urlencode "$1")"
+      local path="api/v4/projects/$project/repository/tree"
+      local query="recursive=true&path=$2"
+      _gcurl "https://gitlab-app.eng.qops.net/$path?$query"
+   }
+
+   _gcurl() {
+      local cache_key="gitlab _gcurl $*"
+      if cache get "$cache_key"; then
+         return 0
+      fi
+      __gcurl "$@" | cache put "$cache_key"
+   }
+
+   __gcurl() {
+      curl -sS -H "Authorization: Bearer $GITLAB_PAT" "$@"
+   }
+
+   "$@"
+)
+
+urlencode() (
+    LC_COLLATE=C
+    local length="${#1}"
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:$i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf '%s' "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+)
+