diff --git a/spike/crawl/gitlab-wiki/scrape.sh b/spike/crawl/gitlab-wiki/scrape.sh new file mode 100644 index 0000000..1f52c39 --- /dev/null +++ b/spike/crawl/gitlab-wiki/scrape.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +GITLAB_PAT="$GITLAB_PAT" +PROJECT="$PROJECT" +BLOB="$BLOB" + +urlencode() { + # urlencode + + old_lc_collate=$LC_COLLATE + LC_COLLATE=C + + local length="${#1}" + for (( i = 0; i < length; i++ )); do + local c="${1:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) printf '%s' "$c" ;; + *) printf '%%%02X' "'$c" ;; + esac + done + + LC_COLLATE=$old_lc_collate +} + +project="$(urlencode $PROJECT)" +blob="$(urlencode $BLOB)" +path="api/v4/projects/$project/wikis/$blob" + +curl -sS -H "Authorization: Bearer $GITLAB_PAT" "https://gitlab-app.eng.qops.net/$path" | jq -r .content