add gitlab scrape to pull a file by project and path

spike/crawl/env.env

@@ -1,3 +1,3 @@
-GITLAB_PAT=LkaMCFFfqdHd-r4Cubnw
-ODO_TOKEN=ac9a9e4d-9c6b-4049-9e8d-c8b97fe053aa
-GDOC_TOKEN=
+export GITLAB_PAT=LkaMCFFfqdHd-r4Cubnw
+export ODO_TOKEN=ac9a9e4d-9c6b-4049-9e8d-c8b97fe053aa
+export GDOC_TOKEN=

spike/crawl/gitlab/scrape.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+
+GITLAB_PAT="$GITLAB_PAT"
+PROJECT="$PROJECT"
+BLOB="$BLOB"
+
+urlencode() {
+    # urlencode <string>
+
+    old_lc_collate=$LC_COLLATE
+    LC_COLLATE=C
+
+    local length="${#1}"
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:$i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf '%s' "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+
+    LC_COLLATE=$old_lc_collate
+}
+
+project="$(urlencode $PROJECT)"
+blob="$(urlencode $BLOB)"
+path="api/v4/projects/$project/repository/files/$blob/raw"
+
+curl -sS -H "Authorization: Bearer $GITLAB_PAT" "https://gitlab-app.eng.qops.net/$path"