From 8aa8b979f1c7aeb1469f7409163f30dabcd39fb8 Mon Sep 17 00:00:00 2001
From: Bel LaPointe <breel@qualtrics.com>
Date: Mon, 31 Jan 2022 15:50:23 -0700
Subject: [PATCH] add gitlab wiki scrape

---
 spike/crawl/gitlab-wiki/scrape.sh | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 spike/crawl/gitlab-wiki/scrape.sh

diff --git a/spike/crawl/gitlab-wiki/scrape.sh b/spike/crawl/gitlab-wiki/scrape.sh
new file mode 100644
index 0000000..1f52c39
--- /dev/null
+++ b/spike/crawl/gitlab-wiki/scrape.sh
@@ -0,0 +1,29 @@
+#! /bin/bash
+
+GITLAB_PAT="$GITLAB_PAT"
+PROJECT="$PROJECT"
+BLOB="$BLOB"
+
+urlencode() {
+    # urlencode <string>
+
+    old_lc_collate=$LC_COLLATE
+    LC_COLLATE=C
+
+    local length="${#1}"
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:$i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf '%s' "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+
+    LC_COLLATE=$old_lc_collate
+}
+
+project="$(urlencode $PROJECT)"
+blob="$(urlencode $BLOB)"
+path="api/v4/projects/$project/wikis/$blob"
+
+curl -sS -H "Authorization: Bearer $GITLAB_PAT" "https://gitlab-app.eng.qops.net/$path" | jq -r .content