From 94fbde9cadfed1af452866037149534c1bf8c4eb Mon Sep 17 00:00:00 2001
From: Bel LaPointe <breel@qualtrics.com>
Date: Mon, 31 Jan 2022 15:48:42 -0700
Subject: [PATCH] add gitlab scrape to pull a file by project and path

---
 spike/crawl/env.env          |  6 +++---
 spike/crawl/gitlab/scrape.sh | 29 +++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 spike/crawl/gitlab/scrape.sh

diff --git a/spike/crawl/env.env b/spike/crawl/env.env
index 3f494d6..6ba8c74 100644
--- a/spike/crawl/env.env
+++ b/spike/crawl/env.env
@@ -1,3 +1,3 @@
-GITLAB_PAT=LkaMCFFfqdHd-r4Cubnw
-ODO_TOKEN=ac9a9e4d-9c6b-4049-9e8d-c8b97fe053aa
-GDOC_TOKEN=
+export GITLAB_PAT=LkaMCFFfqdHd-r4Cubnw
+export ODO_TOKEN=ac9a9e4d-9c6b-4049-9e8d-c8b97fe053aa
+export GDOC_TOKEN=
diff --git a/spike/crawl/gitlab/scrape.sh b/spike/crawl/gitlab/scrape.sh
new file mode 100644
index 0000000..3515ed5
--- /dev/null
+++ b/spike/crawl/gitlab/scrape.sh
@@ -0,0 +1,29 @@
+#! /bin/bash
+
+GITLAB_PAT="$GITLAB_PAT"
+PROJECT="$PROJECT"
+BLOB="$BLOB"
+
+urlencode() {
+    # urlencode <string>
+
+    old_lc_collate=$LC_COLLATE
+    LC_COLLATE=C
+
+    local length="${#1}"
+    for (( i = 0; i < length; i++ )); do
+        local c="${1:$i:1}"
+        case $c in
+            [a-zA-Z0-9.~_-]) printf '%s' "$c" ;;
+            *) printf '%%%02X' "'$c" ;;
+        esac
+    done
+
+    LC_COLLATE=$old_lc_collate
+}
+
+project="$(urlencode $PROJECT)"
+blob="$(urlencode $BLOB)"
+path="api/v4/projects/$project/repository/files/$blob/raw"
+
+curl -sS -H "Authorization: Bearer $GITLAB_PAT" "https://gitlab-app.eng.qops.net/$path"