From 529abd37e9ca8f9ef094ae748ce7c9bfad769b89 Mon Sep 17 00:00:00 2001
From: Bel LaPointe <breel@qualtrics.com>
Date: Thu, 10 Feb 2022 10:49:35 -0700
Subject: [PATCH] neverends

---
 app/crawler/cache.sh                          |  2 +-
 app/crawler/gitlab.sh                         |  5 ++++-
 app/crawler/main.sh                           | 10 ++++++----
 spike/review/reinvent/ezmded/server/server.go |  2 +-
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/app/crawler/cache.sh b/app/crawler/cache.sh
index 52e6dc9..7c89f62 100644
--- a/app/crawler/cache.sh
+++ b/app/crawler/cache.sh
@@ -2,7 +2,7 @@
 
 cache() (
    path() {
-      echo "$CACHE/$(echo "$*" | base64)"
+      echo "$CACHE/$(echo "$*" | base64 | md5sum | awk '{print $1}')"
    }
    get() {
       local path="$(path "$*")"
diff --git a/app/crawler/gitlab.sh b/app/crawler/gitlab.sh
index bcb6ef6..f058a72 100644
--- a/app/crawler/gitlab.sh
+++ b/app/crawler/gitlab.sh
@@ -12,7 +12,10 @@ gitlab() (
       local project="$(_url_to_project_root "$url" | head -n 1)"
       project="$(urlencode "$project")"
       local root="$(_url_to_project_root "$url" | tail -n 1)"
-      blob="$(urlencode "$root/$blob")"
+      blob="$(urlencode "$blob")"
+		if [ -n "$root" ]; then
+      	blob="$(urlencode "$root/$blob")"
+		fi
 
       local path="api/v4/projects/$project/repository/files/$blob/raw"
       _gcurl "https://gitlab-app.eng.qops.net/$path"
diff --git a/app/crawler/main.sh b/app/crawler/main.sh
index ad6991b..4786338 100644
--- a/app/crawler/main.sh
+++ b/app/crawler/main.sh
@@ -65,7 +65,7 @@ _crawl() {
 }
 
 extract_crawlable_source() {
-	echo "$*" | head -n 1 | awk '{print $NF}' | sed 's/^<//' | sed 's/>$//'
+	echo "$*" | head -n 1 | awk '{print $NF}' | sed 's/^<//' | sed 's/>$//' | sed 's/^\///' | sed 's/\/$//'
 }
 
 crawl_with() {
@@ -86,7 +86,7 @@ crawl_with() {
 	log expand $crawlable_source:
 	notes_mkdir_p() {
 		local id="$1"
-		local subtitle="$2"
+		local subtitle="${2%/}"
 		notes put "$id" "$subtitle" "autogenerated content"
 	}
 	one() {
@@ -107,7 +107,9 @@ crawl_with() {
 			for subtitle in $(echo "$i" | base64 --decode | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
 				local subtitle="$(echo "$subtitle" | base64 --decode)"
 				if [ -n "$title_so_far" ]; then
-					notes_mkdir_p "${sum%/}" "${title_so_far%/}" >&2
+					local mkdir_p_title="${title_so_far%/}"
+					mkdir_p_title="${mkdir_p_title##*/}"
+					notes_mkdir_p "${sum%/}" "${mkdir_p_title}" >&2
 				fi
 				sum+="$(echo "$subtitle" | encode)/"
 				title_so_far+="$subtitle/"
@@ -118,7 +120,7 @@ crawl_with() {
 		log "	$ID ($TITLE): ${#CONTENT}"
 		push_crawled "$PID/$ID" "$TITLE" "$CONTENT"
 	}
-	for i in $(seq 1 $(("${#expanded[@]}"-1))); do
+	for i in $(seq 0 $(("${#expanded[@]}"-1))); do
 		one "${expanded[i]}"
 	done
 }
diff --git a/spike/review/reinvent/ezmded/server/server.go b/spike/review/reinvent/ezmded/server/server.go
index 45e4e14..1feb702 100644
--- a/spike/review/reinvent/ezmded/server/server.go
+++ b/spike/review/reinvent/ezmded/server/server.go
@@ -296,7 +296,7 @@ func (server *Server) urlFileId(id []string) string {
 
 func (server *Server) fileId(r *http.Request) []string {
 	return strings.Split(
-		strings.TrimPrefix(
+		strings.Trim(
 			strings.TrimPrefix(
 				strings.Trim(r.URL.Path, "/"),
 				"api/v0/files",