neverends

app/crawler/cache.sh

@@ -2,7 +2,7 @@
 
 cache() (
    path() {
-      echo "$CACHE/$(echo "$*" | base64)"
+      echo "$CACHE/$(echo "$*" | base64 | md5sum | awk '{print $1}')"
    }
    get() {
       local path="$(path "$*")"

app/crawler/gitlab.sh

@@ -12,7 +12,10 @@ gitlab() (
       local project="$(_url_to_project_root "$url" | head -n 1)"
       project="$(urlencode "$project")"
       local root="$(_url_to_project_root "$url" | tail -n 1)"
-      blob="$(urlencode "$root/$blob")"
+      blob="$(urlencode "$blob")"
+		if [ -n "$root" ]; then
+      	blob="$(urlencode "$root/$blob")"
+		fi
 
       local path="api/v4/projects/$project/repository/files/$blob/raw"
       _gcurl "https://gitlab-app.eng.qops.net/$path"

app/crawler/main.sh

@@ -65,7 +65,7 @@ _crawl() {
 }
 
 extract_crawlable_source() {
-	echo "$*" | head -n 1 | awk '{print $NF}' | sed 's/^<//' | sed 's/>$//'
+	echo "$*" | head -n 1 | awk '{print $NF}' | sed 's/^<//' | sed 's/>$//' | sed 's/^\///' | sed 's/\/$//'
 }
 
 crawl_with() {
@@ -86,7 +86,7 @@ crawl_with() {
 	log expand $crawlable_source:
 	notes_mkdir_p() {
 		local id="$1"
-		local subtitle="$2"
+		local subtitle="${2%/}"
 		notes put "$id" "$subtitle" "autogenerated content"
 	}
 	one() {
@@ -107,7 +107,9 @@ crawl_with() {
 			for subtitle in $(echo "$i" | base64 --decode | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
 				local subtitle="$(echo "$subtitle" | base64 --decode)"
 				if [ -n "$title_so_far" ]; then
-					notes_mkdir_p "${sum%/}" "${title_so_far%/}" >&2
+					local mkdir_p_title="${title_so_far%/}"
+					mkdir_p_title="${mkdir_p_title##*/}"
+					notes_mkdir_p "${sum%/}" "${mkdir_p_title}" >&2
 				fi
 				sum+="$(echo "$subtitle" | encode)/"
 				title_so_far+="$subtitle/"
@@ -118,7 +120,7 @@ crawl_with() {
 		log "	$ID ($TITLE): ${#CONTENT}"
 		push_crawled "$PID/$ID" "$TITLE" "$CONTENT"
 	}
-	for i in $(seq 1 $(("${#expanded[@]}"-1))); do
+	for i in $(seq 0 $(("${#expanded[@]}"-1))); do
 		one "${expanded[i]}"
 	done
 }

spike/review/reinvent/ezmded/server/server.go

@@ -296,7 +296,7 @@ func (server *Server) urlFileId(id []string) string {
 
 func (server *Server) fileId(r *http.Request) []string {
 	return strings.Split(
-		strings.TrimPrefix(
+		strings.Trim(
 			strings.TrimPrefix(
 				strings.Trim(r.URL.Path, "/"),
 				"api/v0/files",