diff --git a/app/crawler/cache_test.sh b/app/crawler/cache_test.sh index 999fa07..7059afb 100644 --- a/app/crawler/cache_test.sh +++ b/app/crawler/cache_test.sh @@ -1,7 +1,7 @@ #! /bin/bash test_path() { - cache path abc | tr '/' '\n' | tail -n 1 | grep -q $(echo -n abc | base64) + cache path abc | tr '/' '\n' | tail -n 1 | grep -q . } test_get_put_get() { diff --git a/app/crawler/gitlab.sh b/app/crawler/gitlab.sh index f058a72..817347c 100644 --- a/app/crawler/gitlab.sh +++ b/app/crawler/gitlab.sh @@ -12,12 +12,15 @@ gitlab() ( local project="$(_url_to_project_root "$url" | head -n 1)" project="$(urlencode "$project")" local root="$(_url_to_project_root "$url" | tail -n 1)" - blob="$(urlencode "$blob")" if [ -n "$root" ]; then - blob="$(urlencode "$root/$blob")" + blob="${root%/}/${blob#/}" + blob="${blob#/}" + blob="${blob%/}" fi + blob="$(urlencode "$blob")" local path="api/v4/projects/$project/repository/files/$blob/raw" + log _gcurl "https://gitlab-app.eng.qops.net/$path (blob=$blob, project=$project)" _gcurl "https://gitlab-app.eng.qops.net/$path" } diff --git a/app/crawler/main.sh b/app/crawler/main.sh index 90ba4ac..ffff6f7 100644 --- a/app/crawler/main.sh +++ b/app/crawler/main.sh @@ -112,17 +112,19 @@ crawl_with() { base64 | md5sum | cut -c 1-10 | awk '{print $1}' | tr -d '\n' } local i="$1" - export TITLE="$( - echo "$i" | base64 --decode + local full_title="$( + echo "$i" | base64 --decode | grep . || echo "${crawlable_source##*/}" )" - TITLE="${TITLE##*/}" + full_title="${full_title%/}" + full_title="${full_title#/}" + export TITLE="${full_title##*/}" export CONTENT="$( $backend get "$crawlable_source" "$i" )" export ID="$( local sum="$pid/" local title_so_far="" - for subtitle in $(echo "$i" | base64 --decode | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do + for subtitle in $(echo $full_title | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do local subtitle="$(echo "$subtitle" | base64 --decode)" if [ -n "$title_so_far" ]; then local mkdir_p_title="${title_so_far%/}" @@ -138,9 +140,13 @@ crawl_with() { log " $ID ($TITLE): ${#CONTENT}" push_crawled "$PID/$ID" "$TITLE" "$CONTENT" } - for i in $(seq 0 $(("${#expanded[@]}"-1))); do - one "${expanded[i]}" - done + if [ "${#expanded[@]}" -gt 0 ]; then + for i in $(seq 0 $(("${#expanded[@]}"-1))); do + one "${expanded[i]}" + done + else + one "" + fi } push_crawled() { diff --git a/spike/review/reinvent/ezmded/ui/index.html b/spike/review/reinvent/ezmded/ui/index.html index 9ea2bf3..07cf11c 100644 --- a/spike/review/reinvent/ezmded/ui/index.html +++ b/spike/review/reinvent/ezmded/ui/index.html @@ -126,7 +126,7 @@ throw `failed to push file ${id}: ${status}: ${body}` } drawTree() - drawFile(id) + //drawFile(id) document.getElementById("saveFeedback").innerHTML = "success!" if (saveFeedbackInterval) { clearInterval(saveFeedbackInterval) @@ -283,11 +283,11 @@