delicate but ok import

This commit is contained in:
Bel LaPointe
2022-02-10 11:40:39 -07:00
parent b11c07e55a
commit ac5f86687c
4 changed files with 23 additions and 14 deletions

View File

@@ -1,7 +1,7 @@
#! /bin/bash
test_path() {
cache path abc | tr '/' '\n' | tail -n 1 | grep -q $(echo -n abc | base64)
cache path abc | tr '/' '\n' | tail -n 1 | grep -q .
}
test_get_put_get() {

View File

@@ -12,12 +12,15 @@ gitlab() (
local project="$(_url_to_project_root "$url" | head -n 1)"
project="$(urlencode "$project")"
local root="$(_url_to_project_root "$url" | tail -n 1)"
blob="$(urlencode "$blob")"
if [ -n "$root" ]; then
blob="$(urlencode "$root/$blob")"
blob="${root%/}/${blob#/}"
blob="${blob#/}"
blob="${blob%/}"
fi
blob="$(urlencode "$blob")"
local path="api/v4/projects/$project/repository/files/$blob/raw"
log _gcurl "https://gitlab-app.eng.qops.net/$path (blob=$blob, project=$project)"
_gcurl "https://gitlab-app.eng.qops.net/$path"
}

View File

@@ -112,17 +112,19 @@ crawl_with() {
base64 | md5sum | cut -c 1-10 | awk '{print $1}' | tr -d '\n'
}
local i="$1"
export TITLE="$(
echo "$i" | base64 --decode
local full_title="$(
echo "$i" | base64 --decode | grep . || echo "${crawlable_source##*/}"
)"
TITLE="${TITLE##*/}"
full_title="${full_title%/}"
full_title="${full_title#/}"
export TITLE="${full_title##*/}"
export CONTENT="$(
$backend get "$crawlable_source" "$i"
)"
export ID="$(
local sum="$pid/"
local title_so_far=""
for subtitle in $(echo "$i" | base64 --decode | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
for subtitle in $(echo $full_title | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
local subtitle="$(echo "$subtitle" | base64 --decode)"
if [ -n "$title_so_far" ]; then
local mkdir_p_title="${title_so_far%/}"
@@ -138,9 +140,13 @@ crawl_with() {
log " $ID ($TITLE): ${#CONTENT}"
push_crawled "$PID/$ID" "$TITLE" "$CONTENT"
}
for i in $(seq 0 $(("${#expanded[@]}"-1))); do
one "${expanded[i]}"
done
if [ "${#expanded[@]}" -gt 0 ]; then
for i in $(seq 0 $(("${#expanded[@]}"-1))); do
one "${expanded[i]}"
done
else
one ""
fi
}
push_crawled() {