delicate but ok import
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
#! /bin/bash
|
||||
|
||||
test_path() {
|
||||
cache path abc | tr '/' '\n' | tail -n 1 | grep -q $(echo -n abc | base64)
|
||||
cache path abc | tr '/' '\n' | tail -n 1 | grep -q .
|
||||
}
|
||||
|
||||
test_get_put_get() {
|
||||
|
||||
@@ -12,12 +12,15 @@ gitlab() (
|
||||
local project="$(_url_to_project_root "$url" | head -n 1)"
|
||||
project="$(urlencode "$project")"
|
||||
local root="$(_url_to_project_root "$url" | tail -n 1)"
|
||||
blob="$(urlencode "$blob")"
|
||||
if [ -n "$root" ]; then
|
||||
blob="$(urlencode "$root/$blob")"
|
||||
blob="${root%/}/${blob#/}"
|
||||
blob="${blob#/}"
|
||||
blob="${blob%/}"
|
||||
fi
|
||||
blob="$(urlencode "$blob")"
|
||||
|
||||
local path="api/v4/projects/$project/repository/files/$blob/raw"
|
||||
log _gcurl "https://gitlab-app.eng.qops.net/$path (blob=$blob, project=$project)"
|
||||
_gcurl "https://gitlab-app.eng.qops.net/$path"
|
||||
}
|
||||
|
||||
|
||||
@@ -112,17 +112,19 @@ crawl_with() {
|
||||
base64 | md5sum | cut -c 1-10 | awk '{print $1}' | tr -d '\n'
|
||||
}
|
||||
local i="$1"
|
||||
export TITLE="$(
|
||||
echo "$i" | base64 --decode
|
||||
local full_title="$(
|
||||
echo "$i" | base64 --decode | grep . || echo "${crawlable_source##*/}"
|
||||
)"
|
||||
TITLE="${TITLE##*/}"
|
||||
full_title="${full_title%/}"
|
||||
full_title="${full_title#/}"
|
||||
export TITLE="${full_title##*/}"
|
||||
export CONTENT="$(
|
||||
$backend get "$crawlable_source" "$i"
|
||||
)"
|
||||
export ID="$(
|
||||
local sum="$pid/"
|
||||
local title_so_far=""
|
||||
for subtitle in $(echo "$i" | base64 --decode | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
|
||||
for subtitle in $(echo $full_title | tr '/' '\n' | while read -r subtitle; do echo "$subtitle" | base64; done); do
|
||||
local subtitle="$(echo "$subtitle" | base64 --decode)"
|
||||
if [ -n "$title_so_far" ]; then
|
||||
local mkdir_p_title="${title_so_far%/}"
|
||||
@@ -138,9 +140,13 @@ crawl_with() {
|
||||
log " $ID ($TITLE): ${#CONTENT}"
|
||||
push_crawled "$PID/$ID" "$TITLE" "$CONTENT"
|
||||
}
|
||||
for i in $(seq 0 $(("${#expanded[@]}"-1))); do
|
||||
one "${expanded[i]}"
|
||||
done
|
||||
if [ "${#expanded[@]}" -gt 0 ]; then
|
||||
for i in $(seq 0 $(("${#expanded[@]}"-1))); do
|
||||
one "${expanded[i]}"
|
||||
done
|
||||
else
|
||||
one ""
|
||||
fi
|
||||
}
|
||||
|
||||
push_crawled() {
|
||||
|
||||
Reference in New Issue
Block a user