gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ok i need tests now

master
Bel LaPointe 2022-02-01 08:33:55 -07:00
parent 3c34670b4f
commit 6c3229eeb3
2 changed files with 36 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
app/crawler/gitlab.sh
View File

@ -5,16 +5,13 @@ gitlab() (
echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wiki/' echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wiki/'
} }
_gcurl() { get() {
local cache_key="gitlab _gcurl $*" local project="$1"
if cache get "$cache_key"; then local blob="$2"
return 0 project="$(urlencode "$project")"
fi blob="$(urlencode "$blob")"
__gcurl "$@" | cache put "$cache_key" local path="api/v4/projects/$project/repository/files/$blob/raw"
} _gcurl "https://gitlab-app.eng.qops.net/$path"
__gcurl() {
curl -sS -H "Authorization: Bearer $GITLAB_PAT" "$@"
} }
expand() { expand() {
@ -76,8 +73,9 @@ gitlab() (
done done
i=$((i+1)) i=$((i+1))
done done
echo "$project" | base64
for b64_file in "${b64_files[@]}"; do for b64_file in "${b64_files[@]}"; do
echo "$b64_file" | base64 --decode echo "$b64_file"
done done
} }
@ -88,6 +86,18 @@ gitlab() (
_gcurl "https://gitlab-app.eng.qops.net/$path?$query" _gcurl "https://gitlab-app.eng.qops.net/$path?$query"
} }
_gcurl() {
local cache_key="gitlab _gcurl $*"
if cache get "$cache_key"; then
return 0
fi
__gcurl "$@" | cache put "$cache_key"
}
__gcurl() {
curl -sS -H "Authorization: Bearer $GITLAB_PAT" "$@"
}
"$@" "$@"
) )

16
app/crawler/main.sh
View File

@ -49,7 +49,7 @@ crawl() {
if ! is_crawlable "$content"; then if ! is_crawlable "$content"; then
return 0 return 0
fi fi
local crawlable_source="$(echo "$content" | head -n 1 | awk '{print $NF}')" local crawlable_source="$(extract_crawlable_source "$content")"
for backend in gitlab; do for backend in gitlab; do
if $backend is "$crawlable_source"; then if $backend is "$crawlable_source"; then
crawl_with $backend "$json" crawl_with $backend "$json"
@ -60,9 +60,23 @@ crawl() {
return 1 return 1
} }
extract_crawlable_source() {
echo "$*" | head -n 1 | awk '{print $NF}'
}
crawl_with() { crawl_with() {
local backend="$1" local backend="$1"
local json="$2" local json="$2"
local content="$(echo "$json" | jq -r .content)"
local crawlable_source="$(extract_crawlable_source "$content")"
local expanded=("$($backend expand "$crawlable_source")")
local context="$expanded"
for i in $(seq 1 "${#expanded[@]}"); do
log expand $context, ${expanded[i]}
done
log not impl crawl with log not impl crawl with
return 1 return 1
} }