gogs
/
notea-de-me
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

kinda unexported

master
Bel LaPointe 2022-02-01 08:24:38 -07:00
parent c6b40404cb
commit 3d9cfc2f95
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/crawler/main.sh
View File

@ -141,15 +141,15 @@ gitlab() (
echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wiki/' echo "$*" | grep -q gitlab.app && ! echo "$*" | grep -q '/wiki/'
} }
gcurl() { _gcurl() {
local cache_key="gitlab gcurl $*" local cache_key="gitlab _gcurl $*"
if cache get "$cache_key"; then if cache get "$cache_key"; then
return 0 return 0
fi fi
_gcurl "$@" | cache put "$cache_key" __gcurl "$@" | cache put "$cache_key"
} }
_gcurl() { __gcurl() {
curl -sS -H "Authorization: Bearer $GITLAB_PAT" "$@" curl -sS -H "Authorization: Bearer $GITLAB_PAT" "$@"
} }
@ -163,12 +163,12 @@ gitlab() (
_expand() { _expand() {
local url="$1" local url="$1"
local project="$(url_to_project_root "$url" | head -n 1)" local project="$(_url_to_project_root "$url" | head -n 1)"
local root="$(url_to_project_root "$url" | tail -n 1)" local root="$(_url_to_project_root "$url" | tail -n 1)"
__expand "$project" "$root" __expand "$project" "$root"
} }
url_to_project_root() { _url_to_project_root() {
local url="$1" local url="$1"
local url_path="${url#http*://gitlab*.net/}" local url_path="${url#http*://gitlab*.net/}"
local project="${url_path%%/-/*}" local project="${url_path%%/-/*}"
@ -199,7 +199,7 @@ gitlab() (
| grep . | grep .
} }
while [ "$i" -lt "${#b64_trees[@]}" ]; do while [ "$i" -lt "${#b64_trees[@]}" ]; do
got="$(list_tree "$project" "${b64_trees[i]}")" got="$(_list_tree "$project" "${b64_trees[i]}")"
for b64_tree in $(find_each "tree" "$got"); do for b64_tree in $(find_each "tree" "$got"); do
if ! echo "${b64_trees[@]}" | grep -q "[ ^]$b64_tree[ $]"; then if ! echo "${b64_trees[@]}" | grep -q "[ ^]$b64_tree[ $]"; then
b64_trees+=("$b64_tree") b64_trees+=("$b64_tree")
@ -217,11 +217,11 @@ gitlab() (
done done
} }
list_tree() { _list_tree() {
local project="$(urlencode "$1")" local project="$(urlencode "$1")"
local path="/api/v4/projects/$project/repository/tree" local path="/api/v4/projects/$project/repository/tree"
local query="recursive=true&path=$2" local query="recursive=true&path=$2"
gcurl "https://gitlab-app.eng.qops.net/$path?$query" _gcurl "https://gitlab-app.eng.qops.net/$path?$query"
} }
"$@" "$@"