diff --git a/app/crawler/gitlab_test.sh b/app/crawler/gitlab_test.sh index d087a54..7fec0d2 100644 --- a/app/crawler/gitlab_test.sh +++ b/app/crawler/gitlab_test.sh @@ -47,6 +47,7 @@ EOF } test_url_to_project_root() { + log() { true; }; gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/tree/master | grep -q '^data-store/orchestration/runbooks$' gitlab _url_to_project_root https://gitlab-app.eng.qops.net/data-store/orchestration/runbooks/tree/master | tail -n 1 | grep ^$ diff --git a/app/crawler/notnotea.sh b/app/crawler/notnotea.sh new file mode 100644 index 0000000..2ba7a7f --- /dev/null +++ b/app/crawler/notnotea.sh @@ -0,0 +1,124 @@ +#! /bin/bash + +notnotea() ( + _nncurl() { + curl -sS "$@" + } + + ids() { + local tree="$(_nncurl $NNOTEA_ADDR/api/v0/tree)" + _recurse_ids "" "$tree" + } + + _recurse_ids() { + _echo() { + echo "$*" || true + } + local prefix="$1" + local json="$2" + local b64lines="$(echo "$json" | jq -r '.Branches | keys[]' | base64)" + if [ -z "$b64lines" ]; then + return 0 + fi + for line in $b64lines; do + line="$(echo "$line" | base64 --decode)" + local subfix="$(printf "%s/%s" "$prefix" "$line")" + subfix="${subfix#/}" + _echo "$subfix" + _recurse_ids "$subfix" "$(echo "$json" | jq ".Branches.$line")" + done + } + + ids() { + for id in $(_tree_ids); do + if ! _is_deleted $id; then + echo $id + fi + done + } + + meta() { + local cache_key="notea cache meta $1" + if cache get "$cache_key"; then + return 0 + fi + _meta "$@" | cache put "$cache_key" + } + + _meta() { + _ncurl $NOTEA_ADDR/api/notes/$1/meta + } + + _is_deleted() { + local id="$1" + if [ "$id" == "root" ] || [ "$id" == "null" ]; then + return 1 + fi + local meta="$(meta "$id")" + if echo "$meta" | jq .deleted | grep -q 1; then + return 0 + fi + local pid="$(echo "$meta" | jq -r .pid)" + if [ -z "$pid" ]; then + return 0 + fi + _is_deleted "$pid" + } + + _tree_ids() { + _ncurl $NOTEA_ADDR/api/tree \ + | jq '.items | to_entries[].value.id' \ + | grep -v '^null$' \ + | jq -r . \ + | grep -v '^root$' + } + + get() { + local cache_key="notea cache $1" + if cache get "$cache_key"; then + return 0 + fi + _get "$@" | cache put "$cache_key" + } + + _get() { + _ncurl $NOTEA_ADDR/api/notes/$1 + } + + put() { + set -u + local ret=0 + if ! _put "$@"; then + ret=1 + fi + set +u + return $ret + } + + _put() { + local xsrf_key="xsrf-token" + local contains_tokens="$(_ncurl -i $NOTEA_ADDR/api)" + local xsrf_token="$(echo "$contains_tokens" | grep -o '"csrfToken":[^,]*' | tr ':' '\n' | jq -r . | tail -n 1)" + local xsrf_cookie="$(echo "$contains_tokens" | grep ^set.cookie: | sed 's/^set.cookie: //' | tr ';' '\n' | head -n 1)" + local request="$(echo '{ + "content": '"$(printf "%s\n" "$CONTENT" | jq -Rs)"', + "deleted": 0, + "id": '"$(echo "$ID" | jq -R)"', + "pid": '"$(echo "$PID" | jq -R)"', + "pinned": 0, + "shared": 0, + "title": '"$(echo "$TITLE [generated]" | jq -R)"' + }' | jq -c .)" + echo "$request" | _ncurl \ + -X POST \ + -H "$xsrf_key: $xsrf_token" \ + -b "$xsrf_cookie" \ + -H "Content-Type: application/json" \ + -d @- \ + $NOTEA_ADDR/api/notes \ + | grep -q "$ID" + } + + "$@" +) + diff --git a/app/crawler/notnotea_test.sh b/app/crawler/notnotea_test.sh new file mode 100644 index 0000000..f5caa65 --- /dev/null +++ b/app/crawler/notnotea_test.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +test__recurse_ids() { + local two_levels='{ + "Branches": { + "id": { + "Branches": { + "subid": { + "Branches": {} + } + } + } + } + }' + (notnotea _recurse_ids "" "$two_levels"; true) | grep -q '^id$' || return 101 + (notnotea _recurse_ids "" "$two_levels"; true) | tail -n 1 | grep -q '^id\/subid$' || return 102 + notnotea _recurse_ids "" "$two_levels" | wc -l | grep -q 2 || return 103 +}