notea-de-me/crawler/rclone.sh

63 lines
1.1 KiB
Bash

#! /bin/bash
rclone() (
get_google() {
local cache_key="rclone get google 2 $*"
if cache get "$cache_key"; then
return 0
fi
_get_google "$@" | cache put "$cache_key"
}
_get_google() {
_rate_limit
local id="$1"
local out="$(mktemp -d)"
_cmd backend copyid work-notes-google: --drive-export-formats=csv,html,txt "$id" "$out/"
find "$out" -type f
}
_rate_limit() {
local f="/tmp/rclone.rate.limit"
local last=0
if [ -f "$f" ]; then
last="$(date -r "$f" +%s)"
fi
local now="$(date +%s)"
local since_last=$((now-last))
if ((since_last>2)); then
dur=-2
fi
dur=$((dur+2))
sleep $dur
touch "$f"
}
_ensure() {
which rclone &> /dev/null && rclone version &> /dev/null
}
_cmd() {
_ensure_google_config
__cmd "$@"
}
__cmd() {
_ensure
RCLONE_CONFIG_PASS="$RCLONE_CONFIG_PASS" \
$(which rclone) \
--config "$RCLONE_CONFIG" \
--size-only \
--fast-list \
--retries 10 \
--retries-sleep 10s \
"$@"
}
_ensure_google_config() {
__cmd config show | grep -q work-notes-google
}
"$@"
)