no submod

work/notea/crawler/rclone.sh

@@ -0,0 +1,62 @@
+#! /bin/bash
+
+rclone() (
+	get_google() {
+      local cache_key="rclone get google 2 $*"
+      if cache get "$cache_key"; then
+         return 0
+      fi
+      _get_google "$@" | cache put "$cache_key"
+	}
+
+	_get_google() {
+		_rate_limit
+		local id="$1"
+		local out="$(mktemp -d)"
+		_cmd backend copyid work-notes-google: --drive-export-formats=csv,html,txt "$id" "$out/"
+		find "$out" -type f
+	}
+
+	_rate_limit() {
+		local f="/tmp/rclone.rate.limit"
+		local last=0
+		if [ -f "$f" ]; then
+			last="$(date -r "$f" +%s)"
+		fi
+		local now="$(date +%s)"
+		local since_last=$((now-last))
+		if ((since_last>2)); then
+			dur=-2
+		fi
+		dur=$((dur+2))
+		sleep $dur
+		touch "$f"
+	}
+
+	_ensure() {
+		which rclone &> /dev/null && rclone version &> /dev/null
+	}
+
+	_cmd() {
+		_ensure_google_config
+		__cmd "$@"
+   }
+
+	__cmd() {
+		_ensure
+   	RCLONE_CONFIG_PASS="$RCLONE_CONFIG_PASS" \
+			$(which rclone) \
+   	      --config "$RCLONE_CONFIG" \
+   	      --size-only \
+   	      --fast-list \
+   	      --retries 10 \
+   	      --retries-sleep 10s \
+   	   "$@"
+	}
+
+	_ensure_google_config() {
+		__cmd config show | grep -q work-notes-google
+	}
+
+	"$@"
+)