diff --git a/crawler/google.sh b/crawler/google.sh index e1d5f1a..9c7249c 100644 --- a/crawler/google.sh +++ b/crawler/google.sh @@ -18,29 +18,33 @@ google() ( } get() { - local cache_key="google get $*" - if cache get "$cache_key"; then - return 0 - fi - _get "$@" | cache put "$cache_key" - } - - _get() { local url="$1" local id="${url%/*}" id="${id##*/}" local downloaded="$(rclone get_google "$id")" echo "# ${downloaded##*/}" echo "" - if [ "${downloaded##*.}" == ".csv" ]; then + if [ "${downloaded##*.}" == "csv" ]; then _csv_to_md "$downloaded" + else + cat "$downloaded" fi - cat "$downloaded" } _csv_to_md() { local f="$1" - log _csv_to_md $f + ( + head -n 1 "$f" + head -n 1 "$f" \ + | sed 's/^[^,][^,]*/--- /' \ + | sed 's/[^,][^,]*$/ ---/' \ + | sed 's/,[^,][^,]*/, --- /g' \ + | sed 's/[^|]$/|/' + tail -n +2 "$f" + ) \ + | grep . \ + | sed 's/,/ | /g' \ + | sed 's/^/| /' } expand() {