#! /bin/bash HUGO_POSTS="${HUGO_POSTS:-"./posts"}" RCLONE_ALIAS="${RCLONE_ALIAS:-"hugo"}" RCLONE_CONFIG="${RCLONE_CONFIG:-./rclone.conf}" RCLONE_FOLDER="${RCLONE_FOLDER:-"shared_from_bbarl64"}" RCLONE_OPTS="${RCLONE_OPTS:-"--drive-shared-with-me"}" TMPDIR="${TMPDIR:-"/tmp"}" main() { flags "$@" list | while read -r json; do local filename="$(echo "$json" | jq -r .Name)" if up_to_date "$json"; then log "$filename: up to date" continue fi log "$filename: pull" pull "$json" log "$filename: extract" extract "$filename" log "$filename: import" import "$filename" log "$filename: metadata" metadata "$json" done echo TODO removed } flags() { set -e set -o pipefail cd "$(dirname "$BASH_SOURCE")" } fatal() { log "$@" exit 2 } log() { echo "$(date +%H:%M:%S) > $*" >&2 } list() { gd list } up_to_date() { local filename="$(echo "$1" | jq -r .Name)" local metadatad="$(metadatad "$filename")" if [ -f "$metadatad" ]; then if [ "$1" == "$(cat "$metadatad")" ]; then return 0 fi fi return 1 } pulled() { echo "$TMPDIR/$1" } pull() { gd pull "$filename" } extracted() { local pulled_file="$(pulled "$1")" local date="$(date -r "$pulled_file" +%Y-%m-%d)" local pulled_file_safe_basename="$(basename "$pulled_file" | sed 's/.zip$//' | sed 's/[^a-zA-Z0-9]/_/g')" echo "$TMPDIR/${date}_${pulled_file_safe_basename}" } extract() { local extracted_dir="$(extracted "$1")" local index_html="$extracted_dir/.index.html" local index_md="$extracted_dir/index.md" local pulled_file="$(pulled "$1")" local u_date_iso="$(date -r "$pulled_file" -u +%Y-%m-%dT%H:%M:%S)-00:00" if [ -d "$extracted_dir" ]; then rm -rf "$extracted_dir" fi mkdir -p "$extracted_dir" 7z x -o"$extracted_dir" "$pulled_file" cat "$extracted_dir"/*.html \ | sed 's/.*.*/<\/body>/' \ > "$index_html" local tags=($(cat "$index_html" | sed '/^$/!{s/<[^>]*>//g;p;}' | grep -o '#[a-zA-Z0-9]*' | grep '[a-zA-Z]' | sed 's/^#//' | sort -u)) local tags_csv="$(first=false; for tag in "${tags[@]}"; do true; if $first; then echo -n ", "; fi; first=true; echo -n "$tag"; done)" printf ' --- title: "%s" date: %s draft: false tags: [%s] --- ' \ "$(basename "$pulled_file" | sed 's/.zip$//' | sed 's/"/\\"/g')" \ "$u_date_iso" \ "$tags_csv" \ | sed 's/^[ ]*//' \ | grep . \ > "$index_md" cat "$index_html" >> "$index_md" echo "" >> "$index_md" rm "$index_html" "$extracted_dir"/*.html for ext in png jpg jpeg gif JPG; do find "$extracted_dir" -name "*.$ext"; done | while read -r line; do convert "$line" -ordered-dither o8x8,8,8,4 "$line.2" mv "$line.2" "$line" done rm "$pulled_file" } imported() { local extracted_dir="$(extracted "$1")" echo "$HUGO_POSTS/$(basename "$extracted_dir")" } import() { local extracted_dir="$(extracted "$1")" local target_dir="$(imported "$1")" if [ -d "$target_dir" ]; then rm -rf "$target_dir" fi mv "$extracted_dir" "$target_dir" } metadatad() { local imported="$(imported "$1")" echo "$imported"/.metadata.json } metadata() { local filename="$(echo "$1" | jq -r .Name)" local metadatad="$(metadatad "$filename")" echo "$1" > "$metadatad" } gd() ( list() { rc lsjson "$RCLONE_ALIAS:$RCLONE_FOLDER" | jq -c .[] | grep zip\" } pull() { rc copy "$RCLONE_ALIAS:$RCLONE_FOLDER/$1" "$TMPDIR"/ } rc() { rclone \ --config "$RCLONE_CONFIG" \ --fast-list \ $RCLONE_OPTS \ "$@" } "$@" ) if [ "$0" == "$BASH_SOURCE" ]; then main "$@" fi