local-sandbox
/
dnd-beyond-scrape
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rm logs

master
bel 2022-06-27 19:25:17 -06:00
parent 69a2003398
commit 260815e53c
1 changed files with 0 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrape.sh
View File

@ -234,10 +234,7 @@ url_to_title_candidate() {
html_to_markdown() { html_to_markdown() {
local f="$(mktemp)" local f="$(mktemp)"
log url=$1 log url=$1
log 1: url=$1
cat > "$f" cat > "$f"
log 2: url=$1
#grep '<title>' "$f" | sed 's/.*>\([^<]*\)<.title>.*/# \1/g'
local clean="$( local clean="$(
cat "$f" \ cat "$f" \
| readability "file://$(realpath $f)" \ | readability "file://$(realpath $f)" \
@ -245,9 +242,7 @@ html_to_markdown() {
| sed 's/{[^}]*data[^}]*}//g' \ | sed 's/{[^}]*data[^}]*}//g' \
| sed 's/\[](#[^)]*)//' | sed 's/\[](#[^)]*)//'
)" )"
log 3: url=$1
rm "$f" rm "$f"
log 4: url=$1
echo "$clean" | ( echo "$clean" | (
lastline="" lastline=""
while read -r line; do while read -r line; do
@ -260,9 +255,7 @@ html_to_markdown() {
done done
cat &> /dev/null cat &> /dev/null
) )
log 5: url=$1
echo "$clean" echo "$clean"
log 6: url=$1
return $? return $?
} }