💾 Archived View for calcuode.com › gemlog › kobo-download-gmisub.sh captured on 2023-04-26 at 13:09:19.

View Raw

More Information

⬅️ Previous capture (2023-03-20)

-=-=-=-=-=-=-

#!/bin/sh

# Kobo specific
export HOME=/root
cd /mnt/onboard/gmisub

# Looks in current directory for folders like yyyymmdd
# Gets posts from days after the latest known day and before today

ext='txt'
special='.[]/\*^$()+?{|:"'

# directories are like yyyymmdd
last_date_got=`ls -rd */ | cut -c1-8 | head -1`
today=`date '+%Y%m%d'`

gmni -j once gemini://calcuode.com/gmisub-aggregate.gmi > gmisub.gmi

while read line
do
    # Check if this line is a second level heading (so will have a date)
    new_date=`echo "$line" | sed -n 's/^## //p' | sed 's/-//g'`
    if [ -n "$new_date" ]
    then
        if [ "$new_date" -ge "$today" ]
        then
            # Skip today (and future dates)
            # because probably not all posts have been aggregated yet
            continue
        elif [ "$new_date" -gt "$last_date_got" ]
        then
            # $new_date is more recent than $last_date_got
            mkdir -p "$new_date"
            date=$new_date
        else
            # $new_date is older than (or same as) $last_date_got
            # All new complete days have been got so exit loop
            break
        fi
        continue
    fi

    # If $date is set, check if the current line is a link
    # and if so download the post.
    if [ -n "$date" ]
    then
        link_line=`echo "$line" | sed -n 's/^=> //p'`
        if [ -n "$link_line" ]
        then
            url=`echo "$link_line" | sed -n 's/ .*$//p'`
            # Extract title and replace difficult characters with X
            title=`echo "$link_line" | sed -n 's/^[^ ]* //p' | tr "$special" "X"`
            gmni -j once $url > "$date/$title.$ext"
        fi
        continue
    fi

done < gmisub.gmi

rm gmisub.gmi