💾 Archived View for pub.phreedom.club › ~localhost › indexer › tech_indexer.sh captured on 2023-01-29 at 02:58:30.

View Raw

More Information

-=-=-=-=-=-=-

#!/bin/bash

basedir=$(dirname $0)
urls=$(cat $basedir/urls.index)
main_page=$basedir/tech_indexer.gmi
echo "
# Tech (as possible) indexer
Pretty simply - no diffs, no cache, just hashes
Slow checks, no follow internal links (except manually added), manual adding urls
Updated: $(date --iso=minutes --utc)

" > $main_page

for i in $urls; do
	content=$(gemget -o - $i)
	hash_content=$(echo "$content" | sha256sum)
	capsule_url=$(echo $i | tr '/:' '_')
	if [ ! -f "$basedir/$capsule_url.sha256" ]; then
		echo "=> $i $i (new index)" >> $main_page
		echo "$hash_content" > "$basedir/$capsule_url.sha256"
		sleep 5
		continue
	fi
	echo "$content" | sha256sum -c "$basedir/$capsule_url.sha256"
	if [ $? -eq 0 ]; then
		echo "=> $i $i (not changed)" >> $main_page
	else
		echo "=> $i $i (updated)" >> $main_page
		echo "$hash_content" > "$basedir/$capsule_url.sha256"
	fi
	sleep 10
done

echo '' >> $main_page
echo '=> . Look inside working indexer dir' >> $main_page