going-flying.com gemini git repository
c9132f89279f0f9d39d23647c636a067f89e5bff - Matthew Ernisse - 1612278035
going to have my own capcom I guess.
diff --git a/capcom/capcom.py b/capcom/capcom.py
new file mode 100644
index 0000000..56672bf
--- /dev/null
+++ b/capcom/capcom.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+import argparse
+import datetime
+import os.path
+import time
+import urllib.parse
+
+import feedparser
+import gusmobile
+
+def load_feed_urls(filename="feeds.txt"):
+ feeds = []
+ with open(filename, "r") as fp:
+ for line in fp:
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ feeds.append(line)
+ return feeds
+
+def add_port_to_url(url):
+ # A temporary patch for a bug in gusmobile
+ url = urllib.parse.urlsplit(url)
+ if ":" not in url.netloc:
+ url = url._replace(netloc=url.netloc+":1965")
+ return urllib.parse.urlunsplit(url)
+
+def items_from_feed_string(feed_str):
+ items = []
+ feed_obj = feedparser.parse(feed_str)
+ feed = feed_obj.feed
+ for entry in feed_obj.entries:
+ # Only add gemini:// URLs
+ if not entry.link.startswith("gemini://"):
+ for link in entry.links:
+ if link.rel == "alternate" and link.href.startswith("gemini://"):
+ entry.link = link.href
+ break
+ else:
+ continue
+ # Use published timestamp instead of updated timestamp, if it exists.
+ # Skip entries with no timestamp at all.
+ timestamp = entry.get("updated_parsed", None)
+ timestamp = entry.get("published_parsed", timestamp)
+ if timestamp:
+ items.append((timestamp, entry.link, entry.title, feed.title))
+ return items
+
+def format_aggregated(items, filename, n_feeds):
+ with open(filename, "w") as fp:
+ # Add header
+ if os.path.exists("header.gmi"):
+ with open("header.gmi", "r") as fp2:
+ fp.write(fp2.read())
+ else:
+ fp.write("# CAPCOM Gemini feed aggregator\n\n")
+ # Feed count
+ fp.write("Aggregating {} Atom feeds from Geminispace.\n".format(n_feeds))
+ # List feed entries
+ current_day = (0,0)
+ for updated, link, entry_title, feed_title in items:
+ item_day = (updated.tm_year, updated.tm_yday)
+ if item_day != current_day:
+ current_day = item_day
+ fp.write("\n## " + datetime.datetime.fromtimestamp(time.mktime(updated)).strftime("%Y-%m-%d") + "\n\n")
+ fp.write("=> {} {} - {}\n".format(link, feed_title, entry_title))
+ fp.write("\n")
+ # Add footer
+ if os.path.exists("footer.gmi"):
+ with open("footer.gmi", "r") as fp2:
+ fp.write(fp2.read())
+
+def aggregate(feed_file="feeds.txt", output_file="index.gmi"):
+ # Load feed URLs to query
+ feed_urls = load_feed_urls(feed_file)
+ N = len(feed_urls)
+
+ # Prepare to extract feed items
+ last_accessed = {}
+ skips = 0
+ items = []
+ while feed_urls:
+ # Get a feed URL to fetch
+ feed_url = feed_urls.pop()
+
+ # Don't hammer servers
+ netloc = urllib.parse.urlsplit(feed_url).netloc
+ last = last_accessed.get(netloc, 0)
+ now = time.time()
+ interval = int(now - last)
+ if interval < 5:
+ print("Declining to hit {} again after only {} seconds".format(netloc, interval))
+ feed_urls.insert(0, feed_url)
+ skips += 1
+ if skips == len(feed_urls):
+ # We've hammered every server in the queue! Sleep a bit...
+ print("Sleeping to give all servers a rest!")
+ time.sleep(5)
+ continue
+ skips = 0
+
+ # Good to go
+ print("Fetching ", feed_url)
+ feed_url = add_port_to_url(feed_url)
+ try:
+ resp = gusmobile.fetch(feed_url)
+ if resp.status == "20":
+ last_accessed[netloc] = time.time()
+ items.extend(items_from_feed_string(resp.content))
+ except:
+ print("Error on {}, skipping...".format(feed_url))
+ continue
+
+ # Find 64 most recent items
+ items.sort(reverse=True)
+ items = items[0:64]
+
+ # Format output
+ format_aggregated(items, output_file, N)
+
+def main():
+ # Parse arguments
+ parser = argparse.ArgumentParser(description='Aggregate Atom feeds to a Gemini doc.')
+ parser.add_argument('-f', '--feeds', dest='feed_file', type=str,
+ default="feeds.txt", help="file to read feed URLs from")
+ parser.add_argument('-o', '--output', dest='output', type=str,
+ default="index.gmi", help='output filename')
+ args = parser.parse_args()
+
+ # Aggregate feeds
+ aggregate(args.feed_file, args.output)
+
+if __name__ == "__main__":
+ main()
diff --git a/capcom/feeds.txt b/capcom/feeds.txt
new file mode 100644
index 0000000..89ff900
--- /dev/null
+++ b/capcom/feeds.txt
@@ -0,0 +1,100 @@
+gemini://carcosa.net/send-the-nukes/atom.xml
+gemini://carcosa.net/journal/atom.xml
+gemini://gemini.circumlunar.space/users/solderpunk/gemlog/atom.xml
+gemini://gemini.circumlunar.space/users/solderpunk/pikkulog/atom.xml
+gemini://typed-hole.org/journal/atom.xml
+gemini://mozz.us/journal/atom.xml
+gemini://80h.dev/glog/atom.xml
+gemini://gemini.circumlunar.space/users/supurb/atom.xml
+gemini://samsai.eu/gemlog/atom.xml
+gemini://gemini.circumlunar.space/users/shufei/phlog/atom.xml
+gemini://gem.1.21jiggawatts.net/posts/atom.xml
+gemini://hannuhartikainen.fi/twinlog/atom.xml
+gemini://thelambdalab.xyz/atom.xml
+gemini://gemini.circumlunar.space/users/acdw/atom.xml
+gemini://republic.circumlunar.space/users/itsdave/atom.xml
+gemini://gemini.circumlunar.space/users/druxx/atom.xml
+gemini://acidic.website/musings/atom.xml
+gemini://makeworld.gq/gemlog/atom.xml
+gemini://demifiend.org/atom.xml
+gemini://starbreaker.org/atom.xml
+gemini://tanelorn.city/~vidak/atom.xml
+gemini://otrn.org/atom.xml
+gemini://qd.discordian.de/entries/atom.xml
+gemini://pon.ix.tc/~krixano/gemlog/atom.xml
+gemini://gemini.circumlunar.space/users/alchemist/gemlog/atom.xml
+gemini://makeworld.gq/users/~atyrfingerprints/gemlog/atom.xml
+gemini://gem.pwarren.id.au/gemlog/atom.xml
+gemini://gemini.circumlunar.space/users/parker/gemlog/atom.xml
+gemini://9til.de/users/~julienxx/atom.xml
+gemini://alexschroeder.ch:1965/do/atom
+gemini://gemini.marmaladefoo.com/cgi-bin/atom-feed.cgi?lukee
+gemini://dctrud.randomroad.net/gemlog/atom.xml
+gemini://envs.net/~vee/pikkulog/atom.xml
+gemini://envs.net/~vee/gemlog/atom.xml
+gemini://mrnd.xyz/log/atom.xml
+gemini://gemini.conman.org/boston.atom
+gemini://upyum.com/journal/feed.atom
+gemini://cetacean.club/journal/atom.xml
+gemini://apintandaparma.club/~ajc/log/atom.xml
+gemini://going-flying.com/~mernisse/atom.xml
+gemini://gem.rmgr.dev/blog/atom.xml
+gemini://kvothe.one/gemlog/atom.xml
+gemini://tilde.team/~easeout/glog/atom.xml
+gemini://ebc.li/atom.xml
+gemini://sunshinegardens.org/~xj9/feed.atom
+gemini://tanelorn.city/~bouncepaw/gemlog/feed.rss
+gemini://gemini.go350.com/atom.xml
+gemini://inconsistentuniverse.space/atom.xml
+gemini://avalos.me/gemlog/atom.xml
+gemini://misterbanal.net/~reedwade/atom.xml
+gemini://pulham.info/atom.xml
+gemini://envs.net/~lrb/gemlog/atom.xml
+gemini://perplexing.space/atom.xml
+gemini://idiomdrottning.org/atom.xml
+gemini://l-3.space/atom.xml
+gemini://idf.looting.uk/capslog/atom.xml
+gemini://tilde.team/~sumpygump/gemlog/atom.xml
+gemini://tilde.team/~ivanruvalcaba/glog/atom.xml
+gemini://republic.circumlunar.space/users/flexibeast/gemlog/atom.xml
+gemini://muscar.eu/feeds/feed.atom.xml
+gemini://1436.ninja/gemlog/gemlog.atom
+gemini://nytpu.com/gemlog/atom.xml
+gemini://tilde.club/~lewiscowper/gemlog/atom.xml
+gemini://gemini.susa.net/atom.xml
+gemini://gemini.circumlunar.space/~adiabatic/atom.xml
+gemini://apintandaparma.club/~ajft/phlog/atom.xml
+gemini://posixcafe.org/blogs/feed.atom
+gemini://drewdevault.com/feed.xml
+gemini://celehner.com/feed.xml
+gemini://rawtext.club/~mieum/dallok/atom.xml
+gemini://republic.circumlunar.space/~luminar/atom_feed.xml
+gemini://xhrpb.com/feed.xml
+gemini://tanso.net/atom.xml
+gemini://rosenzweig.io/gemlog/atom.xml
+gemini://rwv.io/atom.xml
+gemini://amami.city/~xdefrag/atom.xml
+gemini://thesudorm.com/atom.xml
+gemini://jfh.me/atom.xml
+gemini://tilde.team/~ew0k/atom.xml
+gemini://shit.cx/atom.xml
+gemini://rootkey.co.uk/posts/atom.xml
+gemini://gemini.omarpolo.com/rss.xml
+gemini://gemini.swinslow.net/gemlog/atom.xml
+gemini://gemini.femmeandroid.com/blog/atom.xml
+gemini://talon.computer/log/atom.xml
+gemini://rawtext.club/~exastencil/atom.xml
+gemini://www.demorrow.net/atom.xml
+gemini://republic.circumlunar.space/users/joneworlds/atom.xml
+gemini://overeducated-redneck.net/glog/atom.xml
+gemini://palm93.com/atom.xml
+gemini://tilde.team/~emilis/feed.xml
+gemini://blekksprut.net/%E6%97%A5%E5%B8%B8%E9%91%91%E8%B3%9E/atom.xml
+gemini://skyjake.fi/gemlog/atom.xml
+gemini://low-key.me/gemlog/atom.xml
+gemini://breadpunk.club/~bagel/atom.xml
+gemini://calcuode.com/gemlog/atom.xml
+gemini://gem.acdw.net/do/rss
+gemini://caolan.uk/atom.xml
+gemini://gemini.sensorstation.co/atom.xml
+gemini://gemini.ctrl-c.club/~nehrman/gemlog/atom.xml