going-flying.com gemini git repository
aef063ca3c44abfa18092093dd0b93b9f64a4509 - Matthew Ernisse - 1619123142
put this here...
diff --git a/files/gemini-stats.py b/files/gemini-stats.py
new file mode 100644
index 0000000..ed5b37f
--- /dev/null
+++ b/files/gemini-stats.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+'''gemini-stats.py (c) 2020-2021 Matthew J Ernisse <matt@going-flying.com>
+All Rights Reserved.
+
+Redistribution and use in source and binary forms,
+with or without modification, are permitted provided
+that the following conditions are met:
+
+ * Redistributions of source code must retain the
+ above copyright notice, this list of conditions
+ and the following disclaimer.
+ * Redistributions in binary form must reproduce
+ the above copyright notice, this list of conditions
+ and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+import datetime
+import pytz
+import re
+import sys
+import os
+from urllib.parse import urlparse
+
+MAXWIDTH=60
+MAXLABEL=30
+
+class MollyStats(object):
+ ''' Look at a Molly Brown logfile and generate some statistics
+ from it.
+
+ Produces:
+ Hits per page
+ Hits per day
+ Errors (url, code tuple)
+ '''
+ logre = re.compile(
+ r'^.*:\s+(?P<date>\d{4}-\d{2}-\d{2}T(\d{2}:){2}\d{2}Z)\s*(?P<addr>\[[a-f0-9:]+\]|[0-9\.]+)\s+(?P<code>\d{2})\s+(?P<url>.+)
+ )
+
+ def __init__(self, logfile):
+ ''' Load logfile and populate the internal data structures. '''
+ self.byDay = {}
+ self.errors = {}
+ self.files = {}
+
+ try:
+ self.start = datetime.datetime.utcfromtimestamp(2**64)
+ except OverflowError:
+ self.start = datetime.datetime.utcfromtimestamp(2**32)
+
+ self.start = self.start.replace(tzinfo=pytz.utc)
+ self.end = datetime.datetime.utcfromtimestamp(0)
+ self.end = self.end.replace(tzinfo=pytz.utc)
+
+ with open(logfile) as fd:
+ for line in fd.readlines():
+ matches = self.logre.match(line)
+ if not matches:
+ continue
+
+ addr = matches.group('addr')
+ code = int(matches.group('code'))
+ date = datetime.datetime.strptime(
+ matches.group('date'),
+ '%Y-%m-%dT%H:%M:%SZ'
+ )
+ date = date.replace(tzinfo=pytz.utc)
+ url = urlparse(matches.group('url')).path
+ url = self.elide(url)
+
+ if date < self.start:
+ self.start = date
+
+ if date > self.end:
+ self.end = date
+
+ if code > 39:
+ key = f'{url}:{code}'
+ if key not in self.errors.keys():
+ self.errors[key] = 1
+ else:
+ self.errors[key] += 1
+
+ continue
+
+ if url not in self.files:
+ self.files[url] = 1
+ else:
+ self.files[url] += 1
+
+ if date.day not in self.byDay:
+ self.byDay[date.day] = 1
+ else:
+ self.byDay[date.day] += 1
+
+ def elide(self, url):
+ global MAXLABEL
+ if url.startswith('/git/cgi/gemini.git/'):
+ url = '/git/cgi/gemini.git...'
+
+ elif len(url) > MAXLABEL:
+ url = url[:MAXLABEL-3] + '...'
+
+ return url
+
+
+def histogram(data, axis_label, sort_keys=True):
+ ''' Pretty print a histogram. '''
+ global MAXLABEL, MAXWIDTH
+
+ if sort_keys:
+ data.sort(key=lambda k: k[1], reverse=True)
+
+ max = data[0][1]
+ min = data[-1][1]
+ else:
+ max = 0
+ min = 2**128
+
+ for v in data:
+ if v[1] > max:
+ max = v[1]
+
+ if v[1] < min:
+ min = v[1]
+
+ scale = 1
+ total = sum([v[1] for v in data])
+
+ axis_sz = 0
+ for v in data:
+ if len(v[0]) > axis_sz:
+ axis_sz = len(v[0]) + 1
+
+ if len(axis_label) > axis_sz:
+ axis_sz = len(axis_label) + 1
+
+ # Calculate maximum bar length based on the axis label size and
+ # the value label size. Magic number 7 is the number of spaces
+ # and decorative characters in the line that are not accounted
+ # for otherwise.
+ bar_sz = MAXWIDTH - (axis_sz + 7)
+ # + len(f'{max}'))
+ while (max / scale) > bar_sz:
+ scale += 1
+
+ print('```')
+
+ # Center-ish the axis label
+ n_sp = axis_sz - len(axis_label)
+ pad = ' ' * int(n_sp / 2)
+ bar = '=' * bar_sz
+
+ print(f' {pad}{axis_label}')
+ print('=' * MAXWIDTH)
+
+ # Finally print the data bars
+ for v in data:
+ n_sp = axis_sz - len(v[0])
+ pad = ' ' * n_sp
+ bar = '∎' * round(v[1] / scale)
+ print(f'{pad}{v[0]} | {bar} ({v[1]})')
+
+ # and the footer
+ print('=' * MAXWIDTH)
+ print(f'Min: {min}, Max: {max}, Total: {total}')
+ print(f'( ∎ = {scale} )')
+ print('```')
+
+def print_stats(stats):
+ tz = pytz.timezone('US/Eastern')
+ start = stats.start.astimezone(tz).strftime('%m/%d/%Y %H:%M %Z')
+ end = stats.end.astimezone(tz).strftime('%m/%d/%Y %H:%M %Z')
+ now = datetime.datetime.now().astimezone(tz).strftime(
+ '%m/%d/%Y %H:%M %Z'
+ )
+
+ print(f'# Gemini Log Stats for {start} to {end}.')
+ print(f'## Hits:')
+ print()
+
+ if stats.files:
+ histogram([(k, v) for k, v in stats.files.items()], 'File')
+ else:
+ print('No Hits Recorded.')
+
+ print()
+ print()
+ print('## Hits Per Day:')
+ print()
+
+ histogram([(str(k), v) for k, v in stats.byDay.items()], 'Day', False)
+
+ print()
+ print()
+ print('## Errors:')
+ print()
+
+ if stats.errors:
+ histogram(
+ [(k, v) for k, v in stats.errors.items()],
+ 'Error Code'
+ )
+ else:
+ print('No Errors Recorded.')
+
+
+ print()
+ print()
+ print(f'Generated at {now}.')
+
+
+if __name__ == '__main__':
+ stats = MollyStats(sys.argv[1])
+ print_stats(stats)