#!/usr/bin/python
#
# Generate html pages based on the Mirmon data
#
# This script will parse the json files that contain the mirmon
# data and have been generated before. The json data is used to
# generate html pages that visualize the mirmon data in a human
# readable way.
#
#
# Leaflet is used to visualize the mirrors on a world map. Further-
# more Datatables are used to display the mirrors.
#
import datetime
import socket
import ssl
import os
import tempfile
import urllib.request, json
import xml.etree.ElementTree as ET
import jinja2
html_folder = "/var/www/mirrorstats.gentoo.org/htdocs/"
template_folder = "/var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/html/"
cache_path = "/var/www/mirrorstats.gentoo.org/var/html_data_cache.json"
mirrorstats = [ 'https://mirrorstats.gentoo.org/rsync/state.json',
'https://mirrorstats.gentoo.org/distfiles/state.json',
'https://mirrorstats.gentoo.org/snapshots/state.json',
'https://mirrorstats.gentoo.org/releases/state.json',
'https://mirrorstats.gentoo.org/experimental/state.json'
]
cache_data = {}
#
# Retrieve and parse the JSON at the given URL
#
def getJson(url):
req = urllib.request.Request(url)
r = urllib.request.urlopen(req).read()
return json.loads(r.decode('utf-8'))
#
# Retrieve and parse the XML at the given URL
#
def getXML(url):
req = urllib.request.Request(url)
r = urllib.request.urlopen(req).read()
return ET.loads(r.decode('utf-8')).getroot()
#
# Get the ip of the given host
#
def getIp(hostname):
try:
ip = socket.gethostbyname(hostname)
except:
ip = ""
return ip
#
# Get TLS version of the given host
#
def getTLSVersion(hostname):
socket.setdefaulttimeout(2)
try:
with socket.create_connection((hostname, 443)) as sock:
with ssl.create_default_context().wrap_socket(sock, server_hostname=hostname) as ssock:
return ssock.version().replace("TLSv", "", 1)
except:
print("err")
return ''
#
# Render the stats template for a given page, i.e. all, rsync, distfiles...
#
def renderStatsTemplate(templateEnv, page):
if page == "all":
mirrorstatsList = mirrorstats
page_name = "index"
else:
mirrorstatsList = ['https://mirrorstats.gentoo.org/' + page + '/state.json']
page_name = page
lastUpdate, hostList = getHostList(mirrorstatsList)
template = templateEnv.get_template("stats.jinja2")
template.stream(lastUpdate=lastUpdate, type=page, mirrors=hostList, locations=json.dumps(hostList)).dump(html_folder + page_name + ".html")
return lastUpdate
# read the cache
if os.path.exists(cache_path):
with open(cache_path, mode='rt') as json_file:
try:
cache_data = json.load(json_file)
if 'ips' not in cache_data:
cache_data['ips'] = {}
if 'hosts' not in cache_data:
cache_data['hosts'] = {}
if 'tls_version' not in cache_data:
cache_data['tls_version'] = {}
except:
pass
#
# The all mirrors that are present in the given list
#
def getHostList(mirrorstatsList):
hosts = {}
lastUpdate = ""
# process all mirrors
for mirror_url in mirrorstatsList:
mirrorData = getJson(mirror_url)
lastUpdate = datetime.datetime.fromtimestamp(int(mirrorData['LastUpdate'])).strftime('%Y-%m-%d %H:%M:%S')
for mirror in mirrorData['Mirrors']:
if len(mirrorData['Mirrors'][mirror]) >= 0:
for mirrorHost in mirrorData['Mirrors'][mirror]:
hostname = mirrorHost['Host']
if hostname not in hosts:
hosts[hostname] = {}
hosts[hostname]['Hostname'] = hostname
if hostname in cache_data['hosts']:
ip = cache_data['hosts'][hostname]
else:
ip = getIp(hostname)
# populate cache with new value
cache_data['hosts'][hostname] = ip
if ip != "":
hosts[hostname]['Ip'] = ip
if 'Stats' not in hosts[hostname]:
hosts[hostname]['Stats'] = []
mirrorHost['Type'] = mirror_url.replace("https://mirrorstats.gentoo.org/", "").replace("/state.json", "")
hosts[hostname]['Stats'].append(mirrorHost)
# compute available protocols
for host in hosts:
protocols = []
for stat in hosts[host]['Stats']:
protocols.append(stat['Protocol'])
hosts[host]['Protocols'] = list(set(protocols))
# compute mirror locations
for host in hosts:
if 'Ip' not in hosts[host]:
continue
ip = hosts[host]['Ip']
if ip in cache_data['ips']:
hosts[host]['Location'] = cache_data['ips'][ip]
else:
mirrorGeoData = getJson("https://ipinfo.io/" + ip + "/json")
hosts[host]['Location'] = mirrorGeoData['loc']
# populate cache with new value
cache_data['ips'][ip] = mirrorGeoData['loc']
for host in hosts:
if 'https' not in hosts[host]['Protocols']:
hosts[host]['TLSVersion'] = ''
continue
if host in cache_data['tls_version']:
hosts[host]['TLSVersion'] = cache_data['tls_version'][host]
else:
tlsVersion = getTLSVersion(host)
hosts[host]['TLSVersion'] = tlsVersion
cache_data['tls_version'][host] = tlsVersion
return lastUpdate, hosts
#
# render jinja2
#
templateLoader = jinja2.FileSystemLoader(searchpath=template_folder)
templateEnv = jinja2.Environment(loader=templateLoader)
## stats
lastUpdate = renderStatsTemplate(templateEnv, "all")
renderStatsTemplate(templateEnv, "rsync")
renderStatsTemplate(templateEnv, "distfiles")
renderStatsTemplate(templateEnv, "snapshots")
renderStatsTemplate(templateEnv, "releases")
renderStatsTemplate(templateEnv, "experimental")
## about
template = templateEnv.get_template("help.jinja2")
template.stream(lastUpdate=lastUpdate).dump(html_folder + "help.html")
#
# write the cache
#
with tempfile.NamedTemporaryFile(dir=os.path.dirname(cache_path), delete=False, mode='wt') as fout:
json.dump(cache_data, fout)
os.chmod(fout.name, 0o644)
os.replace(fout.name, cache_path)
#
# finish
#
#print("Finished.")