# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# This file is part of Polyfemos.
#
# Polyfemos is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or any later version.
#
# Polyfemos is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License and
# GNU General Public License along with Polyfemos. If not, see
# <https://www.gnu.org/licenses/>.'
#
# Author: Henrik Jänkävaara
# -----------------------------------------------------------------------------
"""
Frontend of polyfemos
Utilizes `Flask <http://flask.pocoo.org/>`_
Functions used for different site:
- :func:`~polyfemos.front.main.index_alias`
- :func:`~polyfemos.front.main.home`
- :func:`~polyfemos.front.main.sohtable`
- :func:`~polyfemos.front.main.sohmap`
- :func:`~polyfemos.front.main.datacoverageimage`
- :func:`~polyfemos.front.main.summary`
- :func:`~polyfemos.front.main.datacoveragebrowser`
- :func:`~polyfemos.front.main.plotbrowser`
- :func:`~polyfemos.front.main.alertheat`
See :ref:`Frontend` for documentation.
:copyright:
2019, University of Oulu, Sodankyla Geophysical Observatory
:license:
GNU Lesser General Public License v3.0 or later
(https://spdx.org/licenses/LGPL-3.0-or-later.html)
"""
import os
import itertools
import functools
import math
import csv
import time
from io import BytesIO, StringIO
from datetime import timedelta, datetime
from dateutil import tz
from PIL.PngImagePlugin import PngImageFile
from PIL import Image, ImageDraw, ImageFont
import base64
from urllib.parse import quote
import importlib
from flask import (Flask, make_response, stream_with_context,
send_from_directory)
import numpy as np
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
plt.switch_backend('agg')
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.colors import LinearSegmentedColormap, rgb2hex
import bokeh
from obspy import UTCDateTime
from werkzeug.datastructures import Headers
from werkzeug.wrappers import Response
from polyfemos.parser import typeoperator as to
from polyfemos.util.messenger import messenger
from polyfemos.data import outlierremover
from polyfemos.util import coordinator
from polyfemos.almanac.utils import parse_date, get_jY
from polyfemos.front import colors, userdef, request
from polyfemos.front.sohplot.sohplot import SOHPlot
from polyfemos.front.trafficmonitor import (logged, limited, IPStorage,
check_permission, limit_access)
from polyfemos.front.alertreader import get_sohdict
# Initialize app
app = Flask(__name__, instance_relative_config=True)
app.config.from_object('polyfemos.front.flask_config.ProductionConfig')
app.config['SECRET_KEY'] = userdef.secret_key()
# IPStorage to be used with @limited functions as a decorator argument
ipstorage = IPStorage()
# Replaces the normally used flask's render_template function
render_template = colors.colored_template
if 0:
# If addional logging is wanted
import logging
app.config['LOG_FILE'] = 'application.log'
if not app.debug:
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler(app.config['LOG_FILE'])
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
logger.addHandler(file_handler)
[docs]def render_base(func_):
"""
:type func\_: func
:param func\_: :func:`~flask.render_template`
:rtype: func
:return: decorated function
"""
@functools.wraps(func_)
def wrapper(*args, **kwargs):
utcdatetime_now = UTCDateTime(precision=0)
timestr = []
timestr.append("polyfemos system time {} ".format(
utcdatetime_now.strftime("%Y.%j")))
timestr.append("{}".format(
utcdatetime_now.strftime("%Y-%m-%d %H:%M:%S UTC")))
timestr.append("{}".format(
datetime.now(tz=tz.tzlocal()).strftime("%Y-%m-%d %H:%M:%S %Z")))
if "network_code" not in kwargs:
kwargs["network_code"] = request.cookie("network_code")
return func_(*args, timestr=timestr, **kwargs)
return wrapper
[docs]def get_image_data(image):
"""
Converts the images into ascii
:type image: :obj:`~PIL.PngImagePlugin.PngImageFile` or
:class:`~matplotlib.figure.Figure`, or :obj:`~PIL.Image.Image`
:param image:
:rtype: str
:return: ``image`` as ascii
"""
buffer_ = BytesIO()
if isinstance(image, (PngImageFile, Image.Image)):
image.save(buffer_, 'PNG')
elif isinstance(image, Figure):
canvas = FigureCanvas(image)
canvas.print_png(buffer_)
else:
return ""
data = base64.b64encode(buffer_.getvalue()).decode('ascii')
data = 'data:image/png;base64,{}'.format(quote(data))
return data
[docs]def get_summary(startdate="", enddate="", headerdate="", combinations=[],
remove_irrationals=False, advanced_outlier_removal=False,
fext="csv"):
"""
The function used to get the statistical summary of station/sohpar
combinations.
The execution is little bit slow because the combinations are
dealt separately, which means that, as all of the sohpars are in the same
file, the file is opened and closed multiple times during
the creation of the summary
:type startdate: str or :class:`~datetime.date`
:param startdate:
:type enddate: str or :class:`~datetime.date`
:param enddate:
:type headerdate: str or :class:`~datetime.date`
:param headerdate: The header information of this date is used in
calculations
:type combinations: list[tuple[str, str]]
:param combinations: list containing all unique station and sohpar
combinations
:type remove_irrationals: bool, optional
:param remove_irrationals: defaults to ``False``, If ``True``, irrational
values (limits defined in stf header) are removed from the data.
:type advanced_outlier_removal: bool, optional
:param advanced_outlier_removal: defaults to ``False``. If ``True``,
advanced outlier removal is used for specific station/sohpar
combination. The combinations and outlier removal are provided by
:func:`~polyfemos.front.userdef.summary_outlierremfuncs`, if the
YAML configuration is provided.
:type fext: str, optional
:param fext: defaults to "csv", select "stf" or "csv",
defines the datafile format which is read
:rtype: list, list
:return: rows and header, list of lists and a list.
"""
rows = []
header = []
max_loops = len(combinations)
for i, (station_id, sohpar_name) in enumerate(combinations):
sohplot = SOHPlot(
station_id=station_id,
sohpar_name=sohpar_name,
startdate=startdate,
enddate=enddate,
headerdate=headerdate,
remove_irrationals=remove_irrationals,
advanced_outlier_removal=advanced_outlier_removal,
fext=fext)
dict_ = sohplot.get_statistics_dict()
if not header:
header = ["Station", "Sohpar"] + list(dict_.keys())
row = [station_id, sohpar_name]
# If every numerical value is nan, don't show the parameter
func_ = lambda x: isinstance(x, str) or math.isnan(x)
if all(map(func_, dict_.values())):
continue
rows.append(row + list(dict_.values()))
pr = 100.0 * float(i + 1) / max_loops
msg = "Summary table, percents ready: {:.2f}".format(pr)
messenger(msg, "R")
return rows, header
[docs]def generate_csv(rows):
"""
:type rows: list
:param rows: list of lists to be written into csv
:rtype: generator
:return: generator yielding rows of csv file
"""
data = StringIO()
csvwriter = csv.writer(data)
for row in rows:
csvwriter.writerow(row)
yield data.getvalue()
data.seek(0)
data.truncate(0)
[docs]def csv_response(rows):
"""
:type rows: list
:param rows: list of lists to be written into csv
:rtype: :class:`~werkzeug.wrappers.Response`
:return: csv file as a html response
"""
headers = Headers()
headers.set('Content-Disposition', 'attachment', filename='summary.csv')
return Response(
stream_with_context(generate_csv(rows)),
mimetype='text/csv', headers=headers,
)
[docs]@app.route('/', defaults={'filename': ''})
@app.route('/<path:filename>')
@logged
def index_alias(filename):
"""
Index site of the polyfemos web.
In addition provides the documentation files.
"""
msg = "Documentation filename: {}".format(filename)
messenger(msg, "R")
if not filename:
return render_template('index.htm')
folder = userdef.paths("doc_dir")
return send_from_directory(folder, filename)
[docs]@app.route('/home', methods=['GET', 'POST'])
@logged
def home():
"""
Navigation site consisting of link library for every site
available in polyfemos web.
"""
from polyfemos.front.forms import SelectNetworkForm
form = SelectNetworkForm()
if not form.validate_on_submit():
form.network_code.data = userdef.get_network_code()
network_code = form.network_code.data
resp = render_base(render_template)('home.htm', form=form,
network_code=network_code)
resp = make_response(resp)
resp.set_cookie("network_code", value=network_code)
return resp
[docs]@app.route('/sohtable', methods=['GET', 'POST'])
@logged
@limit_access(access_level=3)
def sohtable():
"""
A state of health table with every station and parameter combination,
incorporating current alerts and alert history.
"""
from polyfemos.front import forms
importlib.reload(forms)
form = forms.SohTableForm()
if not form.validate_on_submit():
form.date.data = UTCDateTime().now().date
form.show_all.data = True
form.realtimeness_bool.data = True
form.realtimeness_limit.data = 120
realtimeness_limit = form.realtimeness_limit.data
realtimeness_bool = form.realtimeness_bool.data
realtimeness = None
if realtimeness_bool:
realtimeness = UTCDateTime() - 60 * realtimeness_limit
date = form.date.data
if form.submit_pd.data:
date += timedelta(days=1)
elif form.submit_sd.data:
date -= timedelta(days=1)
form.date.data = date
julday, year = get_jY(date)
visibilities = {1, 2} if form.show_all.data else {1}
sohpar_names = userdef.sohpars(visibilities=visibilities)
station_ids = userdef.station_ids()
fpf = userdef.filepathformats("alert")
sohdict = get_sohdict(station_ids, year, julday, fpf,
realtimeness=realtimeness)
header = ""
header = "{}.{}".format(year, julday)
return render_base(render_template)(
'sohtable.htm', alertdict=sohdict['alerts'], header=header,
station_ids=station_ids, sohpar_names=sohpar_names, form=form)
[docs]@app.route('/sohmap')
@logged
@limit_access(access_level=3)
def sohmap():
"""
A map of the network area with stations. Alerts are sorted by their
priorities. The innermost circle consists of the alerts of the highest
priority.
"""
alertcolors = {
0: colors.ALERT_GREEN,
1: colors.ALERT_YELLOW,
2: colors.ALERT_RED,
}
filename = userdef.paths("map_file")
filename, extension = os.path.splitext(filename)
imgfile = "web_static/{}{}".format(filename, extension)
mapfile = "web_static/{}{}".format(filename, ".map")
msg = "Img and map files: {}, {}".format(imgfile, mapfile)
messenger(msg, "R")
if not os.path.isfile(imgfile):
return render_base(render_template)('sohmap.htm')
if not os.path.isfile(mapfile):
return render_base(render_template)('sohmap.htm')
# get transformation from WGS84 to pixels
pixel_transform = coordinator.transform_from_ozi_map(mapfile)
# open background map
pil_image = Image.open(imgfile)
draw = ImageDraw.Draw(pil_image)
today = UTCDateTime().now()
julday, year = get_jY(today)
sohpar_names = userdef.sohpars()
station_ids = userdef.station_ids()
# get alerts and priorities with each station and parameter combination
fpf = userdef.filepathformats("alert")
sohdict = get_sohdict(station_ids, year, julday, fpf)
for station_id in station_ids:
sohplot = SOHPlot(
station_id=station_id,
headerdate=today,)
epsg = sohplot.header["EPSG"]
locx = sohplot.header["LOCX"]
locy = sohplot.header["LOCY"]
if epsg is None:
continue
# Convert stations coordinates into WGS84
transform = coordinator.get_transform(epsg, "4326")
px, py = pixel_transform(*transform(locx, locy))
def get_bbox(radius):
return (px - radius, py - radius, px + radius, py + radius)
# Alerts are stored in 3x3 matrix
# Priority in x axis
# Alert (red, yellow or green) in y axis
# Red alert with the highest priority is stored in [0,2]
alertcount = np.zeros((3, 3))
for sohpar_name in sohpar_names:
key = station_id + sohpar_name
if key not in sohdict['alerts']:
continue
alert = to.int_(sohdict['alerts'][key])
priority = to.int_(sohdict['priorities'][key])
if alert is None or priority is None or priority > 4:
continue
priority = min(3, priority) - 1
alertcount[alert, priority] += 1
# TODO clean plotting, own function?
# TODO Scalable map
radius = 21
for i in range(3)[::-1]:
bbox = get_bbox(radius)
draw.ellipse(bbox, fill=colors.BLACK)
radius -= 1
alerts = alertcount[:, i]
alertsum = np.sum(alerts)
bbox = get_bbox(radius)
if alertsum <= 0:
draw.pieslice(bbox, start=0, end=360, fill=colors.GREY_3)
else:
alerts *= 360. / alertsum
start = 0
for j in range(3):
alert = alerts[j]
color = alertcolors[j]
end = start + alert
draw.pieslice(bbox, start=start, end=end, fill=color)
start += alert
radius -= 5 - i
fontpath = userdef.paths("ttf_file")
font = None
if os.path.isfile(fontpath):
font = ImageFont.truetype(fontpath, 20)
str_ = "{} UTC".format(today.strftime("%Y-%m-%d %H:%M:%S"))
draw.text((10, 10), str_, font=font, fill=(0, 0, 0, 128))
max_height = 1500
width, height = pil_image.size
scale = int(max_height / height)
pil_image = pil_image.resize(
(scale * width, scale * height),
resample=Image.ANTIALIAS)
data = get_image_data(pil_image)
return render_base(render_template)('sohmap.htm', sohmapimg=data)
[docs]@app.route('/plotbrowser', methods=['GET', 'POST'])
@logged
@limit_access(access_level=3)
@limited(ipstorage)
def plotbrowser():
"""
Soh plotter site, Plot Browser.
"""
from polyfemos.front import forms
importlib.reload(forms)
form = forms.PlotbrowserForm()
today = UTCDateTime().now().date
full_access = check_permission(1)
message_lines = []
if not form.validate_on_submit():
form.station_ids.data = []
form.sohpar_names.data = []
form.startdate.data = today
form.enddate.data = today
form.headerdate.data = today
form.rirv.data = False
form.ridv.data = True
form.decimate.data = True
form.track_len.data = True
form.aor.data = 'null'
form.fromfileformat.data = "stf"
if request.arguments("b") == "1":
selected_date = UTCDateTime(request.arguments("date")).date
form.station_ids.data = [request.arguments("station_id")]
form.sohpar_names.data = [request.arguments("sohpar_name")]
form.startdate.data = selected_date
form.enddate.data = selected_date
station_ids = form.station_ids.data
sohpar_names = form.sohpar_names.data
startdate = form.startdate.data
enddate = form.enddate.data
headerdate = form.headerdate.data
remove_irrationals = form.rirv.data
remove_identicals = form.ridv.data
decimate = form.decimate.data
track_len = form.track_len.data
aor_selected = form.aor.data
fromfileformat = form.fromfileformat.data
aorkwargs = {
"null": [],
"dtr": [],
"sta": [],
"lip": [],
}
funcs = {
"dtr": [outlierremover.dtr, {}],
"sta": [outlierremover.stalta, {}],
"lip": [outlierremover.lipschitz, {}],
}
for field in form:
split = field.id.split("_")
if len(split) < 2:
continue
id_, kwarg = split
if id_ not in {"dtr", "sta", "lip"}:
continue
aorkwargs[id_].append(field)
funcs[id_][1][kwarg] = field.data
outlierremfunc = None
advanced_outlier_removal = True
if aor_selected in funcs:
aorfunc = funcs[aor_selected][0]
kwargs = funcs[aor_selected][1]
outlierremfunc = lambda data: aorfunc(data, **kwargs)
else:
advanced_outlier_removal = False
combinations = list(itertools.product(station_ids, sohpar_names))
if not full_access:
if not remove_identicals:
str_ = "You don't have permission to uncheck " \
+ "'Remove identical values'."
message_lines.append(str_)
remove_identicals = True
if not decimate:
str_ = "You don't have permission to uncheck 'Decimate'."
message_lines.append(str_)
decimate = True
if len(combinations) > 2:
str_ = "You may only select at most 2 stations or " \
+ "state of health parameters."
message_lines.append(str_)
combinations = combinations[:2]
if abs((startdate - enddate).days) > 35:
str_ = "You don't have permission to select timespan " \
+ "wider than 35 days."
message_lines.append(str_)
startdate = enddate
limit_date = today - timedelta(days=730)
if startdate < limit_date or enddate < limit_date:
str_ = "You don't have permission to view " \
+ "data before {}.".format(str(limit_date))
message_lines.append(str_)
startdate = today
enddate = today
plots = []
for station_id, sohpar_name in combinations:
# Contruct plot
sohplot = SOHPlot(
station_id=station_id,
sohpar_name=sohpar_name,
startdate=startdate,
enddate=enddate,
headerdate=headerdate,
remove_irrationals=remove_irrationals,
advanced_outlier_removal=advanced_outlier_removal,
outlierremfunc=outlierremfunc,
track_datalen=track_len,
remove_identicals=remove_identicals,
fext=fromfileformat)
plotdict = {}
stats_table = []
infolines = []
plotscript, plotdiv = sohplot.get_plot_components(decimate=decimate)
if plotscript:
stats_table = sohplot.get_statistics_table()
infolines = [line
.replace("*", "\xa0")
.replace(">", "\u2192")
for line in sohplot.get_info()]
plotdict['stats_table'] = stats_table
plotdict['plotscript'] = plotscript
plotdict['plotdiv'] = plotdiv
plotdict['infolines'] = infolines
plots.append(plotdict)
return render_base(render_template)(
"plotbrowser.htm", bokeh_version=str(bokeh.__version__),
form=form, plots=plots, aorkwargs=aorkwargs,
message_lines=message_lines)
[docs]@app.route('/datacoveragebrowser', methods=['GET', 'POST'])
@logged
@limit_access(access_level=2)
def datacoveragebrowser():
"""
Creates the datacoveragebrowser view and edits the link
which directs to the actual datacoverage image which is created
according to the given parameters
"""
from polyfemos.front import forms
importlib.reload(forms)
form = forms.DatacoverageForm()
if not form.validate_on_submit():
enddate = UTCDateTime().now()
startdate = enddate - 86400 * 30
form.station_ids.data = []
form.channel_codes.data = []
form.startdate.data = startdate.date
form.enddate.data = enddate.date
startdate = parse_date(form.startdate.data)
enddate = parse_date(form.enddate.data) + 86399
station_ids = form.station_ids.data
channel_codes = form.channel_codes.data
func_ = userdef.datacoveragebrowser_func()
fig = func_(station_ids, channel_codes, startdate,
enddate, userdef.filepathformats("rawdata"))
data = get_image_data(fig)
return render_base(render_template)(
"datacoveragebrowser.htm", form=form,
station_ids=station_ids, channel_codes=channel_codes, dcimage=data)
[docs]@app.route('/datacoverageimage')
@logged
@limit_access(access_level=3)
def datacoverageimage():
"""
A simple site including separately created datacoverage image.
"""
dci_file = os.path.join("web_static", userdef.paths("dci_file"))
if not os.path.isfile(dci_file):
return render_base(render_template)('datacoverageimage.htm')
data = get_image_data(Image.open(dci_file))
return render_base(render_template)('datacoverageimage.htm', dci_file=data)
[docs]@app.route('/summary', methods=['GET', 'POST'])
@logged
@limit_access(access_level=2)
@limited(ipstorage)
def summary():
"""
Creates a parameter summary table
"""
from polyfemos.front import forms
importlib.reload(forms)
form = forms.SummaryForm()
enddate = UTCDateTime().now()
startdate = enddate - 86400
t0 = time.time()
if not form.validate_on_submit():
form.station_ids.data = []
form.sohpar_names.data = []
form.startdate.data = startdate.date
form.enddate.data = enddate.date
form.headerdate.data = enddate.date
form.rirv.data = True
form.aor.data = False
form.csv_requested.data = False
form.fromfileformat.data = "csv"
station_ids = form.station_ids.data
sohpar_names = form.sohpar_names.data
startdate = form.startdate.data
enddate = form.enddate.data
headerdate = form.headerdate.data
remove_irrationals = form.rirv.data
advanced_outlier_removal = form.aor.data
csv_requested = form.csv_requested.data
fromfileformat = form.fromfileformat.data
combinations = list(itertools.product(station_ids, sohpar_names))
rows, header = get_summary(
startdate=startdate,
enddate=enddate,
headerdate=headerdate,
combinations=combinations,
remove_irrationals=remove_irrationals,
advanced_outlier_removal=advanced_outlier_removal,
fext=fromfileformat,
)
exectime = round((time.time() - t0) / 60., 4)
if csv_requested:
rows.insert(0, header)
return csv_response(rows)
return render_base(render_template)(
"summary.htm", header=header, rows=rows, form=form,
aorinfolines=userdef.summary_outlierremfunc_info(), exectime=exectime)
[docs]@app.route('/alertheat', methods=['GET', 'POST'])
@logged
@limit_access(access_level=2)
def alertheat():
"""
Creates a state of health table with every station and parameter
combination. Timespan selection available for time interval anaysis.
"""
from polyfemos.front import forms
importlib.reload(forms)
form = forms.AlertHeatForm()
enddate = UTCDateTime().now()
startdate = enddate - 86400 * 20
if not form.validate_on_submit():
form.startdate.data = startdate.date
form.enddate.data = enddate.date
form.log_color.data = False
form.points_per_thbb.data = 1
form.points_per_tib.data = 2
startdate = parse_date(form.startdate.data)
enddate = parse_date(form.enddate.data)
log_color = form.log_color.data
points = {
"0": 0,
"1": form.points_per_thbb.data,
"2": form.points_per_tib.data,
}
sohpar_names = userdef.sohpars(visibilities={1, 2})
station_ids = userdef.station_ids()
fpf = userdef.filepathformats("alert")
results = {}
while startdate <= enddate:
julday, year = get_jY(startdate)
sohdict = get_sohdict(station_ids, year, julday, fpf)
for k, v in sohdict['alerts'].items():
if k not in results:
results[k] = {'count': 0, 'max': 0}
if v in points:
results[k]['count'] += points[v]
results[k]['max'] += 2.
startdate += 86400
cmap = LinearSegmentedColormap.from_list("", [
colors.ALERT_GREEN,
colors.ALERT_YELLOW,
colors.ALERT_RED,
])
for k, v in results.items():
if v['max'] <= 0.:
v['color'] = colors.GREY_3
v['tooltip'] = "0 / 0\n0.0%"
continue
else:
percentage = v['count'] / v['max']
percents = round(100. * percentage, 2)
if log_color:
color = 255. * np.log(max(1, percents)) / np.log(100)
else:
color = 255. * percentage
color = int(round(color))
v['color'] = rgb2hex(cmap(color)[:3])
v['tooltip'] = "{} / {:.0f}\n{:.1f}%" \
.format(v['count'], v['max'], percents)
return render_base(render_template)(
'alertheat.htm', alertdict=results, station_ids=station_ids,
sohpar_names=sohpar_names, form=form)
# With debug=True, Flask server will auto-reload
# when there are code changes
if __name__ == '__main__':
host = "127.0.0.1"
app.run(host=host, port=5000, debug=False, threaded=True)