# Virus Health Check: a validation tool for HETDEX/VIRUS data
# Copyright (C) 2015, 2016, 2017 "The HETDEX collaboration"
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Create, deal with, render and save the html result file
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import configparser
import os
import re
import webbrowser
import jinja2
import pyhetdex.het.fplane as fp
from libvhc.config import get_config
[docs]class TestResults(object):
"""Simple container for the test result
Parameters
----------
message : string
string describing the test or the message
status : string or bool
whether the test was successful. Accepts ``'success'``, ``'fail'``,
``True`` or ``False``. Any thing else is interpreted as failure
Attributes
----------
message : string
see above
css_status : string
``'success'`` or ``'fail'``
bool_status : bool
``True`` for success, ``False`` for failure
"""
def __init__(self, message, status):
self._message = message
if isinstance(status, bool):
self._bool_status = status
self._css_status = self._bool2css(status)
else:
try:
self._bool_status = self._css2bool(status)
self._css_status = status
except ValueError:
self._bool_status = False
self._css_status = "fail"
@property
def message(self):
"""message or test description"""
return self._message
@property
def css_status(self):
"""string of the status"""
return self._css_status
@property
def bool_status(self):
"""bool of the status"""
return self._bool_status
[docs] def _bool2css(self, bsuccess):
"""convert bool to the string used by css
Parameters
----------
bsuccess : bool
whether it's successful or not
Returns
-------
string
``'success'`` or ``'fail'``
"""
if bsuccess:
return "success"
else:
return "fail"
[docs] def _css2bool(self, ssuccess):
"""convert css string to bool
Parameters
----------
ssuccess : bool
``'success'`` or ``'fail'``
Returns
-------
bool
whether it's a success or not
Raises
------
ValueError
if ``ssuccess`` is not ``'success'`` nor ``'fail'``
"""
if ssuccess == "success":
return True
elif ssuccess == "fail":
return False
else:
msg = "ssuccess must be 'success' or 'fail', not '{}'"
raise ValueError(msg.format(ssuccess))
[docs]class IFU(fp.IFU):
"""Contain the information for the IFU from the focal plane file and the
list of tests for the ifu
Parameters
----------
ifuslot : string
id of the ifu
x, y : float
x and y position of the ifu in the focal plane
specid : int
id of the spectrograph where the ifu is plugged into
specslot : int
id of the spectrograph slot where the spectrograph is plugged into
ifuid : string
id of the virus ifu bundle
ifurot : float
rotation of the IFU in its seat in the IHMP
platescl : float
focal plane plate scale at the position in the IHMP
Attributes
----------
ifuid, x, y, xid, yspecid : as before
xid, yid : string or int
x (column) and y (row) id of the ifu in the ifu head mounting plate
(IHMP), generated from the ifuslot
ifuslot : string
id of the IHMP seat address
success : string
tests : list of :class:`TestResults`
list of the tests run for the current ifu
"""
def __init__(self, *args):
super(IFU, self).__init__(*args)
self._success = 'fail'
self._tests_success = []
self._tests_fail = []
self._valid_success = ['success', 'fail', 'failsome', 'miss',
'successmiss', 'failmiss', 'failsomemiss',
'successextra', 'failextra', 'failsomeextra']
self._valid_success += [i + ' ignored' for i in self._valid_success]
@property
def success(self):
"""global success state. Must be one of::
['success', 'fail', 'failsome', 'miss', 'successmiss',
'failmiss', 'failsomemiss']
Set to 'unknown' when none of the above is set """
return self._success
@property
def tests(self):
"""Returns the test, with the failures first"""
return self._tests_fail + self._tests_success
@success.setter
def success(self, value):
"""global success state"""
if value in self._valid_success:
self._success = value
else:
self._success = 'unknown'
[docs] def rescale(self, xmin, ymin, xscale, yscale):
"""Rescale the ``x`` and ``y`` position of the IFU::
(x - xmin) / xscale
(y - ymin) / yscale
Parameters
----------
xmin, ymin : float
minimum ``x`` and ``y`` coordinate
xscale, yscale : float
divide the ``x`` and ``y`` by these values after subtracting the
minima
"""
self.x = (self.x - xmin) / float(xscale)
self.y = (self.y - ymin) / float(yscale)
[docs] def add_test(self, message, success):
"""Append a new :class:`TestResults`
Parameters
----------
message : string
message of the test
success : string or bool
success status of the test; must be one of the accepted values of
:class:`TestResults`
"""
tr = TestResults(message, success)
if tr.bool_status:
self._tests_success.append(tr)
else:
self._tests_fail.append(tr)
[docs]class FPlane(fp.FPlane):
"""Focal plane.
Contains the dictionary of :class:`IFU` instance, with the ifu id as key,
the list of test common to the focal plane and provide the tools to process
the ifus before rendering the html.
The focal plane is expected to be like::
# IFUSLOT X_FP Y_FP SPECID SPECSLOT IFUID IFUROT PLATESC
013 -450.0 150.0 00 000 000 0.0 1.0
014 -450.0 50.0 00 000 000 0.0 1.0
Parameters
----------
recipe : string
name of the current recipe
fplane_file : string
name of the file containing the ids and position of the IFUs
conf : :class:`confp.ConfigParser` instance
configuration option
Attributes
----------
difus
ids
ifus
tests
tests_orphan
recipe : string
name of the recipe
ntests : int
number of tests expected per IFU. It's used in :meth:`postprocess_ifus`
to determine whether all expected tests are present
template : jinja template
template to render
rendered : string
rendered jinja template
"""
def __init__(self, recipe, fplane_file):
conf = get_config()
# get the IFUSLOT to skip
try:
skip_ifuslot_html = conf.get_list('fplane', 'skip_ifuslot_html')
except configparser.NoOptionError:
skip_ifuslot_html = conf.get_list('fplane', 'skip_ifuslot',
use_default=True)
# then create the fplane file
super(FPlane, self).__init__(fplane_file, ifu_class=IFU,
exclude_ifuslot=skip_ifuslot_html)
self.recipe = recipe
self._tests_success = []
self._tests_fail = []
self._tests_orphan_success = []
self._tests_orphan_fail = []
self.ntests = 0
self._template = None
self.rendered = None
self.config_info = None
self.ignore_ifuslot_html = conf.get_list('fplane',
'ignore_ifuslot_html',
use_default=True)
@property
def tests(self):
"""List of :class:`TestResults`: tests run for whole focal plane
with failures first"""
return self._tests_fail + self._tests_success
@property
def tests_orphan(self):
"""List of :class:`TestResults`: orphaned tests
with failures first.
They should belong to one IFU, but its ID is not known.
"""
return self._tests_orphan_fail + self._tests_orphan_success
[docs] def ifu_rescale(self, xmin, ymin, xscale, yscale):
"""Rescale the ``x`` and ``y`` position of all the ifus. See
:func:`IFU.rescale` for further details
"""
for ifu in self.ifus:
ifu.rescale(xmin, ymin, xscale, yscale)
[docs] def add_test(self, message, success):
"""Append a new :class:`TestResults` to the focal plane tests
Parameters
----------
message : string
message of the test
success : string or bool
success status of the test; must be one of the accepted values of
:class:`TestResults`
"""
tr = TestResults(message, success)
if tr.bool_status:
self._tests_success.append(tr)
else:
self._tests_fail.append(tr)
[docs] def add_orphan_test(self, message, success):
"""Append a new :class:`TestResults` to the orphan tests
Parameters
----------
message : string
message of the test
success : string or bool
success status of the test; must be one of the accepted values of
:class:`TestResults`
"""
tr = TestResults(message, success)
if tr.bool_status:
self._tests_orphan_success.append(tr)
else:
self._tests_orphan_fail.append(tr)
[docs] def add_ifu_test(self, ifuslot, message, success):
"""Add to the ifu identified by ``ifuid``
Parameters
----------
ifuslot : int
id of the ifu slot, if the IFU does not exists, add the message to
the global messages marking it as a failure
message : string
message of the test
success : string or bool
success status of the test; must be one of the accepted values of
:class:`TestResults`
"""
try:
self.by_ifuslot(ifuslot).add_test(message, success)
except KeyError:
msg = "IFU {id_} does not exist. ".format(id_=ifuslot) + message
self.add_orphan_test(msg, success)
[docs] def add_ntests(self, new_tests):
"""Add ``new_tests`` to the number of tests
Parameters
----------
new_tests : int
number on tests to add
"""
self.ntests += new_tests
[docs] def postprocess_ifus(self):
"""Post process the ifu, to check if all the expected checks are
present and set the :attr:`IFU.success` correspondingly
"""
for k, v in self.difus_ifuslot.items():
statuses = [t.bool_status for t in v.tests]
if len(statuses) == 0:
v.success = 'miss'
elif len(statuses) == self.ntests:
if all(statuses):
v.success = 'success'
elif not any(statuses):
v.success = 'fail'
else:
v.success = 'failsome'
elif len(statuses) < self.ntests:
if all(statuses):
v.success = 'successmiss'
elif not any(statuses):
v.success = 'failmiss'
else:
v.success = 'failsomemiss'
else:
if all(statuses):
v.success = 'successextra'
elif not any(statuses):
v.success = 'failextra'
else:
v.success = 'failsomeextra'
if k in self.ignore_ifuslot_html:
v.success += " ignored"
[docs] def load_template(self):
'''Load the default vhc template and save it :attr:`template`
property'''
self.template = get_template()
@property
def template(self):
"""Jinja template"""
return self._template
@template.setter
def template(self, template):
"Set the jinja template"
self._template = template
[docs] def render(self, path, logs, v_results, driver):
"""Render the template.
Parameters
----------
path : string
directory in which vhc has run
logs, v_results, driver : strings
name of the logs, the result and the driver files
"""
# create the entries for the first two lines of the legend
legends = [TestResults("this is a successful test", "success"),
TestResults("this is a failed test", "fail")]
with open(logs) as f:
logf = _log_to_list(f)
with open(v_results) as f:
resf = _log_to_list(f)
with open(driver) as f:
driverf = f.readlines()
self.rendered = self.template.render(fplane=self, legends=legends,
path=path, v_results=resf,
logs=logf, drivers=driverf)
[docs] def write_recap(self, fname, auto_open=False):
"""Write the rendered template into file ``fname``
Parameters
----------
fname : string
name of the file where to save the rendered recap
auto_open : bool
if true, auto open the rendered html
"""
with open(fname, 'w') as f:
f.write(self.rendered)
if auto_open:
webbrowser.open(fname)
[docs]def _log_to_list(lines):
'''Loop through the lines and collect all log lines into a list collecting
multi-lines into one entry.
Empty lines are ignored and new lines are stripped.
.. warning::
adapt the regex pattern if the log format is changed in
:func:`libvhc.loggers.set_logger`
Parameters
----------
lines : iterable
lines to parser
Returns
-------
logs : list
list of log messages
'''
pattern = r'\[.*?,.*?\].*? (DEBUG|INFO|WARNING|ERROR|CRITICAL):'
p = re.compile(pattern=pattern)
logs = []
for line in lines:
line = line.strip('\n')
if not line.strip():
continue
elif p.match(line) is None:
logs[-1] += "\n" + line
else:
logs.append(line)
return logs
[docs]def get_template():
"""Returns the Jinja template used by vhc"""
# create the jinja environment and get the template
curdir = os.path.dirname(os.path.abspath(__file__))
loader = jinja2.FileSystemLoader(os.path.join(curdir, "templates"))
env = jinja2.Environment(loader=loader, trim_blocks=True,
lstrip_blocks=True)
template = env.get_or_select_template("layout.html")
return template