From d9c5387a48e13c720c149347b3a2c66c6ee258a3 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 19 Dec 2023 09:38:39 +0100 Subject: [PATCH 01/11] bump all product versions to 2 --- stixcore/products/product.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 23fead36..b0183de1 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -117,7 +117,7 @@ class BaseProduct: Base TMProduct that all other product inherit from contains the registry for the factory pattern """ - PRODUCT_PROCESSING_VERSION = 1 + PRODUCT_PROCESSING_VERSION = 2 _registry = {} From 2d810addea1c2b8dd4ca4e1da37767a80006ca52 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 19 Dec 2023 09:38:39 +0100 Subject: [PATCH 02/11] bump all product versions to 2 Add support for trigger scaling in compression mode S=0,k=0,M=7 (#377) * Unscale data when compressed with mode S=0,k=0,M=7 * QL - LC, BKG, SPECTRA * BSD - CPD, SCPD, VIS and SPEC bump all product versions to 2 (#378) * bump all product versions to 2 * roleout IDB version 2.26.27 * fix test with new version number in mock object * fix test * fix for latest IDB as fallback Fix duplicated hk time bin bug Ddpdupdates2 (#380) * bump all product versions to 2 * prep for L2 ddpd * integrate IDB release 2.26.38 * update for common CONF submodule * fir text for latest IDB to 2.26.38 Config fix (#381) * bump all product versions to 2 * fix module path Fix bugs found when processing new scaled trigger data. (#382) * Fix bug processing new scaled trigger data. * Allow for QL file to unscale by packet * Fix bug that create extra (replica) triggers in Spectrogram Publish_L2 / RID LUT Manager (#385) * bump all product versions to 2 * RID LUT data manager update and use in processing for trigger descaling * Update stixcore/io/RidLutManager.py * fixed API mock via static files on pup99 --------- Co-authored-by: Shane Maloney Add warning if trigger scaling changes in request (#388) End2end rid lut manager (#390) * bump all product versions to 2 * add copy and parent option * fix missing init of singleton bump all product versions to 2 (#389) Bump conf version (#391) * bump all product versions to 2 * bump common version Flexible way to fill DATAMAX/MIN BUNIT EXPOSURE in FITS header keywords (#393) * bump all product versions to 2 * add properties for dedicated fits header keywords with class inhe. / override * add test and minor fixes after review * fix XPOSURE/XPOMAX header error * fix tests for dmax... * relax end2end fitsdiff * less tolerance in end2end test Solo anc stix ephemeris (#399) * bump all product versions to 2 * add more FITS header keywords * fix for ANC also get fits updated headers while publishing Upgrade_watchdog (#401) * bump all product versions to 2 * bump version * fix version label * pin common CONF version for end2end test * ignore VERS_CFG in end2end testing * add test for TM folder observing * bump watchdog==6.0.0 * integrate review fixes Update docks (#409) * bump all product versions to 2 * update documentation * add doc for end to end tetsing * fix format * add doc for manual (re)processing of fits products E2e fix sym add (#412) * bump all product versions to 2 * dep np.bool8 * disable test Get rid reason/scaling lookup error handling (#411) * bump all product versions to 2 * add global error detection in pipeline logger (not just main process) * add exception handling for get_reason daily processing pipeline (#405) * bump all product versions to 2 * initial * daily processing task * add singleStep * move ANC-aspect from pipeline to daily_pipeline * removed requirement for the moment * fix #283 and option to process only latest fits file versions * update ddpd scripts * fix circular imports * test git dep * skio test on win * skip test on win/mac * Update stixcore/io/FlareListManager.py Co-authored-by: Shane Maloney * fix format error --------- Co-authored-by: Shane Maloney Add Zenodo file (#413) * Add Zenodo author information Update high resolution transmission to match IDL defaults (#414) Pin major and minor versions of some packages (#415) V1.5.1_fixes (#420) * bump all product versions to 2 * fix #417 * fix #418 * daily report log files could have different names if the scripts run over multiple days * fix #419 * end2end test: better report if no corresponding file found * rewrite pipeline to cronjob based approach #422 * cleanup before merge Fix bug in spectrogram time bin calculations (#423) * Only use the closing time offset (NIX00269) for the last science substrure Update .zenodo.json Maintenance update project and dependencies (#428) * Update project use project.toml and move config into own files * Fix most of the tests * ruff-check, ruff-format and codespell * Some spelling mistakes left need to check the meaning * More updates and fixes * Few more small fixes * Fix doc build * Revert unintentional spelling correction in IDB sql script. * Update tox test dep name * Add version.py to git * Skip unclear test add skipped test for later * Update workflow python version * Add remote data to github CI bump all product versions to 2 Add support for trigger scaling in compression mode S=0,k=0,M=7 (#377) * Unscale data when compressed with mode S=0,k=0,M=7 * QL - LC, BKG, SPECTRA * BSD - CPD, SCPD, VIS and SPEC bump all product versions to 2 (#378) * bump all product versions to 2 * roleout IDB version 2.26.27 * fix test with new version number in mock object * fix test * fix for latest IDB as fallback Fix duplicated hk time bin bug Ddpdupdates2 (#380) * bump all product versions to 2 * prep for L2 ddpd * integrate IDB release 2.26.38 * update for common CONF submodule * fir text for latest IDB to 2.26.38 Config fix (#381) * bump all product versions to 2 * fix module path Fix bugs found when processing new scaled trigger data. (#382) * Fix bug processing new scaled trigger data. * Allow for QL file to unscale by packet * Fix bug that create extra (replica) triggers in Spectrogram Publish_L2 / RID LUT Manager (#385) * bump all product versions to 2 * RID LUT data manager update and use in processing for trigger descaling * Update stixcore/io/RidLutManager.py * fixed API mock via static files on pup99 --------- Co-authored-by: Shane Maloney Add warning if trigger scaling changes in request (#388) End2end rid lut manager (#390) * bump all product versions to 2 * add copy and parent option * fix missing init of singleton bump all product versions to 2 (#389) Bump conf version (#391) * bump all product versions to 2 * bump common version Flexible way to fill DATAMAX/MIN BUNIT EXPOSURE in FITS header keywords (#393) * bump all product versions to 2 * add properties for dedicated fits header keywords with class inhe. / override * add test and minor fixes after review * fix XPOSURE/XPOMAX header error * fix tests for dmax... * relax end2end fitsdiff * less tolerance in end2end test Solo anc stix ephemeris (#399) * bump all product versions to 2 * add more FITS header keywords * fix for ANC also get fits updated headers while publishing Upgrade_watchdog (#401) * bump all product versions to 2 * bump version * fix version label * pin common CONF version for end2end test * ignore VERS_CFG in end2end testing * add test for TM folder observing * bump watchdog==6.0.0 * integrate review fixes Update docks (#409) * bump all product versions to 2 * update documentation * add doc for end to end tetsing * fix format * add doc for manual (re)processing of fits products E2e fix sym add (#412) * bump all product versions to 2 * dep np.bool8 * disable test Get rid reason/scaling lookup error handling (#411) * bump all product versions to 2 * add global error detection in pipeline logger (not just main process) * add exception handling for get_reason daily processing pipeline (#405) * bump all product versions to 2 * initial * daily processing task * add singleStep * move ANC-aspect from pipeline to daily_pipeline * removed requirement for the moment * fix #283 and option to process only latest fits file versions * update ddpd scripts * fix circular imports * test git dep * skio test on win * skip test on win/mac * Update stixcore/io/FlareListManager.py Co-authored-by: Shane Maloney * fix format error --------- Co-authored-by: Shane Maloney Add Zenodo file (#413) * Add Zenodo author information Update high resolution transmission to match IDL defaults (#414) Pin major and minor versions of some packages (#415) fix #417 fix #418 daily report log files could have different names if the scripts run over multiple days fix #419 end2end test: better report if no corresponding file found rewrite pipeline to cronjob based approach #422 --- docs/pipelineconfiguration.rst | 19 +++++++++++++++++++ stixcore/products/product.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/pipelineconfiguration.rst b/docs/pipelineconfiguration.rst index e9a4c624..86006101 100644 --- a/docs/pipelineconfiguration.rst +++ b/docs/pipelineconfiguration.rst @@ -75,6 +75,25 @@ Sync FITS products with Platform (stix data center) # sync current fits to NAS (connection to Platform) 0 */6 * * * sudo rsync -avLo --delete /var/www/data/fits/ /mnt/nas05/stix/stixcore_fits/ +Sync STIX-CONF repo in all used instances +***************************************** + +``` +# sync the STIX-CONF to the latest release update version number +# in the publix www folder +10 */4 * * * cd /var/www/data/STIX-CONF && git fetch --tags && latestTag=$(git describe --tags `git rev-list --tags --max-count=1`) && git checkout $latestTag; echo $latestTag > VERSION.TXT +# but also in the Pipeline setup once a day +# had to be copied into the venv folder as well to get activated +20 22 * * * cd /home/stixcore/STIXCore/stixcore/config/data/common/ && git fetch --tags && latestTag=$(git describe --tags `git rev-list --tags --max-count=1`) && git checkout $latestTag; echo $latestTag > VERSION.TXT && cp -r /home/stixcore/STIXCore/stixcore/config/data/common /home/stixcore/STIXCore/venv/lib/python3.9/site-packages/stixcore/config/data/ +``` + +Sync FITS products with Platform (stix data center) +*************************************************** + +``` +# sync current fits to NAS (connection to Platform) +0 */6 * * * sudo rsync -avLo --delete /var/www/data/fits/ /mnt/nas05/stix/stixcore_fits/ +``` Sync SPICE kernels and SOOP data ******************************** diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 67ab6c98..265f3d00 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -550,7 +550,7 @@ def max_exposure(self): # default for FITS HEADER return 0.0 - def find_parent_products(self, root): + def find_parent_products(self, root): """ Convenient way to get access to the parent products. From ca4f7012b85f44870b10f526a16597e84d8a1072 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 11 Apr 2025 14:15:58 +0200 Subject: [PATCH 03/11] flare list, flare list with position, flare list with soop, flare list preview maps --- stixcore/data/stixcore.ini | 4 + stixcore/io/FlareListManager.py | 161 +++++- stixcore/io/fits/processors.py | 322 +++++++++-- stixcore/processing/FLtoFL.py | 210 ++++++++ stixcore/processing/FLtoL3.py | 92 ---- stixcore/processing/FlareListANC.py | 145 +++++ stixcore/processing/LL.py | 172 ++++++ stixcore/processing/SingleStep.py | 87 --- stixcore/processing/pipeline_daily.py | 120 +++-- stixcore/processing/publish.py | 53 +- stixcore/products/ANC/aspect.py | 20 +- stixcore/products/ANC/flarelist.py | 528 +++++++++++++++++++ stixcore/products/level3/flarelistproduct.py | 53 +- stixcore/products/lowlatency/quicklookLL.py | 40 +- stixcore/products/product.py | 61 ++- stixcore/util/util.py | 9 + 16 files changed, 1748 insertions(+), 329 deletions(-) create mode 100644 stixcore/processing/FLtoFL.py delete mode 100644 stixcore/processing/FLtoL3.py create mode 100644 stixcore/processing/FlareListANC.py create mode 100644 stixcore/processing/LL.py create mode 100644 stixcore/products/ANC/flarelist.py diff --git a/stixcore/data/stixcore.ini b/stixcore/data/stixcore.ini index 7f2fdf50..ad37b228 100644 --- a/stixcore/data/stixcore.ini +++ b/stixcore/data/stixcore.ini @@ -3,6 +3,7 @@ tm_archive = fits_archive = /data/stix/out/test/pipeline_auto_test spice_kernels = soop_files = +fido_search_url = file:///data/stix/out/fits_v1.2.0 [Logging] stop_on_error = False [IDLBridge] @@ -13,6 +14,7 @@ status_server_port = 12345 parallel_batchsize_L0 = 300 parallel_batchsize_L1 = 300 parallel_batchsize_L2 = 100 +flareid_sdc_lut_file = ./stixcore/data/publish/flarelist_sdc_lut.csv [Publish] db_file = ./stixcore/data/publish/published.sqlite include_levels = L0 @@ -29,3 +31,5 @@ endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api user = smaloney password = set_in_user_ini soop_files_download = ./stixcore/data/soop +[Processing] +flarelist_sdc_min_count = 1000 diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index c64c3486..60e05e3c 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -2,11 +2,21 @@ import time from datetime import datetime, timedelta +import numpy as np import pandas as pd from stixdcpy.net import Request as stixdcpy_req +from sunpy.net import attrs as a +import astropy.units as u +from astropy.table import Column, QTable, vstack +from astropy.time import Time + +from stixcore.config.config import CONFIG +from stixcore.products.ANC.flarelist import FlarelistSDC +from stixcore.products.product import Product from stixcore.util.logging import get_logger from stixcore.util.singleton import Singleton +from stixcore.util.util import url_to_path __all__ = ["FlareListManager", "SDCFlareListManager"] @@ -26,6 +36,10 @@ def flarelist(self, value): def flarelistname(self): return type(self).__name__ + @property + def productCls(self): + return self._product_cls + class SDCFlareListManager(FlareListManager, metaclass=Singleton): """Manages a local copy of the operational flarelist provided by stix data datacenter @@ -34,7 +48,8 @@ class SDCFlareListManager(FlareListManager, metaclass=Singleton): """ def __init__(self, file, update=False): - """Creates the manager by pointing to the flarelist file (csv) and setting the update strategy. + """Creates the manager by pointing to the flarelist file (csv) and setting the update + strategy. Parameters ---------- @@ -46,6 +61,7 @@ def __init__(self, file, update=False): self.file = file self.update = update self._flarelist = SDCFlareListManager.read_flarelist(self.file, self.update) + self._product_cls = FlarelistSDC def __str__(self) -> str: return f"{self.flarelistname}: file: {self.file} update: {self.update} size: {len(self.flarelist)}" @@ -84,13 +100,14 @@ def read_flarelist(cls, file, update=False): today = datetime.now() # - timedelta(days=60) flare_df_lists = [] if file.exists(): - old_list = pd.read_csv(file) - mds = old_list["start_UTC"].max() + old_list = pd.read_csv(file, keep_default_na=True, na_values=['None']) + mds = old_list['start_UTC'].max() try: last_date = datetime.strptime(mds, "%Y-%m-%dT%H:%M:%S") except ValueError: last_date = datetime.strptime(mds, "%Y-%m-%dT%H:%M:%S.%f") flare_df_lists = [old_list] + last_date -= timedelta(days=60) if not file.parent.exists(): logger.info(f"path not found to flare list file dir: {file.parent} creating dir") file.parent.mkdir(parents=True, exist_ok=True) @@ -118,10 +135,146 @@ def read_flarelist(cls, file, update=False): full_flare_list.to_csv(file, index_label=False) else: logger.info(f"read flare list from {file}") - full_flare_list = pd.read_csv(file) + full_flare_list = pd.read_csv(file, keep_default_na=True, na_values=['None']) return full_flare_list + @staticmethod + def filter_flare_function(col): + return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', + fallback=1000) + + def get_data(self, *, start, end, fido_client): + month_data = self.flarelist[(self.flarelist['start_UTC'] >= start.isoformat()) & + (self.flarelist['start_UTC'] < end.isoformat())] + + if len(month_data) == 0: + return None, None, None + + mt = QTable(month_data.to_numpy(), names=month_data.columns) + data = QTable() + control = QTable() + energy = QTable() + + data['flare_id'] = Column(mt['flare_id'].astype(int), + description=f'unique flare id for flarelist {self.flarelistname}') + data['start_UTC'] = Column(0, description="start time of flare") + data['start_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['start_UTC']] + data['duration'] = Column(mt['duration'].astype(float) * u.s, + description="duration of flare") + data['end_UTC'] = Column(0, description="end time of flare") + data['end_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['end_UTC']] + data['peak_UTC'] = Column(0, description="flare peak time") + data['peak_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['peak_UTC']] + data['att_in'] = Column(mt['att_in'].astype(bool), + description="was attenuator in during flare") + data['bkg_baseline'] = Column(mt['LC0_BKG'] * u.ct, + description='background baseline at 4-10 keV') + data['GOES_class'] = Column(mt['GOES_class'].astype(str), + description="GOES class of the GOES XRS data at time of flare" + " - not derived from STIX data. Do not use when " + "flare isn't visible to Earth") + data['goes_min_class_est'] = Column(mt['goes_estimated_min_class'].astype(str), + description="min GOES class estimate derived " + "from STIX data") + data['goes_max_class_est'] = Column(mt['goes_estimated_max_class'].astype(str), + description="max GOES class estimate derived " + "from STIX data") + data['goes_mean_class_est'] = Column(mt['goes_estimated_mean_class'].astype(str), + description="mean GOES class estimate derived " + "from STIX data") + + data['GOES_flux'] = Column(mt['GOES_flux'].astype(float) * u.W / u.m**2, + description="GOES flux of the GOES XRS data at time of flare" + "- not derived from STIX data. Do not use when the " + "flare isn't visible to Earth") + data['goes_min_flux_est'] = Column(mt['goes_estimated_min_flux'].astype(float) * + u.W / u.m**2, + description="min GOES flux estimate " + "derived from STIX data") + data['goes_max_flux_est'] = Column(mt['goes_estimated_max_flux'].astype(float) * + u.W / u.m**2, + description="max GOES flux estimate " + "derived from STIX data") + data['goes_mean_flux_est'] = Column(mt['goes_estimated_mean_flux'].astype(float) * + u.W / u.m**2, + description="mean GOES flux estimate " + "derived from STIX data") + + # data['cfl_x'] = Column(mt['CFL_X_arcsec'].astype(float) * u.arcsec, + # description="corse flare location in x direction provided by" + # "onboard algorithm. (0,0) represents disk center") + # data['cfl_y'] = Column(mt['CFL_Y_arcsec'].astype(float) * u.arcsec, + # description="corse flare location in y direction provided by" + # "onboard algorithm. (0,0) represents disk center") + + data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, + mt['LC1_PEAK_COUNTS_4S'].value, + mt['LC2_PEAK_COUNTS_4S'].value, + mt['LC3_PEAK_COUNTS_4S'].value, + mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), + description="counts in 4s peak window from quicklook lightcurve", + dtype=np.int64) + + data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, + mt['LC1_PEAK_COUNTS_4S'].value, + mt['LC2_PEAK_COUNTS_4S'].value, + mt['LC3_PEAK_COUNTS_4S'].value, + mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), + description="background counts in 4s peak window" + "from quicklook lightcurve", + dtype=np.int64) + + data['energy_index'] = Column(0, description="energy band index", dtype=np.int8) + + data.add_index('flare_id') + + # add energy axis for the lightcurve peek time data for each flare + # the energy bins are taken from the daily ql-lightcurve products + # as the definition of the lc energy chanel's are will change only very seldom + # the ql-lightcurve products assume a constant definition for an entire day. + # So we do the lookup also just grouped by peak day in order to save file lookups + + energy_look_up = {} + data['peak_day'] = [d.datetime.day for d in data['peak_UTC']] + data_by_day = data.group_by('peak_day') + + for day, flares in zip(data_by_day.groups.keys, data_by_day.groups): + time = flares['peak_UTC'][0] + lc_data = fido_client.search(a.Time(time, time), + a.Instrument.stix, + a.stix.DataProduct.ql_lightcurve) + lc_data.filter_for_latest_version() + url_to_path(lc_data) + + if len(lc_data) == 0: + logger.warning(f'No lightcurve data found for flare at time {time}') + continue + lc = Product(lc_data['path'][0]) + + energy_table_hash = frozenset(pd.core.util.hashing.hash_array(lc.energies.as_array())) + + # add the energy table to the energy table list if not already + # present and define a new index number + if energy_table_hash not in energy_look_up: + e_idx = len(energy_look_up.keys()) + energy_look_up[energy_table_hash] = e_idx + lc.energies["index"] = Column(e_idx, description='energy edge table index', + dtype=np.int8) + energy = vstack([energy, lc.energies]) + if e_idx > 0: + logger.warning(f'multiple energy ql-lc tables found for month {start}') + + # add the energy index to the flare data to all flares of the same day + # https://docs.astropy.org/en/latest/table/modify_table.html#caveats + replace = data.loc[flares['flare_id']] + replace['energy_index'] = energy_look_up[energy_table_hash] + data.loc[flares['flare_id']] = replace + + del data['peak_day'] + + return data, control, energy + if "pytest" in sys.modules: # only set the global in test scenario diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 6ef83c24..01e17ec0 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -1,9 +1,11 @@ """Module for the different processing levels.""" from math import isnan +from pathlib import Path from datetime import datetime import numpy as np +from matplotlib import pyplot as plt import astropy.units as u from astropy.io import fits @@ -19,7 +21,8 @@ from stixcore.util.logging import get_logger from stixcore.util.util import get_complete_file_name_and_path -__all__ = ["SEC_IN_DAY", "FitsProcessor", "FitsLBProcessor", "FitsL0Processor", "FitsL1Processor", "FitsL2Processor"] +__all__ = ['SEC_IN_DAY', 'FitsProcessor', 'FitsLBProcessor', 'FitsL0Processor', + 'FitsL1Processor', 'FitsL2Processor', 'PlotProcessor'] logger = get_logger(__name__) @@ -95,13 +98,14 @@ def generate_filename(cls, product, *, version, date_range, status="", header=Tr The filename """ - user_req = "" - if "request_id" in product.control.colnames: + user_req = '' + if product.control and 'request_id' in product.control.colnames: user_req = f"_{product.control['request_id'][0]:010d}" - tc_control = "" - if "tc_packet_seq_control" in product.control.colnames and user_req != "": - tc_control = f"-{product.control['tc_packet_seq_control'][0]:05d}" + tc_control = '' + if product.control and 'tc_packet_seq_control' in product.control.colnames \ + and user_req != '': + tc_control = f'-{product.control["tc_packet_seq_control"][0]:05d}' if status == "" and (not header) and (product.level != "LB" and product.fits_daily_file is True): status = "U" @@ -744,12 +748,11 @@ def __init__(self, archive_path): self.archive_path = archive_path @classmethod - def generate_filename(cls, product, *, version=0, status="", header=True): - date_range = ( - f"{product.utc_timerange.start.strftime('%Y%m%dT%H%M%S')}-" - + f"{product.utc_timerange.end.strftime('%Y%m%dT%H%M%S')}" - ) - if product.type != "sci" or product.name == "burst-aspect": + def generate_filename(cls, product, *, version=0, status='', header=True): + + date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT%H%M%S")}-' +\ + f'{product.utc_timerange.end.strftime("%Y%m%dT%H%M%S")}' + if product.type not in ['sci', 'flarelist'] or product.name == 'burst-aspect': date_range = product.utc_timerange.center.strftime("%Y%m%d") return FitsProcessor.generate_filename( @@ -766,8 +769,8 @@ def generate_primary_header(self, filename, product, *, version=0): ("DATAMIN", empty_if_nan(product.dmin), "Minimum valid physical value"), ("DATAMAX", empty_if_nan(product.dmax), "Maximum valid physical value"), ("BUNIT", product.bunit, "Units of physical value, after application of BSCALE, BZERO"), - ("XPOSURE", product.exposure, "[s] shortest exposure time"), - ("XPOMAX", product.max_exposure, "[s] maximum exposure time"), + ("XPOSURE", empty_if_nan(product.exposure), "[s] shortest exposure time"), + ("XPOMAX", empty_if_nan(product.max_exposure), "[s] maximum exposure time"), ) soop_keywords = SOOPManager.instance.get_keywords( @@ -837,7 +840,7 @@ def write_fits(self, product, *, version=0): # day boundary parts = [prod.level, prod.utc_timerange.center.strftime("%Y/%m/%d"), prod.type.upper()] # for science data use start date - if prod.type == "sci": + if prod.type in ["sci", 'flarelist']: parts[1] = prod.utc_timerange.start.strftime("%Y/%m/%d") path = self.archive_path.joinpath(*[str(x) for x in parts]) path.mkdir(parents=True, exist_ok=True) @@ -866,15 +869,8 @@ def write_fits(self, product, *, version=0): if col.endswith("_comp_err"): data[col].description = "Error due only to integer compression" - idb_versions = QTable( - rows=[ - (version, range.start.as_float(), range.end.as_float()) - for version, range in prod.idb_versions.items() - ], - names=["version", "obt_start", "obt_end"], - ) - - primary_header, header_override = self.generate_primary_header(filename, prod, version=version) + primary_header, header_override = self.generate_primary_header(filename, prod, + version=version) primary_hdu = fits.PrimaryHDU() primary_hdu.header.update(primary_header) primary_hdu.header.update(header_override) @@ -914,12 +910,16 @@ def write_fits(self, product, *, version=0): data_hdu = add_default_tuint(data_hdu) data_hdu.name = "DATA" + hdul = [primary_hdu, control_hdu, data_hdu] + + idb_versions = QTable(rows=[(version, range.start.as_float(), range.end.as_float()) + for version, range in prod.idb_versions.items()], + names=["version", "obt_start", "obt_end"]) idb_enc = fits.connect._encode_mixins(idb_versions) idb_hdu = table_to_hdu(idb_enc) idb_hdu = add_default_tuint(idb_hdu) idb_hdu.name = "IDB_VERSIONS" - - hdul = [primary_hdu, control_hdu, data_hdu, idb_hdu] + hdul.append(idb_hdu) FitsL0Processor.add_optional_energy_table(prod, hdul) @@ -984,24 +984,156 @@ def generate_primary_header(self, filename, product, *, version=0): ("DATAMIN", empty_if_nan(product.dmin), "Minimum valid physical value"), ("DATAMAX", empty_if_nan(product.dmax), "Maximum valid physical value"), ("BUNIT", product.bunit, "Units of physical value, after application of BSCALE, BZERO"), - ("XPOSURE", product.exposure, "[s] shortest exposure time"), - ("XPOMAX", product.max_exposure, "[s] maximum exposure time"), + ("XPOSURE", empty_if_nan(product.exposure), "[s] shortest exposure time"), + ("XPOMAX", empty_if_nan(product.max_exposure), "[s] maximum exposure time"), ) return L1headers, L2headers -class FitsL3Processor(FitsL2Processor): +class FitsANCProcessor(FitsL2Processor): def __init__(self, archive_path): super().__init__(archive_path) - def write_fits(self, product, *, version=0): + def write_fits(self, prod, *, version=0): + """Writes ANC products into fits files. + + Parameters + ---------- + product : `stixcore.product.ANC` + + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + + Returns + ------- + list + of created file as `pathlib.Path` + + """ + if version == 0: + version = prod.get_processing_version() + + filename = self.generate_filename(product=prod, version=version, header=False) + + parts = [prod.level, prod.utc_timerange.center.strftime('%Y/%m/%d'), prod.type.upper()] + # for science data use start date + if prod.type in ['flarelist']: + parts[1] = prod.utc_timerange.start.strftime('%Y/%m/%d') + path = self.archive_path.joinpath(*[str(x) for x in parts]) + path.mkdir(parents=True, exist_ok=True) + + fitspath = path / filename + fitspath_complete = get_complete_file_name_and_path(fitspath) + + if fitspath.exists(): + logger.warning('Fits file %s exists data will be overriden', fitspath.name) + elif fitspath_complete.exists(): + logger.warning('Complete Fits file %s exists will be overriden', fitspath.name) + + data = prod.data + + primary_header, header_override = self.generate_primary_header(filename, prod, + version=version) + primary_hdu = fits.PrimaryHDU() + primary_hdu.header.update(primary_header) + primary_hdu.header.update(header_override) + primary_hdu.header.update(prod.get_additional_header_keywords()) + + # Add comment and history + [primary_hdu.header.add_comment(com) for com in prod.comment] + [primary_hdu.header.add_history(com) for com in prod.history] + primary_hdu.header.update({'HISTORY': 'Processed by STIXCore ANC'}) + + data_enc = fits.connect._encode_mixins(data) + data_hdu = table_to_hdu(data_enc) + data_hdu = set_bscale_unsigned(data_hdu) + data_hdu = add_default_tuint(data_hdu) + data_hdu.name = "DATA" + + hdul = [primary_hdu, data_hdu] + + if prod.energy and len(prod.energy) > 0: + energy_enc = fits.connect._encode_mixins(prod.energy) + energy_hdu = table_to_hdu(energy_enc) + energy_hdu = add_default_tuint(energy_hdu) + energy_hdu.name = "ENERGIES" + hdul.append((energy_hdu)) + + hdul = fits.HDUList(hdul) + + filetowrite = path / filename + logger.info(f"Writing fits file to {filetowrite}") + hdul.writeto(filetowrite, overwrite=True, checksum=True) + return [filetowrite] + + +class PlotProcessor(FitsL2Processor): + """A file product processor for plot images""" + def __init__(self, archive_path): + """Creates a new PlotProcessor object. + + Parameters + ---------- + archive_path : Path + the output root path where the files should be created + """ + super().__init__(archive_path) + + def generate_filename(self, product, *, version=0, suffix='.svg'): + """Generates a SOAR conform filename based on product characteristics. + + Parameters + ---------- + product : Product + The data product the file name should be generated for + version : int, optional + The file version, by default 0 = detect from codebase + suffix : str, optional + file name suffix like svg, png, ..., by default '.svg' + + Returns + ------- + Path + a Path object with full name and path + """ + p = Path(super().generate_filename(product=product, version=version, header=True)) + return p.with_suffix(suffix).name + + def generate_primary_header(self, filename, product, *, version=0): + """Transforms the fits header into a more generic header dict + that might be used in other output file formats + + Parameters + ---------- + filename : Path + The envisoned file name and path for the product + product : Product + The products holding the data + version : int, optional + teh processing version, by default 0 = detect from codebase + + Returns + ------- + dict + a dict of header keywords and values """ - Write level 3 products into fits files. + l1, l2 = super().generate_primary_header(filename, product, version=version) + header = dict() + for k, v in l1: + header[k] = v + for kv in l2: + header[kv[0]] = kv[1] + return header + + def write_plot(self, product, *, version=0): + """ + Write products into a plot image file. Parameters ---------- - product : `stixcore.product.level3` + product : `stixcore.product.level2` version : `int` the version modifier for the filename @@ -1016,9 +1148,133 @@ def write_fits(self, product, *, version=0): if version == 0: version = product.get_processing_version() - return super().write_fits(product, version=version) + filename = self.generate_filename(product=product, version=version) + + # headers = self.generate_primary_header(filename, product, version=version) + # headers['parent'] = get_complete_file_name(product.parent_file_path.name) + + parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), + product.type.upper()] + # for science data use start date + if product.type in ["sci", "flarelist"]: + parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d") + path = self.archive_path.joinpath(*[str(x) for x in parts]) + path.mkdir(parents=True, exist_ok=True) + + plot_path = path / filename + + fig = product.get_plot() + fig.savefig(plot_path, format="svg") + plt.close(fig) + return plot_path + + +class FitsL3Processor(FitsL2Processor): + def __init__(self, archive_path): + super().__init__(archive_path) + + def write_fits(self, prod, *, version=0): + """Writes ANC products into fits files. + + Parameters + ---------- + product : `stixcore.product.ANC` + + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + + Returns + ------- + list + of created file as `pathlib.Path` + + """ + if version == 0: + version = prod.get_processing_version() + + filename = self.generate_filename(product=prod, version=version, header=False) + + parts = [prod.level, prod.utc_timerange.center.strftime("%Y/%m/%d"), prod.type.upper()] + # for science data use start date + if prod.type in ["flarelist"]: + parts[1] = prod.utc_timerange.start.strftime("%Y/%m/%d") + path = self.archive_path.joinpath(*[str(x) for x in parts]) + path.mkdir(parents=True, exist_ok=True) + + fitspath = path / filename + fitspath_complete = get_complete_file_name_and_path(fitspath) + + if fitspath.exists(): + logger.warning('Fits file %s exists data will be overriden', fitspath.name) + elif fitspath_complete.exists(): + logger.warning('Complete Fits file %s exists will be overriden', fitspath.name) + + data = prod.data + + primary_header, header_override = self.generate_primary_header(filename, prod, + version=version) + + primary_hdu = fits.PrimaryHDU() + primary_hdu.header.update(primary_header) + primary_hdu.header.update(header_override) + primary_hdu.header.update(prod.get_additional_header_keywords()) + hdul = [primary_hdu] + + # Add comment and history + [primary_hdu.header.add_comment(com) for com in prod.comment] + [primary_hdu.header.add_history(com) for com in prod.history] + primary_hdu.header.update({'HISTORY': 'Processed by STIXCore ANC'}) + + if hasattr(prod, "maps") and len(prod.maps) > 0: + # fig = plt.figure(figsize=(12, 6)) + # map_t = Map(prod.maps[0][0], prod.maps[0][1]) + # map_nt = Map(prod.maps[1][0], prod.maps[1][1]) + + # ax = fig.subplot_mosaic( + # [ + # ["t", "nt"] + # ], + # subplot_kw={"projection": map_t.wcs}, + # ) + + # mt = map_t.plot(axes=ax["t"]) + # ax["t"].set_title("thermal") + # map_nt.plot(axes=ax["nt"]) + # ax["nt"].set_title("non thermal") + # fig.colorbar(mt, ax=ax.values()) + # fig.savefig(fitspath.with_suffix(".svg"), format="svg") + # plt.close(fig) + for idx, map in enumerate(prod.maps): + map_data, header = map + image_hdu = fits.ImageHDU(map_data, header=header) + image_hdu.name = f'MAP_{idx}' + hdul.append(image_hdu) + + data_enc = fits.connect._encode_mixins(data) + data_hdu = table_to_hdu(data_enc) + data_hdu = set_bscale_unsigned(data_hdu) + data_hdu = add_default_tuint(data_hdu) + data_hdu.name = "DATA" + + hdul.append(data_hdu) + + if prod.energy and len(prod.energy) > 0: + energy_enc = fits.connect._encode_mixins(prod.energy) + energy_hdu = table_to_hdu(energy_enc) + energy_hdu = add_default_tuint(energy_hdu) + energy_hdu.name = "ENERGIES" + hdul.append(energy_hdu) + + hdul = fits.HDUList(hdul) + + filetowrite = path / filename + logger.info(f'Writing fits file to {filetowrite}') + hdul.writeto(filetowrite, overwrite=True, checksum=True) + return [filetowrite] def generate_primary_header(self, filename, product, *, version=0): + if product.fits_header is None: L2, o = super().generate_primary_header(filename, product, version=version) L2headers = L2 + o diff --git a/stixcore/processing/FLtoFL.py b/stixcore/processing/FLtoFL.py new file mode 100644 index 00000000..34beccd4 --- /dev/null +++ b/stixcore/processing/FLtoFL.py @@ -0,0 +1,210 @@ +from pathlib import Path +from datetime import datetime, timedelta + +from stixpy.net.client import STIXClient + +from astropy.io import fits +from astropy.table import QTable + +from stixcore.ephemeris.manager import Spice +from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.processing.SingleStep import ( + SingleProcessingStepResult, + SingleProductProcessingStepMixin, + TestForProcessingResult, +) +from stixcore.products.ANC.flarelist import ( + FlareListProduct, + FlarePeekPreviewMixin, + FlarePositionMixin, + FlareSOOPMixin, +) +from stixcore.products.product import Product +from stixcore.soop.manager import SOOPManager +from stixcore.util.logging import get_logger +from stixcore.util.util import get_complete_file_name_and_path + +__all__ = ['FLtoFL'] + +logger = get_logger(__name__) + + +class FLtoFL(SingleProductProcessingStepMixin): + """Processing step from a HK L1 fits file to a solo_ANC_stix-asp-ephemeris*.fits file. + """ + INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits" + + def __init__(self, source_dir: Path, output_dir: Path, *, + input_pattern: str = '', products_in_out, + cadence: timedelta = timedelta(hours=5)): + """Crates a new Processor. + + Parameters + ---------- + source_dir : Path or pathlike + where to search for input fits files + output_dir : Path or pathlike + where to write out the generated fits files + """ + self.source_dir = Path(source_dir) + self.output_dir = Path(output_dir) + self.cadence = cadence + self.in_products = {} + + for in_product, out_product in products_in_out: + self.in_products[in_product] = out_product + + def find_processing_candidates(self): + """Performs file pattern search in the source directory + + Returns + ------- + list[Path] + a list of fits files candidates + """ + for product_in, product_out in self.in_products.items(): + pattern = (f"solo_{product_in.LEVEL}_stix-{product_in.TYPE}-" + f"{product_in.NAME}_*.fits") + for c in self.source_dir.rglob(pattern): + yield product_in, product_out, c + + def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]: + """Performs file pattern search in the source directory + + Returns + ------- + list[Path] + a list of fits files candidates + """ + fl_to_process = list() + for product_in, product_out, fl_can in self.find_processing_candidates(): + res = self.test_for_processing(fl_can, product_in, product_out, phs) + if res == TestForProcessingResult.Suitable: + fl_to_process.append((product_in, product_out, fl_can)) + return fl_to_process + + def test_for_processing(self, candidate: Path, + product_in: FlareListProduct, + product_out: FlareListProduct, + phm: ProcessingHistoryStorage) -> TestForProcessingResult: + """_summary_ + + Parameters + ---------- + candidate : Path + a fits file candidate + phm : ProcessingHistoryStorage + the processing history persistent handler + + Returns + ------- + TestForProcessingResult + what should happen with the candidate in the next processing step + """ + try: + c_header = fits.getheader(candidate) + f_data_end = datetime.fromisoformat(c_header['DATE-END']) + f_create_date = datetime.fromisoformat(c_header['DATE']) + + cfn = get_complete_file_name_and_path(candidate) + + wp = phm.has_processed_fits_products(product_out.NAME, + product_out.LEVEL, + product_out.TYPE, + product_out.get_cls_processing_version(), + str(cfn), f_create_date) + + # found already in the processing history + if wp: + return TestForProcessingResult.NotSuitable + + # safety margin of 1day until we process higher products with position and pointing + # only use flown spice kernels not predicted once as pointing information + # can be "very off" + if (f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") + - timedelta(hours=24))): + return TestForProcessingResult.NotSuitable + + # safety margin of x until we start with processing the list files + if (f_create_date >= (datetime.now() - self.cadence)): + return TestForProcessingResult.NotSuitable + + return TestForProcessingResult.Suitable + except Exception as e: + logger.error(e) + return TestForProcessingResult.NotSuitable + + def process_fits_files(self, flarelists, *, soopmanager: SOOPManager, + spice_kernel_path: Path, fl_processor, + img_processor, config) -> list[Path]: + """Performs the processing (expected to run in a dedicated python process) from a + list flare list product to an enhanced flare list product. + + Parameters + ---------- + files : list[Path] + list input flare list product fits files + soopmanager : SOOPManager + the SOOPManager from the main process + spice_kernel_path : Path + the used spice kernel paths from the main process + fl_processor : _type_ + the processor from the main process + config : + the config from the main process + + Returns + ------- + list[Path] + list of all generated fits files + """ + CONFIG = config + SOOPManager.instance = soopmanager + Spice.instance = Spice(spice_kernel_path) + fido_client = STIXClient(source=CONFIG.get('Paths', 'fido_search_url', + fallback=STIXClient.baseurl)) + all_files = list() + + for in_product, out_product, file_path in flarelists: + try: + prod = Product(file_path) + control = QTable() + + data = prod.data + energy = prod.energies + month = prod.utc_timerange.start.datetime.date() + + # add flare position if not already present + if issubclass(out_product, FlarePositionMixin) and \ + not issubclass(in_product, FlarePositionMixin): + out_product.add_flare_position(data, fido_client, month=month) + + # add soop information if not already present + if issubclass(out_product, FlareSOOPMixin) and \ + not issubclass(in_product, FlareSOOPMixin): + out_product.add_soop(data) + + # add peek preview images if not already present + if issubclass(out_product, FlarePeekPreviewMixin) and \ + not issubclass(in_product, FlarePeekPreviewMixin): + out_product.add_peek_preview(data, energy, file_path.name, fido_client, + img_processor, month=month) + + out_prod = out_product(control=control, data=data, month=month, energy=energy) + out_prod.parent = file_path.name + + # call upgrade method to enhance the product + out_prod.enhance_from_product(prod) + + new_f = [SingleProcessingStepResult(out_product.NAME, out_product.LEVEL, + out_product.TYPE, + out_product.get_cls_processing_version(), + fop, get_complete_file_name_and_path(file_path), + datetime.now()) + for fop in fl_processor.write_fits(out_prod)] + + all_files.extend(new_f) + except Exception as e: + logger.error(e, stack_info=True) + + return all_files diff --git a/stixcore/processing/FLtoL3.py b/stixcore/processing/FLtoL3.py deleted file mode 100644 index 7d7e4667..00000000 --- a/stixcore/processing/FLtoL3.py +++ /dev/null @@ -1,92 +0,0 @@ -from pathlib import Path -from collections import defaultdict -from concurrent.futures import ProcessPoolExecutor - -from stixcore.config.config import CONFIG -from stixcore.ephemeris.manager import Spice -from stixcore.io.fits.processors import FitsL1Processor, FitsL3Processor -from stixcore.soop.manager import SOOPManager -from stixcore.util.logging import get_logger -from stixcore.util.util import get_complete_file_name - -logger = get_logger(__name__) - - -class FLLevel3: - """Processing step from a flare list entry to L3.""" - - def __init__(self, source_dir, output_dir, dbfile): - self.source_dir = Path(source_dir) - self.output_dir = Path(output_dir) - self.dbfile = dbfile - self.processor = FitsL3Processor(self.output_dir) - - def process_fits_files(self, files): - all_files = list() - if files is None: - files = self.level0_files - product_types = defaultdict(list) - product_types_batch = defaultdict(int) - batch_size = CONFIG.getint("Pipeline", "parallel_batchsize_L1", fallback=150) - - for file in files: - # group by service,subservice, ssid example: 'L0/21/6/30' as default - # or (prio, service, subservice, [SSID], [BATCH]) if all data is available - batch = 0 - prio = 3 - product_type = str(file.parent) - if "L0" in file._parts: - product_type = tuple(map(int, file._parts[file._parts.index("L0") + 1 : -1])) - if product_type[0] == 21 and product_type[-1] in {20, 21, 22, 23, 24, 42}: # sci data - product_types_batch[product_type] += 1 - prio = 2 - elif product_type[0] == 21: # ql data - prio = 1 - batch = product_types_batch[product_type] // batch_size - product_types[(prio,) + product_type + (batch,)].append(file) - - jobs = [] - with ProcessPoolExecutor() as executor: - # simple heuristic that the daily QL data takes longest so we start early - for pt, files in sorted(product_types.items()): - jobs.append( - executor.submit( - process_type, - files, - processor=FitsL1Processor(self.output_dir), - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - config=CONFIG, - ) - ) - - for job in jobs: - try: - new_files = job.result() - all_files.extend(new_files) - except Exception: - logger.error("error", exc_info=True) - - return list(set(all_files)) - - -def process_type(timeranges, productcls, flarelistparent, *, processor, soopmanager, spice_kernel_path, config): - SOOPManager.instance = soopmanager - all_files = list() - Spice.instance = Spice(spice_kernel_path) - CONFIG = config - file = 1 - prod = productcls() - for tr in timeranges: - logger.info(f"processing timerange: {timeranges}") - try: - # see https://github.com/i4Ds/STIXCore/issues/350 - get_complete_file_name(file.name) - l3 = prod.from_timerange(tr, flarelistparent=flarelistparent) - all_files.extend(processor.write_fits(l3)) - except Exception as e: - logger.error("Error processing timerange %s", tr, exc_info=True) - logger.error("%s", e) - if CONFIG.getboolean("Logging", "stop_on_error", fallback=False): - raise e - return all_files diff --git a/stixcore/processing/FlareListANC.py b/stixcore/processing/FlareListANC.py new file mode 100644 index 00000000..785bb129 --- /dev/null +++ b/stixcore/processing/FlareListANC.py @@ -0,0 +1,145 @@ +from pathlib import Path +from datetime import date, datetime, timedelta + +import pandas as pd +from stixpy.net.client import STIXClient + +from stixcore.ephemeris.manager import Spice +from stixcore.io import FlareListManager +from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.processing.SingleStep import ( + SingleProcessingStepResult, + SingleProductProcessingStepMixin, + TestForProcessingResult, +) +from stixcore.products.ANC.flarelist import FlarePositionMixin, FlareSOOPMixin +from stixcore.soop.manager import SOOPManager +from stixcore.util.logging import get_logger + +__all__ = ['FlareListANC', 'FlarePositionMixin'] + +logger = get_logger(__name__) + + +class FlareListANC(SingleProductProcessingStepMixin): + """Processing step from a FlareListManager to monthly solo_ANC_stix-flarelist-*.fits file. + """ + + STARTDATE = date(2024, 1, 1) + + def __init__(self, flm: FlareListManager, output_dir: Path): + """Crates a new Processor. + + Parameters + ---------- + flm : FlareListManager + a FlareListManager instance + output_dir : Path or pathlike + where to write out the generated fits files + """ + self.flm = flm + self.output_dir = Path(output_dir) + + def find_processing_months(self, phs: ProcessingHistoryStorage) -> list[date]: + """Performs file pattern search in the source directory + + Returns + ------- + list[Path] + a list of fits files candidates + """ + fl_months = list() + for month in pd.date_range(FlareListANC.STARTDATE, date.today(), freq='MS').date: + if self.test_for_processing(month, phs) == TestForProcessingResult.Suitable: + fl_months.append(month) + return fl_months + + def test_for_processing(self, month: date, + phm: ProcessingHistoryStorage) -> TestForProcessingResult: + """_summary_ + + Parameters + ---------- + candidate : Path + a fits file candidate + phm : ProcessingHistoryStorage + the processing history persistent handler + + Returns + ------- + TestForProcessingResult + what should happen with the candidate in the next processing step + """ + try: + wp = phm.has_processed_fits_products(self.flm.productCls.NAME, + self.flm.productCls.LEVEL, + self.flm.productCls.TYPE, + self.flm.productCls.get_cls_processing_version(), + str(month), FlareListANC.STARTDATE) + + # found already in the processing history + if wp: + return TestForProcessingResult.NotSuitable + + # safety margin of 2 month until we start with processing of monthly flare list files + if (month >= (date.today() - timedelta(days=50)).replace(day=1)): + return TestForProcessingResult.NotSuitable + + return TestForProcessingResult.Suitable + except Exception as e: + logger.error(e) + return TestForProcessingResult.NotSuitable + + def process_fits_files(self, months: list[date], *, soopmanager: SOOPManager, + spice_kernel_path: Path, processor, config) -> list[Path]: + """Performs the processing (expected to run in a dedicated python process) from a + list of solo_L1_stix-hk-maxi_*.fits into solo_ANC_stix-asp-ephemeris*.fits files. + + Parameters + ---------- + files : list[Path] + list of HK level 1 fits files + soopmanager : SOOPManager + the SOOPManager from the main process + spice_kernel_path : Path + the used spice kernel paths from the main process + fits_processor : _type_ + the processor from the main process + config : + the config from the main process + + Returns + ------- + list[Path] + list of all generated fits files + """ + CONFIG = config + SOOPManager.instance = soopmanager + Spice.instance = Spice(spice_kernel_path) + all_files = list() + fido_url = CONFIG.get('Paths', 'fido_search_url', + fallback='https://pub099.cs.technik.fhnw.ch/data/fits') + fido_client = STIXClient(source=fido_url) + + for month in months: + data, control, energy = self.flm.get_data(start=month, + end=(month + pd.DateOffset(months=1)).date(), + fido_client=fido_client) + + if not data: + logger.warning(f'No data found for month {month}') + continue + + if issubclass(self.flm.productCls, FlarePositionMixin): + self.flm.productCls.add_flare_position(data, fido_client, filter_function=self.flm.filter_flare_function) # noqa + + if issubclass(self.flm.productCls, FlareSOOPMixin): + self.flm.productCls.add_soop(data) + + anc = self.flm.productCls(control=control, data=data, month=month, energy=energy) + + all_files.extend([SingleProcessingStepResult(anc.name, anc.level, anc.type, + anc.get_processing_version(), + fop, str(month), datetime.now()) + for fop in processor.write_fits(anc)]) + return all_files diff --git a/stixcore/processing/LL.py b/stixcore/processing/LL.py new file mode 100644 index 00000000..8b69b751 --- /dev/null +++ b/stixcore/processing/LL.py @@ -0,0 +1,172 @@ +from pathlib import Path +from datetime import datetime, timedelta + +from astropy.io import fits + +from stixcore.io.fits.processors import PlotProcessor +from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.processing.SingleStep import ( + SingleProcessingStepResult, + SingleProductProcessingStepMixin, + TestForProcessingResult, +) +from stixcore.products.level1.quicklookL1 import LightCurve +from stixcore.products.lowlatency.quicklookLL import LightCurveL3 +from stixcore.products.product import Product +from stixcore.soop.manager import SOOPManager +from stixcore.util.logging import get_logger +from stixcore.util.util import get_complete_file_name_and_path + +__all__ = ['LL03QL'] + +logger = get_logger(__name__) + + +class LL03QL(SingleProductProcessingStepMixin): + """Processing step from a ql ligtcurve L1 fits file to LL level 3 plot chart image. + """ + INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits" + + def __init__(self, source_dir: Path, output_dir: Path, *, + input_pattern: str = '', in_product: LightCurve, out_product: LightCurveL3, + cadence: timedelta = timedelta(minutes=1)): + """Crates a new Processor. + + Parameters + ---------- + source_dir : Path or pathlike + where to search for input fits files + output_dir : Path or pathlike + where to write out the generated fits files + """ + self.source_dir = Path(source_dir) + self.output_dir = Path(output_dir) + self.cadence = cadence + self.in_product = in_product + self.out_product = out_product + + self.input_pattern = "solo_L1_stix-ql-lightcurve_*.fits" + + def find_processing_candidates(self) -> list[Path]: + """Performs file pattern search in the source directory + + Returns + ------- + list[Path] + a list of fits files candidates + """ + return list(self.source_dir.rglob(self.input_pattern)) + + def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]: + """find all input files that are ready for processing + + Returns + ------- + list[Path] + a list of fits files for processing + """ + ql_to_process = list() + for ql_can in self.find_processing_candidates(): + if self.test_for_processing(ql_can, phs) == TestForProcessingResult.Suitable: + ql_to_process.append(ql_can) + return ql_to_process + + def test_for_processing(self, candidate: Path, + phm: ProcessingHistoryStorage) -> TestForProcessingResult: + """Performs a check on each found candidate if it should be processed or skipped + + Parameters + ---------- + candidate : Path + a fits file candidate + phm : ProcessingHistoryStorage + the processing history persistent handler + + Returns + ------- + TestForProcessingResult + what should happen with the candidate in the next processing step + """ + try: + c_header = fits.getheader(candidate) + f_data_end = datetime.fromisoformat(c_header['DATE-END']) + f_create_date = datetime.fromisoformat(c_header['DATE']) + + cfn = get_complete_file_name_and_path(candidate) + + wp = phm.has_processed_fits_products(self.out_product.NAME, + self.out_product.LEVEL, + self.out_product.TYPE, + self.out_product.get_cls_processing_version(), + str(cfn), f_create_date) + + # found already in the processing history + if wp: + return TestForProcessingResult.NotSuitable + + # safety margin of x until we start with processing the list files + if (f_create_date >= (datetime.now() - self.cadence)): + return TestForProcessingResult.NotSuitable + + # only process QL files with data for full 24 hours no created of half a day plots + header_data = fits.getheader(candidate, "DATA") + tbins = header_data["NAXIS2"] + # small rounding margin of 4 minutes is allowed (-60) + # input files oldr then a month will be processed also as for not full 24 hours + file_age = (datetime.now() - f_data_end).days + if (tbins < 60/4 * 60 * 24 - 60) and file_age < 30: + return TestForProcessingResult.NotSuitable + + return TestForProcessingResult.Suitable + except Exception as e: + logger.error(e) + return TestForProcessingResult.NotSuitable + + def process_fits_files(self, files: list[Path], *, soopmanager: SOOPManager, + spice_kernel_path: Path, processor: PlotProcessor, config) -> list[Path]: + """Performs the processing (expected to run in a dedicated python process) from a + list of ql lightcurve products into LL plots. + + Parameters + ---------- + files : list[Path] + list input flare list product fits files + processor : _type_ + the processor from the main process + config : + the config from the main process + + Returns + ------- + list[Path] + list of all generated plot files + """ + + all_files = list() + + for file_path in files: + try: + in_prod = Product(file_path) + + control = in_prod.control + data = in_prod.data + energy = in_prod.energies + + out_prod = self.out_product(control=control, data=data, energy=energy, + parent_file_path=file_path, header=in_prod.fits_header) + + plot_file = processor.write_plot(out_prod) + logger.info(f"Generated plot file: {plot_file}") + + new_f = SingleProcessingStepResult(self.out_product.NAME, self.out_product.LEVEL, + self.out_product.TYPE, + self.out_product.get_cls_processing_version(), + plot_file, + get_complete_file_name_and_path(file_path), + datetime.now()) + + all_files.append(new_f) + except Exception as e: + logger.error(e, stack_info=True) + + return all_files diff --git a/stixcore/processing/SingleStep.py b/stixcore/processing/SingleStep.py index 7316923c..e4dff97a 100644 --- a/stixcore/processing/SingleStep.py +++ b/stixcore/processing/SingleStep.py @@ -3,15 +3,9 @@ from pathlib import Path from datetime import datetime from collections import defaultdict -from concurrent.futures import ProcessPoolExecutor -import stixcore.io.fits.processors as fitsp -from stixcore.config.config import CONFIG -from stixcore.ephemeris.manager import Spice from stixcore.products.product import GenericProduct -from stixcore.soop.manager import SOOPManager from stixcore.util.logging import get_logger -from stixcore.util.util import get_complete_file_name __all__ = ["TestForProcessingResult", "SingleProductProcessingStepMixin", "SingleProcessingStepResult"] @@ -103,84 +97,3 @@ def process(self, product: GenericProduct) -> GenericProduct: def write_fits(self, product: GenericProduct, folder: Path) -> Path: pass - - -# NOT NEEDED NOW -class FLLevel3: - """Processing step from a flare list entry to L3.""" - - def __init__(self, source_dir, output_dir, dbfile): - self.source_dir = Path(source_dir) - self.output_dir = Path(output_dir) - self.dbfile = dbfile - self.processor = fitsp.FitsL3Processor(self.output_dir) - - def process_fits_files(self, files): - all_files = list() - if files is None: - files = self.level0_files - product_types = defaultdict(list) - product_types_batch = defaultdict(int) - batch_size = CONFIG.getint("Pipeline", "parallel_batchsize_L1", fallback=150) - - for file in files: - # group by service,subservice, ssid example: 'L0/21/6/30' as default - # or (prio, service, subservice, [SSID], [BATCH]) if all data is available - batch = 0 - prio = 3 - product_type = str(file.parent) - if "L0" in file._parts: - product_type = tuple(map(int, file._parts[file._parts.index("L0") + 1 : -1])) - if product_type[0] == 21 and product_type[-1] in {20, 21, 22, 23, 24, 42}: # sci data - product_types_batch[product_type] += 1 - prio = 2 - elif product_type[0] == 21: # ql data - prio = 1 - batch = product_types_batch[product_type] // batch_size - product_types[(prio,) + product_type + (batch,)].append(file) - - jobs = [] - with ProcessPoolExecutor() as executor: - # simple heuristic that the daily QL data takes longest so we start early - for pt, files in sorted(product_types.items()): - jobs.append( - executor.submit( - process_type, - files, - processor=fitsp.FitsL1Processor(self.output_dir), - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - config=CONFIG, - ) - ) - - for job in jobs: - try: - new_files = job.result() - all_files.extend(new_files) - except Exception: - logger.error("error", exc_info=True) - - return list(set(all_files)) - - -def process_type(timeranges, productcls, flarelistparent, *, processor, soopmanager, spice_kernel_path, config): - SOOPManager.instance = soopmanager - all_files = list() - Spice.instance = Spice(spice_kernel_path) - CONFIG = config - file = 1 - prod = productcls() - for tr in timeranges: - logger.info(f"processing timerange: {timeranges}") - try: - # see https://github.com/i4Ds/STIXCore/issues/350 - get_complete_file_name(file.name) - l3 = prod.from_timerange(tr, flarelistparent=flarelistparent) - all_files.extend(processor.write_fits(l3)) - except Exception as e: - logger.error("Error processing timerange %s", tr, exc_info=True) - logger.error("%s", e) - if CONFIG.getboolean("Logging", "stop_on_error", fallback=False): - raise e - return all_files diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index d3d81084..7c2e2254 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -4,20 +4,31 @@ import smtplib import argparse from pathlib import Path -from datetime import date +from datetime import date, timedelta from concurrent.futures import ProcessPoolExecutor from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager -from stixcore.io.fits.processors import FitsL2Processor +from stixcore.io.fits.processors import ( + FitsANCProcessor, + FitsL2Processor, + FitsL3Processor, + PlotProcessor, +) +from stixcore.io.FlareListManager import SDCFlareListManager from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage from stixcore.io.RidLutManager import RidLutManager from stixcore.processing.AspectANC import AspectANC +from stixcore.processing.FlareListANC import FlareListANC +from stixcore.processing.FLtoFL import FLtoFL +from stixcore.processing.LL import LL03QL from stixcore.processing.pipeline import PipelineStatus from stixcore.processing.SingleStep import SingleProcessingStepResult, TestForProcessingResult +from stixcore.products.ANC.flarelist import FlarelistSDCLoc, FlarelistSDCLocImg +from stixcore.products.level1.quicklookL1 import LightCurve +from stixcore.products.lowlatency.quicklookLL import LightCurveL3 from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger -from stixcore.util.util import get_complete_file_name_and_path logger = get_logger(__name__) @@ -156,21 +167,9 @@ def run_daily_pipeline(args): dest="log_level", ) - parser.add_argument( - "-r", - "--rid_lut_file", - help=("Path to the rid LUT file"), - default=CONFIG.get("Publish", "rid_lut_file"), - type=str, - ) - - parser.add_argument( - "--update_rid_lut", - help="update rid lut file before publishing", - default=False, - action="store_true", - dest="update_rid_lut", - ) + parser.add_argument("-r", "--rid_lut_file", + help=("Path to the rid LUT file"), + default=CONFIG.get('Publish', 'rid_lut_file'), type=str) args = parser.parse_args(args) @@ -194,12 +193,16 @@ def run_daily_pipeline(args): Spice.instance = Spice(spicemeta) - SOOPManager.instance = SOOPManager(Path(CONFIG.get("Paths", "soop_files"))) - - RidLutManager.instance = RidLutManager(Path(args.rid_lut_file), update=args.update_rid_lut) + SOOPManager.instance = SOOPManager(Path(CONFIG.get('Paths', 'soop_files'))) Path(CONFIG.get("Paths", "fits_archive")) + flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sdc_lut_file')) + SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) + + RidLutManager.instance = RidLutManager(Path(CONFIG.get('Publish', 'rid_lut_file')), + update=False) + db_file = Path(args.db_file) fits_in_dir = Path(args.fits_in_dir) if not fits_in_dir.exists(): @@ -228,30 +231,71 @@ def run_daily_pipeline(args): aspect_anc_processor = AspectANC(fits_in_dir, fits_out_dir) + flarelist_sdc = FlareListANC(SDCFlareListManager.instance, fits_out_dir) + fl_to_fl = FLtoFL(fits_in_dir, + fits_out_dir, + products_in_out=[ # (FlarelistSDC, FlarelistSDCLoc), + (FlarelistSDCLoc, FlarelistSDCLocImg)], + cadence=timedelta(seconds=1)) + + ll03ql = LL03QL(fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, + cadence=timedelta(seconds=1)) + + plot_writer = PlotProcessor(fits_out_dir) l2_fits_writer = FitsL2Processor(fits_out_dir) + l3_fits_writer = FitsL3Processor(fits_out_dir) + anc_fits_writer = FitsANCProcessor(fits_out_dir) # should be done later: internally hk_in_files = [] - candidates = aspect_anc_processor.find_processing_candidates() + candidates = list() # aspect_anc_processor.find_processing_candidates() v_candidates = aspect_anc_processor.get_version(candidates, version="latest") for fc in v_candidates: tr = aspect_anc_processor.test_for_processing(fc, phs) if tr == TestForProcessingResult.Suitable: hk_in_files.append(fc) + fl_months = flarelist_sdc.find_processing_months(phs) + fl_months = [] + + ll_candidates = ll03ql.get_processing_files(phs) + ll_candidates = [] + + fl_to_fl_files = fl_to_fl.get_processing_files(phs) + # flsdc_to_flcdspos_files = [] + + # all processing files should be terminated before the next step as the different + # processing steeps might create new candidates # let each processing "task" run in its own process jobs = [] with ProcessPoolExecutor() as executor: - jobs.append( - executor.submit( - aspect_anc_processor.process_fits_files, - hk_in_files, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l2_fits_writer, - config=CONFIG, - ) - ) + jobs.append(executor.submit(aspect_anc_processor.process_fits_files, hk_in_files, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l2_fits_writer, + config=CONFIG)) + + jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_months, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=anc_fits_writer, + config=CONFIG)) + + # TODO a owen processing step for each flarelist file? + # for fl_to_fl_file in fl_to_fl_files: + jobs.append(executor.submit(fl_to_fl.process_fits_files, + fl_to_fl_files, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + fl_processor=anc_fits_writer, + img_processor=l3_fits_writer, + config=CONFIG)) + + jobs.append(executor.submit(ll03ql.process_fits_files, ll_candidates, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=plot_writer, + config=CONFIG)) # wait for all processes to end all_files = [] @@ -266,15 +310,9 @@ def run_daily_pipeline(args): # create an entry for each generated file in the ProcessingHistoryStorage for pr in all_files: if isinstance(pr, SingleProcessingStepResult): - phs.add_processed_fits_products( - pr.name, - pr.level, - pr.type, - pr.version, - get_complete_file_name_and_path(pr.in_path), - pr.out_path, - pr.date, - ) + phs.add_processed_fits_products(pr.name, pr.level, pr.type, + pr.version, pr.in_path, + pr.out_path, pr.date) phs.close() diff --git a/stixcore/processing/publish.py b/stixcore/processing/publish.py index 6caa2476..f9bd168f 100644 --- a/stixcore/processing/publish.py +++ b/stixcore/processing/publish.py @@ -6,6 +6,7 @@ import sqlite3 import argparse import tempfile +import itertools from io import StringIO from enum import Enum from pprint import pformat @@ -666,6 +667,9 @@ def publish_fits_to_esa(args): wait_period = u.Quantity(args.waiting_period) wait_period_s = wait_period.to(u.s).value + LL_wait_period = u.Quantity(CONFIG.get('Publish', 'LL_waiting_period', fallback="1min")) + LL_wait_period_s = LL_wait_period.to(u.s).value + target_dir = Path(args.target_dir) if (scp is None) and (not target_dir.exists()): logger.info(f"path not found to target dir: {target_dir} creating dir") @@ -695,7 +699,9 @@ def publish_fits_to_esa(args): include_all_products = False include_products = dict([(prod, 1) for prod in args.include_products.lower().replace(" ", "").split(",")]) - candidates = fits_dir.rglob("solo_*.fits") + fits_candidates = fits_dir.rglob("solo_*.fits") + img_candidates = fits_dir.rglob("solo_*.svg") + candidates = itertools.chain(img_candidates, fits_candidates) to_publish = list() now = datetime.now().timestamp() next_week = datetime.now(timezone.utc) + timedelta(hours=24 * 7) @@ -741,6 +747,7 @@ def publish_fits_to_esa(args): logger.info(f"target_host: {args.target_host}") logger.info(f"scp: {scp is not None}") logger.info(f"wait_period: {wait_period}") + logger.info(f"LL wait_period: {LL_wait_period}") logger.info(f"fits_dir: {fits_dir}") logger.info(f"temp_dir: {tempdir_path}") logger.info(f"rid_lut_file: {args.rid_lut_file}") @@ -761,7 +768,7 @@ def publish_fits_to_esa(args): for c in candidates: n_candidates += 1 - parts = c.name[:-5].split("_") + parts = c.stem.split("_") level = parts[1].lower() if not include_all_versions: @@ -775,8 +782,10 @@ def publish_fits_to_esa(args): if level not in include_levels: continue + wp_s = LL_wait_period_s if level.startswith('ll') else wait_period_s + # is the waiting time after last modification done - if (now - last_mod) < wait_period_s: + if (now - last_mod) < wp_s: continue if not include_all_products: @@ -807,33 +816,37 @@ def publish_fits_to_esa(args): for p in to_publish: try: - comment = add_BSD_comment(p) + isFits = p.suffix == '.fits' + if isFits: + comment = add_BSD_comment(p) if is_incomplete_file_name(p.name): p = p.rename(p.parent / get_complete_file_name(p.name)) - update_ephemeris_headers(p, spice) + if isFits: + update_ephemeris_headers(p, spice) add_res, data = hist.add(p) - try: - # do not publish products from the future - p_header = fits.getheader(p) - if "DATE-BEG" in p_header and ":" not in p_header["DATE-BEG"]: - p_data_beg = datetime.fromisoformat(p_header["DATE-BEG"]) - else: - p_data_beg = SCETime.from_float(p_header["OBT_BEG"] * u.s).to_datetime() - if p_data_beg > next_week: - hist.set_result(PublishResult.IGNORED, data[0]["id"]) - published[PublishResult.IGNORED].append(data) - continue - except Exception: - pass + if isFits: + try: + # do not publish products from the future + p_header = fits.getheader(p) + if "DATE-BEG" in p_header and ":" not in p_header["DATE-BEG"]: + p_data_beg = datetime.fromisoformat(p_header["DATE-BEG"]) + else: + p_data_beg = SCETime.from_float(p_header["OBT_BEG"] * u.s).to_datetime() + if p_data_beg > next_week: + hist.set_result(PublishResult.IGNORED, data[0]["id"]) + published[PublishResult.IGNORED].append(data) + continue + except Exception: + pass if p.name in blacklist_files: hist.set_result(PublishResult.BLACKLISTED, data[0]["id"]) published[PublishResult.BLACKLISTED].append(data) elif add_res == PublishConflicts.ADDED: - ok, error = copy_file(scp, p, target_dir) + ok, error = copy_file(scp, p, target_dir, add_history_entry=isFits) if ok: hist.set_result(PublishResult.PUBLISHED, data[0]["id"]) published[PublishResult.PUBLISHED].append(data) @@ -843,7 +856,7 @@ def publish_fits_to_esa(args): elif add_res == PublishConflicts.SAME_EXISTS: # do nothing but add to report published[PublishResult.IGNORED].append(data) - elif add_res == PublishConflicts.SAME_ESA_NAME: + elif add_res == PublishConflicts.SAME_ESA_NAME and isFits: # esa naming conflict equal_data_once = False new_entry_id = None diff --git a/stixcore/products/ANC/aspect.py b/stixcore/products/ANC/aspect.py index 89d7b93a..011b1e04 100644 --- a/stixcore/products/ANC/aspect.py +++ b/stixcore/products/ANC/aspect.py @@ -208,17 +208,15 @@ def postprocessing(self, result, fits_processor): aux = Ephemeris(control=control, data=data, idb_versions=HK.idb_versions) aux.add_additional_header_keyword( - ("STX_GSW", result.idlgswversion[0].decode(), "Version of STX-GSW that provided data") - ) - aux.add_additional_header_keyword(("HISTORY", "aspect data processed by STX-GSW", "")) - files.extend( - [ - SingleProcessingStepResult( - aux.name, aux.level, aux.type, aux.get_processing_version(), fop, file_path, datetime.now() - ) - for fop in fits_processor.write_fits(aux) - ] - ) + ("STX_GSW", result.idlgswversion[0].decode(), + "Version of STX-GSW that provided data")) + aux.add_additional_header_keyword( + ("HISTORY", "aspect data processed by STX-GSW", "")) + files.extend([SingleProcessingStepResult(aux.name, aux.level, aux.type, + aux.get_processing_version(), fop, + get_complete_file_name_and_path(file_path), + datetime.now()) + for fop in fits_processor.write_fits(aux)]) else: logger.error("IDL ERROR") diff --git a/stixcore/products/ANC/flarelist.py b/stixcore/products/ANC/flarelist.py new file mode 100644 index 00000000..5b0ab3db --- /dev/null +++ b/stixcore/products/ANC/flarelist.py @@ -0,0 +1,528 @@ +from pathlib import Path +from datetime import datetime + +import numpy as np +from stixpy.calibration.visibility import ( + calibrate_visibility, + create_meta_pixels, + create_visibility, +) +from stixpy.coordinates.transforms import get_hpc_info +from stixpy.net.client import STIXClient +from stixpy.product import Product as STIXPYProduct +from sunpy.coordinates import HeliographicStonyhurst, Helioprojective +from sunpy.map import make_fitswcs_header +from sunpy.net import attrs as a +from sunpy.time import TimeRange +from xrayvision.clean import vis_clean + +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.table import Column, QTable +from astropy.time import Time + +from stixcore.config.config import CONFIG +from stixcore.ephemeris.manager import Spice +from stixcore.products.level3.flarelistproduct import PeekPreviewImage +from stixcore.products.product import CountDataMixin, GenericProduct, L2Mixin, read_qtable +from stixcore.soop.manager import SOOPManager +from stixcore.time import SCETime, SCETimeRange +from stixcore.util.logging import get_logger +from stixcore.util.util import url_to_path + +from stixpy.map.stix import STIXMap # noqa + +__all__ = ['FlarelistSDC', 'FlarePositionMixin', 'FlareSOOPMixin', + 'FlareListProduct', 'FlarelistSDCLocImg', 'FlarePeekPreviewMixin'] + +logger = get_logger(__name__) + + +def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_angle, + energy_range): + """Create a FITS WCS header for the given data.""" + header = make_fitswcs_header(data, flare_position, + telescope="STIX", + observatory="Solar Orbiter", + scale=scale, + exposure=exposure, + rotation_angle=rotation_angle) + + if energy_range is not None and header["wcsaxes"] == 2: + # add energy range to WCS header + # as a fake 3rd axis + header["wcsaxes"] = 3 + header["CRVAL3"] = energy_range.mean().value + header["CRPIX3"] = 1 + header["CDELT3"] = energy_range[1].value - energy_range[0].value + header["CUNIT3"] = str(energy_range[0].unit) + header["CTYPE3"] = "ENER" + header["NAXIS"] = 3 + header["NAXIS3"] = 1 + + return header + + +class FlarePositionMixin: + """_summary_ + """ + @classmethod + def add_flare_position(cls, data, fido_client: STIXClient, *, + filter_function=lambda x: True, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + keep_all_flares=True, + month=None + ): + + data['flare_position'] = [SkyCoord(0, 0, frame="icrs", unit="deg") + for i in range(0, len(data))] + + data['aux_ephemeris_path'] = Column(" " * 500, dtype=str, + description='TDB') + data['cpd_path'] = Column(" " * 500, dtype=str, + description='TDB') + data['_position_status'] = Column(False, dtype=bool, + description='TDB') + data['_position_message'] = Column(" " * 500, dtype=str, + description='TDB') + to_remove = [] + pass_filter = 0 + no_ephemeris = 0 + no_cpd = 0 + many_cpd = 0 + one_cpd = 0 + total_flares = len(data) + + day_asp_ephemeris_cache = dict() + + for i, row in enumerate(data): + if filter_function(row): + pass_filter += 1 + peak_time = row[peek_time_colname] + start_time = row[start_time_colname] + end_time = row[end_time_colname] + + day = peak_time.to_datetime().date() + + if day in day_asp_ephemeris_cache: + anc_res = day_asp_ephemeris_cache[day] + else: + anc_res = fido_client.search(a.Time(peak_time, peak_time), + a.Instrument.stix, + a.stix.DataProduct.asp_ephemeris) + if anc_res: + anc_res.filter_for_latest_version() + url_to_path(anc_res) + day_asp_ephemeris_cache[day] = anc_res + + if len(anc_res) < 1: + logger.warning(f'No ephemeris data found for flare at time ' + f'{start_time} : {end_time}') + data[i]['_position_message'] = "no ephemeris data found" + no_ephemeris += 1 + continue + data[i]['aux_ephemeris_path'] = anc_res["path"][0] + + if start_time.datetime.hour < 2: + start_time = start_time - 2 * u.hour + cpd_res = fido_client.search(a.Time(start_time, end_time), + a.Instrument.stix, + a.stix.DataProduct.sci_xray_cpd) + if cpd_res: + cpd_res.filter_for_latest_version() + url_to_path(cpd_res) + + if len(cpd_res) < 1: + logger.warning(f'No CPD data found for flare at time {start_time} : {end_time}') + data[i]['_position_message'] = "no CPD data found" + no_cpd += 1 + continue + if len(cpd_res) > 1: + logger.debug(f'Many CPD data found for flare at time {start_time} : {end_time}') + # select the best available CPD data file + cpd_res["inc_peak"] = np.logical_and(peak_time >= cpd_res["Start Time"], + peak_time <= cpd_res["End Time"]) + cpd_res["file_size"] = [path.stat().st_size for path in cpd_res["path"]] + cpd_res["exposure"] = 0.0 + cpd_res["duration"] = 0.0 + cpd_res["tbins"] = 0 + cpd_res["ebins"] = 0 + cpd_res["estart"] = 0.0 + cpd_res["eend"] = 0.0 + cpd_res["maxcount"] = 0 + many_cpd += 1 + + for i, path in enumerate(cpd_res["path"]): + header = fits.getheader(path) + header_data = fits.getheader(path, "DATA") + energies = read_qtable(path, "ENERGIES") + cpd_res["maxcount"][i] = header["DATAMAX"] + cpd_res["exposure"][i] = header["XPOSURE"] + cpd_res["tbins"][i] = header_data["NAXIS2"] + cpd_res["ebins"][i] = len(energies) + cpd_res["estart"][i] = energies["e_low"][0].value + cpd_res["eend"][i] = energies["e_high"][-1].value if len(energies) < 31 \ + else energies["e_high"][-2].value + cpd_res["duration"][i] = header["OBT_END"]-header["OBT_BEG"] + + # TODO: add more criteria to select the best CPD file + cpd_res.sort(["tbins", "duration"]) + # cpd_res.pprint() + best_cpd_idx = 0 + else: + one_cpd += 1 + best_cpd_idx = 0 + data[i]['cpd_path'] = cpd_res["path"][best_cpd_idx] + + # do the calculations with stixpy + + data[i]['flare_position'] = SkyCoord(1, 1, frame="icrs", unit="deg") + data[i]['_position_status'] = True + data[i]['_position_message'] = "OK" + + else: + to_remove.append(i) + + if not keep_all_flares: + data.remove_rows(to_remove) + + logger.info(f'Flare position calculated for month {month} with {total_flares} flares, ' + f'passed filter: {pass_filter} no ephemeris data found for {no_ephemeris} ' + f'flares, no CPD data found for {no_cpd} flares, many CPD data found for ' + f'{many_cpd} flares, one CPD data found for {one_cpd} flares') + + +class FlareSOOPMixin: + """_summary_ + """ + + @classmethod + def add_soop(self, data, *, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC'): + + soop_encoded_type = list() + soop_id = list() + soop_type = list() + + for row in data: + soops = SOOPManager.instance.find_soops(start=row[peek_time_colname]) + if soops: + soop = soops[0] + soop_encoded_type.append(soop.encodedSoopType) + soop_id.append(soop.soopInstanceId) + soop_type.append(soop.soopType) + else: + soop_encoded_type.append(None) + soop_id.append(None) + soop_type.append(None) + + data['soop_encoded_type'] = Column(soop_encoded_type, dtype=str, description='campaign ID') + data['soop_id'] = Column(soop_id, dtype=str, description='SOOP ID') + data['soop_type'] = Column(soop_type, dtype=str, description='name of the SOOP campaign') + + +class FlarePeekPreviewMixin: + """_summary_ + """ + @classmethod + def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + aux_ephemeris_path_colname='aux_ephemeris_path', + cpd_path_colname='cpd_path', + product_name_suffix='fl', + keep_all_flares=True, + month=None + ): + data['peek_preview_path'] = Column(" " * 500, dtype=str, + description='TDB') + data['preview_start_UTC'] = [Time(d, format='isot', scale='utc') + for d in data[peek_time_colname]] + data['preview_end_UTC'] = [Time(d, format='isot', scale='utc') + for d in data[peek_time_colname]] + data['_peek_preview_status'] = Column(False, dtype=bool, + description='TDB') + data['_peek_preview_message'] = Column(" " * 500, dtype=str, + description='TDB') + to_remove = [] + products = [] + images = 0 + + for i, row in enumerate(data): + peak_time = row[peek_time_colname] + row[start_time_colname] + row[end_time_colname] + + aux_ephemeris_path = Path(row[aux_ephemeris_path_colname]) + cpd_path = Path(row[cpd_path_colname]) + + status = False + + peek_preview_start = row[peek_time_colname] + peek_preview_end = row[peek_time_colname] + + if aux_ephemeris_path.exists() and cpd_path.exists(): + try: + status = True + # do the imaging with stixpy + + preview_data = data[i:i+1] + del preview_data['peek_preview_path'] + del preview_data['_peek_preview_status'] + del preview_data['_peek_preview_message'] + + peek_preview_start = row[peek_time_colname] - 10 * u.s + peek_preview_end = row[peek_time_colname] + 10 * u.s + + preview_data['preview_start_UTC'] = peek_preview_start + preview_data['preview_end_UTC'] = peek_preview_end + + cpd_sci = STIXPYProduct(cpd_path) + time_range_sci = [peek_preview_start, peek_preview_end] + maps = [] + for energy_range in [[4, 18], [18, 120]] * u.keV: + # flare_position = preview_data['flare_position'][0] + # flare_position = [0, 0] * u.arcsec + comments = [] + helio_frame = Helioprojective(observer='earth', obstime=peak_time) + flare_position = SkyCoord(0*u.deg, 0*u.deg, frame=helio_frame) + + meta_pixels_sci = create_meta_pixels(cpd_sci, + time_range=time_range_sci, + energy_range=energy_range, + flare_location=flare_position, + no_shadowing=True + ) + vis = create_visibility(meta_pixels_sci) + cal_vis = calibrate_visibility(vis, flare_location=flare_position) + isc_10_3 = [3, 20, 22, 16, 14, 32, 21, 26, 4, 24, 8, 28, 15, 27, + 31, 6, 30, 2, 25, 5, 23, 7, 29, 1] + col_idx = np.argwhere(np.isin(cal_vis.meta["isc"], isc_10_3)).ravel() + cal_vis.meta["offset"] = flare_position + vis10_3 = cal_vis[col_idx] + + imsize = [129, 129] * u.pixel # number of pixels of the map to reconstruct + pixel = [2, 2] * u.arcsec / u.pixel # pixel size in arcsec + + vis_tr = TimeRange(vis.meta["time_range"]) + roll, solo_xyz, pointing = get_hpc_info(vis_tr.start, vis_tr.end) + solo = HeliographicStonyhurst(*solo_xyz, obstime=vis_tr.center, + representation_type="cartesian") + + clean_map, model_map, resid_map = vis_clean( + vis10_3, imsize, pixel_size=pixel, gain=0.1, niter=200, + clean_beam_width=20 * u.arcsec) + comments.append(f"clean map with {len(col_idx)} visibilities") + comments.append(f"clean gain: {0.1}, niter: {200}, " + f"clean beam width: {20 * u.arcsec}") + comments.append(f"det: {', '.join(sorted(vis10_3.meta['vis_labels']))}") + + map_with_erange = clean_map.data[np.newaxis, ...] + map_with_erange[0, :, :] = clean_map.data + fp_hp = flare_position.transform_to(Helioprojective(obstime=vis_tr.center, + observer=solo)) + header = make_stix_fitswcs_header(map_with_erange, fp_hp, + scale=pixel, + exposure=vis_tr.seconds, + rotation_angle=90 * u.deg + roll, + energy_range=energy_range, + ) + + header = fits.Header(header) + # Add comments + [header.add_comment(com) for com in comments] + + header["IMG_METH"] = ("clean", "STIX image reconstruction method used") + + maps.append((map_with_erange, header)) + + ppi = PeekPreviewImage(control=QTable(), data=preview_data, month=month, + energy=energies, maps=maps, + product_name_suffix=product_name_suffix, + parents=[parent, aux_ephemeris_path.name, cpd_path.name]) + + for f in img_processor.write_fits(ppi): + products.append(f) + images += len(ppi.maps) + message = "OK" + except Exception as e: + logger.error(e, stack_info=True) + status = False + message = str(e) + + data[i]['preview_start_UTC'] = peek_preview_start + data[i]['preview_end_UTC'] = peek_preview_end + data[i]['peek_preview_path'] = "test" + data[i]['_peek_preview_status'] = status + data[i]['_peek_preview_message'] = message + + if not keep_all_flares: + data.remove_rows(to_remove) + + logger.info(f'Flare images created for month {month} with {len(data)} flares, ' + f'{len(products)} peek previews created, with total {images} images') + + return products + + +class FlareListProduct(CountDataMixin, GenericProduct, L2Mixin): + + LEVEL = 'ANC' + TYPE = 'flarelist' + + def __init__(self, *, service_type=0, service_subtype=0, ssid, data, **kwargs): + super().__init__(service_type=0, service_subtype=0, + ssid=ssid, data=data, **kwargs) + self.level = FlareListProduct.LEVEL + self.type = FlareListProduct.TYPE + self.service_subtype = 0 + self.service_type = 0 + self._parent = set() + + @property + def parent(self): + return list(self._parent) + + @parent.setter + def parent(self, value): + self._parent.add(value) + + def enhance_from_product(self, in_prod: GenericProduct): + pass + + +class FlarelistSDC(FlareListProduct, FlareSOOPMixin): + """Flarelist product class for StixDataCenter flares. + + In ANC product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'sdc' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=2, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=2, data=data, **kwargs) + + self.name = FlarelistSDC.NAME + self.ssid = 2 + + self._start_datetime = datetime.combine(month, datetime.min.time()) + self._end_datetime = self.data['end_UTC'].max() if len(self.data) > 0 \ + else self._start_datetime + + @property + def utc_timerange(self): + return TimeRange(self._start_datetime, self._end_datetime) + + @property + def scet_timerange(self): + tr = self.utc_timerange + start = SCETime.from_string(Spice.instance.datetime_to_scet(tr.start)[2:]) + end = SCETime.from_string(Spice.instance.datetime_to_scet(tr.end)[2:]) + return SCETimeRange(start=start, end=end) + + def split_to_files(self): + return [self] + + @property + def dmin(self): + return (self.data['lc_peak'].sum(axis=1)).min().value if len(self.data) > 0 else np.nan + + @property + def dmax(self): + return (self.data['lc_peak'].sum(axis=1)).max().value if len(self.data) > 0 else np.nan + + @property + def exposure(self): + return self.data['duration'].min().to_value('s') if len(self.data) > 0 else np.nan + + @property + def max_exposure(self): + return self.data['duration'].max().to_value('s') if len(self.data) > 0 else np.nan + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'ANC' and service_type == 0 + and service_subtype == 0 and ssid == 2) + + +class FlarelistSDCLoc(FlarelistSDC, FlarePositionMixin): + """Flarelist product class for StixDataCenter flares. + + In ANC product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'sdcloc' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=3, + data=data, month=month, **kwargs) + + self.name = FlarelistSDCLoc.NAME + self.ssid = 3 + + def enhance_from_product(self, in_prod: GenericProduct): + pass + + @classmethod + def filter_flare_function(cls, col): + return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', + fallback=1000) + + @classmethod + def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): + super().add_flare_position(data, fido_client, + filter_function=cls.filter_flare_function, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + keep_all_flares=False, month=month) + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'ANC' and service_type == 0 + and service_subtype == 0 and ssid == 3) + + +class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeekPreviewMixin): + """Flarelist product class for StixDataCenter flares. + + In ANC product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'sdclocimg' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=4, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=4, + data=data, month=month, **kwargs) + + self.name = FlarelistSDCLocImg.NAME + self.ssid = 4 + + def enhance_from_product(self, in_prod: GenericProduct): + pass + + @classmethod + def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, + *, month=None): + super().add_peek_preview(data, energies, parent, fido_client, img_processor, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + aux_ephemeris_path_colname='aux_ephemeris_path', + cpd_path_colname='cpd_path', + product_name_suffix=FlarelistSDC.NAME, + keep_all_flares=False, month=month) + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'ANC' and service_type == 0 + and service_subtype == 0 and ssid == 4) diff --git a/stixcore/products/level3/flarelistproduct.py b/stixcore/products/level3/flarelistproduct.py index bb9542fd..5b90d175 100644 --- a/stixcore/products/level3/flarelistproduct.py +++ b/stixcore/products/level3/flarelistproduct.py @@ -1,5 +1,9 @@ +import numpy as np +from sunpy.time.timerange import TimeRange + +from stixcore.ephemeris.manager import Spice from stixcore.products.product import GenericProduct, L3Mixin -from stixcore.time.datetime import SCETimeRange +from stixcore.time.datetime import SCETime, SCETimeRange __all__ = ["FlareListProduct", "FlareOverviewImage"] @@ -15,17 +19,44 @@ def from_timerange(cls, timerange: SCETimeRange, *, flarelistparent: str = ""): pass -class FlareOverviewImage(FlareListProduct): +class PeekPreviewImage(FlareListProduct): PRODUCT_PROCESSING_VERSION = 1 - - def __init__(self, *, service_type=0, service_subtype=0, ssid=2, **kwargs): - self.name = "overview" - self.level = "L3" - self.type = "img" - self.ssid = ssid - self.service_subtype = service_subtype - self.service_type = service_type + Level = "L3" + Type = "sci" + Name = "peekpreviewimg" + + def __init__(self, control, data, energy, maps, parents, *, product_name_suffix="", **kwargs): + super().__init__(service_type=0, service_subtype=0, ssid=4, control=control, + data=data, energy=energy, **kwargs) + self.name = f"{PeekPreviewImage.Name}-{product_name_suffix}" + self.level = PeekPreviewImage.Level + self.type = PeekPreviewImage.Type + self.energy = energy + self.maps = maps + self.parents = parents + + self.add_additional_header_keyword(('NR_MAPS', len(maps) if maps else 0, + 'number of maps in file')) + + @property + def parent(self): + return np.atleast_1d(self.parents) + + @property + def utc_timerange(self): + return TimeRange(self.data["preview_start_UTC"][0], self.data["preview_end_UTC"][0]) + + @property + def scet_timerange(self): + tr = self.utc_timerange + start = SCETime.from_string(Spice.instance.datetime_to_scet(tr.start)[2:]) + end = SCETime.from_string(Spice.instance.datetime_to_scet(tr.end)[2:]) + return SCETimeRange(start=start, end=end) + + def split_to_files(self): + return [self] @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 2 + return (kwargs['level'] == PeekPreviewImage.Level and service_type == 0 + and service_subtype == 0 and ssid == 4) diff --git a/stixcore/products/lowlatency/quicklookLL.py b/stixcore/products/lowlatency/quicklookLL.py index fb283a59..db5b1645 100644 --- a/stixcore/products/lowlatency/quicklookLL.py +++ b/stixcore/products/lowlatency/quicklookLL.py @@ -1,9 +1,15 @@ +from pathlib import Path from collections import defaultdict -from stixcore.products.product import GenericProduct +import matplotlib.pyplot as plt +from stixpy.timeseries import quicklook # noqa This registers the STIX timeseries with sunpy +from sunpy.timeseries import TimeSeries + +from stixcore.products.level1.quicklookL1 import QLProduct +from stixcore.products.product import GenericProduct, L1Mixin from stixcore.time.datetime import SCETimeRange -__all__ = ["LightCurve", "FlareFlag"] +__all__ = ["LightCurve", "FlareFlag", "LightCurveL3"] class LightCurve(GenericProduct): @@ -33,6 +39,36 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "LL01" and service_type == 21 and service_subtype == 6 and ssid == 30 +class LightCurveL3(QLProduct, L1Mixin): + LEVEL = "LL03" + TYPE = "ql" + PRODUCT_PROCESSING_VERSION = 2 + NAME = "lightcurve" + + """"Low Latency Quick Look Light Curve data product. + + for Low Latency Processing + Level 3 format - svg chart + """ + def __init__(self, *, control, data, parent_file_path: Path, **kwargs): + super().__init__(service_type=21, service_subtype=6, + ssid=34, control=control, data=data, **kwargs) + self.name = LightCurveL3.NAME + self.level = LightCurveL3.LEVEL + self.type = LightCurveL3.TYPE + self.parent_file_path = parent_file_path + if "header" in kwargs: + self.fits_header = kwargs.get("header") + + def get_plot(self): + ql_lightcurves = TimeSeries(self.parent_file_path) + fig = plt.figure(figsize=(12, 5), layout="tight") + ax = ql_lightcurves.plot() + ax.set_xlabel("Time [UTC]") + fig.add_axes(ax) + return fig + + class FlareFlag(GenericProduct): """Low Latency Quick Look Flare Flag and Location data product. diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 265f3d00..371105ae 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -71,12 +71,12 @@ def read_qtable(file, hdu, hdul=None): `astropy.table.QTable` The corrected QTable with correct data types """ - qtable = QTable.read(file, hdu) + qtable = QTable.read(file, hdu, astropy_native=True) if hdul is None: hdul = fits.open(file) for col in hdul[hdu].data.columns: - if col.unit: + if col.unit and col.name in qtable.colnames: logger.debug(f"Unit present dtype correction needed for {col}") dtype = col.dtype @@ -89,7 +89,10 @@ def read_qtable(file, hdu, hdul=None): if hasattr(dtype, "subdtype"): dtype = dtype.base - qtable[col.name] = qtable[col.name].astype(dtype) + if col.coord_type != 'UTC': + qtable[col.name] = qtable[col.name].astype(dtype) + else: + qtable[col.name].format = "isot" return qtable @@ -228,13 +231,17 @@ def __call__(self, *args, **kwargs): pri_header.get("TIMESYS") hdul = fits.open(file_path) - control = read_qtable(file_path, hdu="CONTROL", hdul=hdul) - # Weird issue where time_stamp wasn't a proper table column? - if "time_stamp" in control.colnames: - ts = control["time_stamp"].value - control.remove_column("time_stamp") - control["time_stamp"] = ts * u.s + if "CONTROL" in hdul: + control = read_qtable(file_path, hdu="CONTROL", hdul=hdul) + + # Weird issue where time_stamp wasn't a proper table column? + if "time_stamp" in control.colnames: + ts = control["time_stamp"].value + control.remove_column("time_stamp") + control["time_stamp"] = ts * u.s + else: + control = QTable() data = read_qtable(file_path, hdu="DATA", hdul=hdul) @@ -250,7 +257,8 @@ def __call__(self, *args, **kwargs): service_subtype = 6 ssid = 34 - if level not in ["LB", "LL01"]: + if level not in ["LB", "LL01"] and 'timedel' in data.colnames \ + and 'time' in data.colnames: data["timedel"] = SCETimeDelta(data["timedel"]) offset = SCETime.from_float(pri_header["OBT_BEG"] * u.s) @@ -262,7 +270,7 @@ def __call__(self, *args, **kwargs): data["time"] = offset + data["time"] energies = None - if level == "L1": + if level in ["L1", "ANC"] and "ENERGIES" in hdul: try: energies = read_qtable(file_path, hdu="ENERGIES") except KeyError: @@ -276,7 +284,7 @@ def __call__(self, *args, **kwargs): start=SCETime.from_float(row[1]), end=SCETime.from_float(row[2]) ) except KeyError: - logger.warn(f"no IDB data found in FITS: {file_path}") + logger.warning(f"no IDB data found in FITS: {file_path}") Product = self._check_registered_widget( level=level, @@ -288,20 +296,15 @@ def __call__(self, *args, **kwargs): energies=energies, ) - p = Product( - level=level, - service_type=service_type, - service_subtype=service_subtype, - ssid=ssid, - control=control, - data=data, - energies=energies, - idb_versions=idb_versions, - raw=raw, - parent=parent, - comment=comment, - history=history, - ) + month = None + if hasattr(Product, "TYPE") and Product.TYPE == "flarelist": + month = datetime.fromisoformat(pri_header["DATE-BEG"]).date() + + p = Product(level=level, service_type=service_type, + service_subtype=service_subtype, + ssid=ssid, control=control, + data=data, energies=energies, idb_versions=idb_versions, + raw=raw, parent=parent, comment=comment, history=history, month=month) # store the old fits header for later reuse if isinstance(p, (L1Mixin, L2Mixin)): @@ -519,11 +522,13 @@ def scet_timerange(self): @property def raw(self): - return np.unique(self.control["raw_file"]).tolist() + return np.unique(self.control["raw_file"]).tolist() \ + if "raw_file" in self.control.colnames else [] @property def parent(self): - return np.unique(self.control["parent"]).tolist() + return np.unique(self.control["parent"]).tolist() \ + if "parent" in self.control.colnames else [] @property def dmin(self): diff --git a/stixcore/util/util.py b/stixcore/util/util.py index 6f020b48..8dd2446a 100644 --- a/stixcore/util/util.py +++ b/stixcore/util/util.py @@ -1,12 +1,15 @@ import re from pathlib import Path +from stixpy.net.client import StixQueryResponse + __all__ = [ "get_complete_file_name", "get_incomplete_file_name", "get_complete_file_name_and_path", "get_incomplete_file_name_and_path", "is_incomplete_file_name", + "url_to_path" ] @@ -33,3 +36,9 @@ def get_incomplete_file_name_and_path(path): def get_incomplete_file_name(name): # see https://github.com/i4Ds/STIXCore/issues/350 return re.sub(r"_V([0-9]+)([\._])", r"_V\1U\2", name) + + +def url_to_path(fido_res: StixQueryResponse): + if 'url' in fido_res.columns: + fido_res['path'] = [Path(url.replace("file:", "")) + if url.startswith("file:") else None for url in fido_res['url']] From ca8a9d82f47a226a1208a1d90d08f65ba846b771 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 23 Jun 2025 12:02:43 +0200 Subject: [PATCH 04/11] change flarelist to L3 --- processed.sqlite | Bin 0 -> 24576 bytes stixcore/data/stixcore.ini | 1 + stixcore/io/FlareListManager.py | 288 +++++++++++++++++- stixcore/io/fits/processors.py | 3 +- stixcore/processing/AspectANC.py | 16 + stixcore/processing/FLtoFL.py | 10 +- .../{FlareListANC.py => FlareListL3.py} | 14 +- stixcore/processing/pipeline_daily.py | 62 ++-- stixcore/processing/publish.py | 2 +- .../products/{ANC => level3}/flarelist.py | 208 +++++++++++-- stixcore/products/level3/flarelistproduct.py | 4 +- stixcore/products/product.py | 2 +- 12 files changed, 542 insertions(+), 68 deletions(-) create mode 100644 processed.sqlite rename stixcore/processing/{FlareListANC.py => FlareListL3.py} (91%) rename stixcore/products/{ANC => level3}/flarelist.py (74%) diff --git a/processed.sqlite b/processed.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..9c99a5ac2e3ea722f0283d14532dc885a1bced40 GIT binary patch literal 24576 zcmeI2O>YxN7{_fVvEwARAprs;63d|yh-5u)`x+~4N?nv<0u82sda&HYsFp$!ViQ3R zm26c-YCk{^{RBO5w7a*n zQK{7`o27fzdaVSfn-4eOO?9*1SEHBL^DD)?wz~FS{*E@_dhPn!gf`^pbNSHgE^L?Y zR~EFb%6?^QL90LBg_HftUah*by`Z(6RJTjJ<@%==Z^YuWmo7!`*2|x4Rca5mAZsht zDi0o3w!ydmbmYKUapm1YUJHEL{ztg-;uo=KVs>dM`ed&4n|&9KoL*`6ahs*5l{pa0YyL&_`eevo96fCqnTeL@KA>$pa>`eihv@Z z2q*%IfFhs>C<2OrBA^Joxdi5-BTLafJ2d7ez}Ng(Ba-mX1e^39E zewzL|T}@v}zcu+Zes27mvBt!Mq>=o2G82C>bv5yE;?LBRso&mQEK0c|pa>`e|1p8a z{P>yU=c3I_w6?jiwX;#6_qNJ=m91*69?QZmu5z|kuRhAc4zO&!QmbbxwQ>o*u9Rz) z690f^3p7g*VOazZoPEErvYs!j-YjNoJ6k)Y0xj|D7rh@{guf~Ac*GDcqSmq4IyYM9 zj8R(p2oarcJe!xOj)P8q?Tx;csJ95Ho{UmZqmb{0mSy0h}sVJxH$ltqr?N-rlCAha$%t};eim5aG zC5y~<$(qRVO7@gQb(WYqE1=G9m#l*I^Y;?f$J>vIfP_Dh%3F?@mZmXN#$8 zv+All=wT=(9=jQd>co1KgErb#hg<9jGP>%f<*CC+h}}W&>#6JXe+Dx>cE@F@gGwjV z-x(;_-w}+9*uf_2_QuB~suL>NPVd`c*HKF>Y>_Aj{gNb`2GLs$bKaAcKY+wi>be+mL@-&c%J+tX^rnE zp2c6r>Db;VjlGOMgCJ=~GdDdMX-|8bCs+RQ#8;@w!+;(brVi=M1ZUo%1;7NqJ^4OV z1A?)r+w9uS%Ur~$h^dPV7g;(X21B%&J1!C55%jiV_`(b5;0x2^2k=ps*gB?;P3&gw zm^6GxG8~BD6A^HLuhYr*bozWqvI!yG^raUcaShvLhHg=t+N7Dw$ix@A?eK;ZYP=G} z7g~Ua&);}EIDKPg0H5I!q%+g7F>NfTCnBx!w3*>u-{B*wpa=O=62AcgRexhxVt~))1WQ1E(hx9!T*J{} z*$gpg#L2-1T9ZvN14#ZFl-~d{T}E|S$}lV_f$<>*_*_`ffc%s6U3A3XOPlhyk7?2^x^U2p=%OTm7ZzyRUB z5cEL~u^Y>=AqIFBD`-G$Bn~XkKz7y24t+No0tTQNVBNvUFrW%WhZqnaiNP>D!r#a7 z&qg7=eSK}tZ-CjZ0poySz(~yV|A~>WBI)l_&8f?iqsbq}zf4?-kB|NcAF4wUPz3%Z z0?pjKw6yw2ZUzf>!=VSl&Z1738`8ZU8HHZVHDFc3q7FgL+?-5&@&^$?e4zz+`26=7 zh7M$WF6}a#C-sa1zOyp%$xmej@r4%P;R|s~-(UNf=~AR)!W@R1xib>+9YHQGhA+H; z4nEQK0H}S~#g^_kg!0CFS|+~GZKt&sA7(8^5MO8k9zK8L?JRTpHbwYrAGKUUb=bvY zJFp@27i literal 0 HcmV?d00001 diff --git a/stixcore/data/stixcore.ini b/stixcore/data/stixcore.ini index ad37b228..38aba050 100644 --- a/stixcore/data/stixcore.ini +++ b/stixcore/data/stixcore.ini @@ -15,6 +15,7 @@ parallel_batchsize_L0 = 300 parallel_batchsize_L1 = 300 parallel_batchsize_L2 = 100 flareid_sdc_lut_file = ./stixcore/data/publish/flarelist_sdc_lut.csv +flareid_sc_lut_file = ./stixcore/data/publish/flarelist_sc_lut.csv [Publish] db_file = ./stixcore/data/publish/published.sqlite include_levels = L0 diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index 60e05e3c..e01c6149 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -12,13 +12,13 @@ from astropy.time import Time from stixcore.config.config import CONFIG -from stixcore.products.ANC.flarelist import FlarelistSDC +from stixcore.products.level3.flarelist import FlarelistSC, FlarelistSDC from stixcore.products.product import Product from stixcore.util.logging import get_logger from stixcore.util.singleton import Singleton from stixcore.util.util import url_to_path -__all__ = ["FlareListManager", "SDCFlareListManager"] +__all__ = ["FlareListManager", "SDCFlareListManager", "SCFlareListManager"] logger = get_logger(__name__) @@ -41,6 +41,270 @@ def productCls(self): return self._product_cls +class SCFlareListManager(FlareListManager, metaclass=Singleton): + """Manages a local copy of the flarelist provided by STIXCore or runs the flare detection + + TODO + """ + + def __init__(self, file, fido_client, update=False): + """Creates the manager by pointing to the flarelist file (csv) and setting the update + strategy. + + Parameters + ---------- + file : Path + points to the csv file + update : bool, optional + Update strategy: is the flarelist file updated via API?, by default False + """ + self.file = file + self.update = update + self._product_cls = FlarelistSC + self.fido_client = fido_client + self._flarelist = self.read_flarelist() + + def __str__(self) -> str: + return (f"{self.flarelistname}: file: {self.file} update: " + f"{self.update} size: {len(self.flarelist)}") + + def update_list(self): + """Updates the flarelist file via api request. + + Will create a new file if not available or do a incremental update otherwise, + using the last entry time stamp. + """ + self.flarelist = SDCFlareListManager.read_flarelist(self.file, update=self.update) + + def read_flarelist(self): + """Reads or creates the LUT of all STIXCore flares. + + On creation or update a flare detection is run on the STIXCore data + to get the information and persists as a LUT locally. + + Parameters + ---------- + file : Path + path the to LUT file. + update : bool, optional + should the LUT be updated at start up?, by default False + + Returns + ------- + Table + the table of flares. + """ + if self.update or not self.file.exists(): + last_date = datetime(2020, 1, 1, 0, 0, 0) + last_date = datetime(2025, 4, 12, 0, 0, 0) + # only run flare detection for time oldet than 7 days + today = datetime.now() - timedelta(days=7) + flare_df_lists = [] + if self.file.exists(): + old_list = pd.read_csv(self.file, keep_default_na=True, na_values=['None']) + mds = old_list['start_UTC'].max() + try: + last_date = datetime.strptime(mds, '%Y-%m-%dT%H:%M:%S') + except ValueError: + last_date = datetime.strptime(mds, '%Y-%m-%dT%H:%M:%S.%f') + flare_df_lists = [old_list] + last_date = last_date.replace(hour=0, minute=0, second=0, microsecond=0) + if not self.file.parent.exists(): + logger.info(f'path not found: {self.file.parent} creating dir') + self.file.parent.mkdir(parents=True, exist_ok=True) + + try: + while (last_date < today): + # run flare detection for batches of 7 days + # 2 days overlap to ensure no flares are missed + start = last_date - timedelta(days=2) + end = last_date + timedelta(days=7) + + ql_lc_files = self.fido_client.search(a.Time(start, end), + a.Instrument.stix, + a.stix.DataProduct.ql_lightcurve) + try: + if len(ql_lc_files) > 0: + ql_lc_files.filter_for_latest_version() + url_to_path(ql_lc_files) + except Exception as e: + logger.error(f'Error filtering for latest version of lightcurve files: {e}') + ql_lc_files = [] + + ql_bg_files = self.fido_client.search(a.Time(start, end), + a.Instrument.stix, + a.stix.DataProduct.ql_background) + if len(ql_bg_files) > 0: + ql_bg_files.filter_for_latest_version() + url_to_path(ql_bg_files) + logger.info(f'flare detection chunk: {start.isoformat()}/{end.isoformat()} ' + f'with {len(ql_lc_files)} lightcurve files and ' + f'{len(ql_bg_files)} background files') + # flares = stixpy.detect_flares(start, end, + # ql_lc_files=ql_lc_files, + # ql_bg_files=ql_bg_files) + + flares = [] + if len(flares) > 0: + flare_df_lists.append(pd.DataFrame(flares)) + logger.info(f'found {len(flares)} flares') + last_date += timedelta(days=7) + + except Exception: + logger.error("FLARE DETECTION ERROR", exc_info=True) + + full_flare_list = flare_df_lists + # full_flare_list = pd.concat(flare_df_lists) + + # full_flare_list.drop_duplicates(inplace=True) + # full_flare_list.sort_values(by="peak_UTC", inplace=True) + # full_flare_list.reset_index(inplace=True, drop=True) + logger.info(f'write total {len(full_flare_list)} flares to local storage') + # full_flare_list.to_csv(self.file, index_label=False) + else: + logger.info(f"read flare list from {self.file}") + full_flare_list = pd.read_csv(self.file, keep_default_na=True, na_values=['None']) + + return full_flare_list + + @staticmethod + def filter_flare_function(col): + return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sc_min_count', + fallback=1000) + + def get_data(self, *, start, end, fido_client): + month_data = self.flarelist[(self.flarelist['start_UTC'] >= start.isoformat()) & + (self.flarelist['start_UTC'] < end.isoformat())] + + if len(month_data) == 0: + return None, None, None + + mt = QTable(month_data.to_numpy(), names=month_data.columns) + data = QTable() + control = QTable() + energy = QTable() + + data['flare_id'] = Column(mt['flare_id'].astype(int), + description=f'unique flare id for flarelist {self.flarelistname}') + data['start_UTC'] = Column(0, description="start time of flare") + data['start_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['start_UTC']] + data['duration'] = Column(mt['duration'].astype(float) * u.s, + description="duration of flare") + data['end_UTC'] = Column(0, description="end time of flare") + data['end_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['end_UTC']] + data['peak_UTC'] = Column(0, description="flare peak time") + data['peak_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['peak_UTC']] + data['att_in'] = Column(mt['att_in'].astype(bool), + description="was attenuator in during flare") + data['bkg_baseline'] = Column(mt['LC0_BKG'] * u.ct, + description='background baseline at 4-10 keV') + data['GOES_class'] = Column(mt['GOES_class'].astype(str), + description="GOES class of the GOES XRS data at time of flare" + " - not derived from STIX data. Do not use when " + "flare isn't visible to Earth") + data['goes_min_class_est'] = Column(mt['goes_estimated_min_class'].astype(str), + description="min GOES class estimate derived " + "from STIX data") + data['goes_max_class_est'] = Column(mt['goes_estimated_max_class'].astype(str), + description="max GOES class estimate derived " + "from STIX data") + data['goes_mean_class_est'] = Column(mt['goes_estimated_mean_class'].astype(str), + description="mean GOES class estimate derived " + "from STIX data") + + data['GOES_flux'] = Column(mt['GOES_flux'].astype(float) * u.W / u.m**2, + description="GOES flux of the GOES XRS data at time of flare" + "- not derived from STIX data. Do not use when the " + "flare isn't visible to Earth") + data['goes_min_flux_est'] = Column(mt['goes_estimated_min_flux'].astype(float) * + u.W / u.m**2, + description="min GOES flux estimate " + "derived from STIX data") + data['goes_max_flux_est'] = Column(mt['goes_estimated_max_flux'].astype(float) * + u.W / u.m**2, + description="max GOES flux estimate " + "derived from STIX data") + data['goes_mean_flux_est'] = Column(mt['goes_estimated_mean_flux'].astype(float) * + u.W / u.m**2, + description="mean GOES flux estimate " + "derived from STIX data") + + # data['cfl_x'] = Column(mt['CFL_X_arcsec'].astype(float) * u.arcsec, + # description="corse flare location in x direction provided by" + # "onboard algorithm. (0,0) represents disk center") + # data['cfl_y'] = Column(mt['CFL_Y_arcsec'].astype(float) * u.arcsec, + # description="corse flare location in y direction provided by" + # "onboard algorithm. (0,0) represents disk center") + + data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, + mt['LC1_PEAK_COUNTS_4S'].value, + mt['LC2_PEAK_COUNTS_4S'].value, + mt['LC3_PEAK_COUNTS_4S'].value, + mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), + description="counts in 4s peak window from quicklook lightcurve", + dtype=np.int64) + + data['lc_bgk_peak'] = Column((np.vstack((mt['LC0_BKG_COUNTS_4S'].value, + mt['LC1_BKG_COUNTS_4S'].value, + mt['LC2_BKG_COUNTS_4S'].value, + mt['LC3_BKG_COUNTS_4S'].value, + mt['LC4_BKG_COUNTS_4S'].value)) + .T * u.ct).astype(int), + description="background counts in 4s peak window" + "from quicklook lightcurve", + dtype=np.int64) + + data['energy_index'] = Column(0, description="energy band index", dtype=np.int8) + + data.add_index('flare_id') + + # add energy axis for the lightcurve peek time data for each flare + # the energy bins are taken from the daily ql-lightcurve products + # as the definition of the lc energy chanel's are will change only very seldom + # the ql-lightcurve products assume a constant definition for an entire day. + # So we do the lookup also just grouped by peak day in order to save file lookups + + energy_look_up = {} + data['peak_day'] = [d.datetime.day for d in data['peak_UTC']] + data_by_day = data.group_by('peak_day') + + for day, flares in zip(data_by_day.groups.keys, data_by_day.groups): + time = flares['peak_UTC'][0] + lc_data = fido_client.search(a.Time(time, time), + a.Instrument.stix, + a.stix.DataProduct.ql_lightcurve) + lc_data.filter_for_latest_version() + url_to_path(lc_data) + + if len(lc_data) == 0: + logger.warning(f'No lightcurve data found for flare at time {time}') + continue + lc = Product(lc_data['path'][0]) + + energy_table_hash = frozenset(pd.core.util.hashing.hash_array(lc.energies.as_array())) + + # add the energy table to the energy table list if not already + # present and define a new index number + if energy_table_hash not in energy_look_up: + e_idx = len(energy_look_up.keys()) + energy_look_up[energy_table_hash] = e_idx + lc.energies["index"] = Column(e_idx, description='energy edge table index', + dtype=np.int8) + energy = vstack([energy, lc.energies]) + if e_idx > 0: + logger.warning(f'multiple energy ql-lc tables found for month {start}') + + # add the energy index to the flare data to all flares of the same day + # https://docs.astropy.org/en/latest/table/modify_table.html#caveats + replace = data.loc[flares['flare_id']] + replace['energy_index'] = energy_look_up[energy_table_hash] + data.loc[flares['flare_id']] = replace + + del data['peak_day'] + + return data, control, energy + + class SDCFlareListManager(FlareListManager, metaclass=Singleton): """Manages a local copy of the operational flarelist provided by stix data datacenter @@ -212,18 +476,20 @@ def get_data(self, *, start, end, fido_client): mt['LC1_PEAK_COUNTS_4S'].value, mt['LC2_PEAK_COUNTS_4S'].value, mt['LC3_PEAK_COUNTS_4S'].value, - mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), + mt['LC4_PEAK_COUNTS_4S'].value)) + .T * u.ct).astype(int), description="counts in 4s peak window from quicklook lightcurve", dtype=np.int64) - data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, - mt['LC1_PEAK_COUNTS_4S'].value, - mt['LC2_PEAK_COUNTS_4S'].value, - mt['LC3_PEAK_COUNTS_4S'].value, - mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), - description="background counts in 4s peak window" - "from quicklook lightcurve", - dtype=np.int64) + data['lc_bgk_peak'] = Column((np.vstack((mt['LC0_BKG_COUNTS_4S'].value, + mt['LC1_BKG_COUNTS_4S'].value, + mt['LC2_BKG_COUNTS_4S'].value, + mt['LC3_BKG_COUNTS_4S'].value, + mt['LC4_BKG_COUNTS_4S'].value)) + .T * u.ct).astype(int), + description="background counts in 4s peak window" + "from quicklook lightcurve", + dtype=np.int64) data['energy_index'] = Column(0, description="energy band index", dtype=np.int8) diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 01e17ec0..53c15f64 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -1098,7 +1098,8 @@ def generate_filename(self, product, *, version=0, suffix='.svg'): Path a Path object with full name and path """ - p = Path(super().generate_filename(product=product, version=version, header=True)) + p = Path(super().generate_filename(product=product, version=version, + header=True, status='C')) return p.with_suffix(suffix).name def generate_primary_header(self, filename, product, *, version=0): diff --git a/stixcore/processing/AspectANC.py b/stixcore/processing/AspectANC.py index 84558870..7babd29a 100644 --- a/stixcore/processing/AspectANC.py +++ b/stixcore/processing/AspectANC.py @@ -39,6 +39,22 @@ def __init__(self, source_dir: Path, output_dir: Path): self.source_dir = Path(source_dir) self.output_dir = Path(output_dir) + def get_processing_files(self, phs) -> list[Path]: + """Returns a list of all files which should be processed by this step. + + Returns + ------- + list[Path] + a list of fits files candidates + """ + hk_in_files = [] + + for fc in self.find_processing_candidates(): + tr = self.test_for_processing(fc, phs) + if tr == TestForProcessingResult.Suitable: + hk_in_files.append(fc) + return hk_in_files + def find_processing_candidates(self) -> list[Path]: """Performs file pattern search in the source directory diff --git a/stixcore/processing/FLtoFL.py b/stixcore/processing/FLtoFL.py index 34beccd4..a6cd9b88 100644 --- a/stixcore/processing/FLtoFL.py +++ b/stixcore/processing/FLtoFL.py @@ -13,8 +13,8 @@ SingleProductProcessingStepMixin, TestForProcessingResult, ) -from stixcore.products.ANC.flarelist import ( - FlareListProduct, +from stixcore.products.level3.flarelist import ( + FlareList, FlarePeekPreviewMixin, FlarePositionMixin, FlareSOOPMixin, @@ -30,7 +30,7 @@ class FLtoFL(SingleProductProcessingStepMixin): - """Processing step from a HK L1 fits file to a solo_ANC_stix-asp-ephemeris*.fits file. + """Processing step from enhance monthly flare list files to next processing steep """ INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits" @@ -84,8 +84,8 @@ def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]: return fl_to_process def test_for_processing(self, candidate: Path, - product_in: FlareListProduct, - product_out: FlareListProduct, + product_in: FlareList, + product_out: FlareList, phm: ProcessingHistoryStorage) -> TestForProcessingResult: """_summary_ diff --git a/stixcore/processing/FlareListANC.py b/stixcore/processing/FlareListL3.py similarity index 91% rename from stixcore/processing/FlareListANC.py rename to stixcore/processing/FlareListL3.py index 785bb129..97cbe9a0 100644 --- a/stixcore/processing/FlareListANC.py +++ b/stixcore/processing/FlareListL3.py @@ -12,17 +12,17 @@ SingleProductProcessingStepMixin, TestForProcessingResult, ) -from stixcore.products.ANC.flarelist import FlarePositionMixin, FlareSOOPMixin +from stixcore.products.level3.flarelist import FlarePositionMixin, FlareSOOPMixin from stixcore.soop.manager import SOOPManager from stixcore.util.logging import get_logger -__all__ = ['FlareListANC', 'FlarePositionMixin'] +__all__ = ['FlareListL3'] logger = get_logger(__name__) -class FlareListANC(SingleProductProcessingStepMixin): - """Processing step from a FlareListManager to monthly solo_ANC_stix-flarelist-*.fits file. +class FlareListL3(SingleProductProcessingStepMixin): + """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file. """ STARTDATE = date(2024, 1, 1) @@ -49,7 +49,7 @@ def find_processing_months(self, phs: ProcessingHistoryStorage) -> list[date]: a list of fits files candidates """ fl_months = list() - for month in pd.date_range(FlareListANC.STARTDATE, date.today(), freq='MS').date: + for month in pd.date_range(FlareListL3.STARTDATE, date.today(), freq='MS').date: if self.test_for_processing(month, phs) == TestForProcessingResult.Suitable: fl_months.append(month) return fl_months @@ -75,7 +75,7 @@ def test_for_processing(self, month: date, self.flm.productCls.LEVEL, self.flm.productCls.TYPE, self.flm.productCls.get_cls_processing_version(), - str(month), FlareListANC.STARTDATE) + str(month), FlareListL3.STARTDATE) # found already in the processing history if wp: @@ -93,7 +93,7 @@ def test_for_processing(self, month: date, def process_fits_files(self, months: list[date], *, soopmanager: SOOPManager, spice_kernel_path: Path, processor, config) -> list[Path]: """Performs the processing (expected to run in a dedicated python process) from a - list of solo_L1_stix-hk-maxi_*.fits into solo_ANC_stix-asp-ephemeris*.fits files. + list of time periods (months) and creates a FITS file for each one. Parameters ---------- diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index 7c2e2254..3960df5c 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -7,6 +7,8 @@ from datetime import date, timedelta from concurrent.futures import ProcessPoolExecutor +from stixpy.net.client import STIXClient + from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager from stixcore.io.fits.processors import ( @@ -15,17 +17,24 @@ FitsL3Processor, PlotProcessor, ) -from stixcore.io.FlareListManager import SDCFlareListManager +from stixcore.io.FlareListManager import SCFlareListManager, SDCFlareListManager from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage from stixcore.io.RidLutManager import RidLutManager from stixcore.processing.AspectANC import AspectANC -from stixcore.processing.FlareListANC import FlareListANC +from stixcore.processing.FlareListL3 import FlareListL3 from stixcore.processing.FLtoFL import FLtoFL from stixcore.processing.LL import LL03QL from stixcore.processing.pipeline import PipelineStatus -from stixcore.processing.SingleStep import SingleProcessingStepResult, TestForProcessingResult -from stixcore.products.ANC.flarelist import FlarelistSDCLoc, FlarelistSDCLocImg +from stixcore.processing.SingleStep import SingleProcessingStepResult from stixcore.products.level1.quicklookL1 import LightCurve +from stixcore.products.level3.flarelist import ( + FlarelistSC, + FlarelistSCLoc, + FlarelistSCLocImg, + FlarelistSDC, + FlarelistSDCLoc, + FlarelistSDCLocImg, +) from stixcore.products.lowlatency.quicklookLL import LightCurveL3 from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger @@ -197,6 +206,13 @@ def run_daily_pipeline(args): Path(CONFIG.get("Paths", "fits_archive")) + fido_url = CONFIG.get('Paths', 'fido_search_url', + fallback='https://pub099.cs.technik.fhnw.ch/data/fits') + fido_client = STIXClient(source=fido_url) + + flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sc_lut_file')) + SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True) + flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sdc_lut_file')) SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) @@ -231,11 +247,14 @@ def run_daily_pipeline(args): aspect_anc_processor = AspectANC(fits_in_dir, fits_out_dir) - flarelist_sdc = FlareListANC(SDCFlareListManager.instance, fits_out_dir) + flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) + flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir) fl_to_fl = FLtoFL(fits_in_dir, fits_out_dir, - products_in_out=[ # (FlarelistSDC, FlarelistSDCLoc), - (FlarelistSDCLoc, FlarelistSDCLocImg)], + products_in_out=[(FlarelistSDC, FlarelistSDCLoc), + (FlarelistSDCLoc, FlarelistSDCLocImg), + (FlarelistSC, FlarelistSCLoc), + (FlarelistSCLoc, FlarelistSCLocImg)], cadence=timedelta(seconds=1)) ll03ql = LL03QL(fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, @@ -246,23 +265,20 @@ def run_daily_pipeline(args): l3_fits_writer = FitsL3Processor(fits_out_dir) anc_fits_writer = FitsANCProcessor(fits_out_dir) - # should be done later: internally + # hk_in_files = aspect_anc_processor.get_processing_files(phs) hk_in_files = [] - candidates = list() # aspect_anc_processor.find_processing_candidates() - v_candidates = aspect_anc_processor.get_version(candidates, version="latest") - for fc in v_candidates: - tr = aspect_anc_processor.test_for_processing(fc, phs) - if tr == TestForProcessingResult.Suitable: - hk_in_files.append(fc) - fl_months = flarelist_sdc.find_processing_months(phs) - fl_months = [] + fl_sdc_months = flarelist_sdc.find_processing_months(phs) + # fl_sdc_months = [] + + # fl_sc_months = flarelist_sc.find_processing_months(phs) + fl_sc_months = [] - ll_candidates = ll03ql.get_processing_files(phs) + # ll_candidates = ll03ql.get_processing_files(phs) ll_candidates = [] fl_to_fl_files = fl_to_fl.get_processing_files(phs) - # flsdc_to_flcdspos_files = [] + # fl_to_fl_files = [] # all processing files should be terminated before the next step as the different # processing steeps might create new candidates @@ -275,10 +291,16 @@ def run_daily_pipeline(args): processor=l2_fits_writer, config=CONFIG)) - jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_months, + jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_sdc_months, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l3_fits_writer, + config=CONFIG)) + + jobs.append(executor.submit(flarelist_sc.process_fits_files, fl_sc_months, soopmanager=SOOPManager.instance, spice_kernel_path=Spice.instance.meta_kernel_path, - processor=anc_fits_writer, + processor=l3_fits_writer, config=CONFIG)) # TODO a owen processing step for each flarelist file? diff --git a/stixcore/processing/publish.py b/stixcore/processing/publish.py index f9bd168f..54d1d478 100644 --- a/stixcore/processing/publish.py +++ b/stixcore/processing/publish.py @@ -587,7 +587,7 @@ def publish_fits_to_esa(args): "--include_levels", help="what levels should be published", type=str, - default=CONFIG.get("Publish", "include_levels", fallback="L0, L1, L2, ANC"), + default=CONFIG.get("Publish", "include_levels", fallback="L0, L1, L2, ANC, LL03"), ) parser.add_argument( diff --git a/stixcore/products/ANC/flarelist.py b/stixcore/products/level3/flarelist.py similarity index 74% rename from stixcore/products/ANC/flarelist.py rename to stixcore/products/level3/flarelist.py index 5b0ab3db..f4a73a5c 100644 --- a/stixcore/products/ANC/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -34,7 +34,8 @@ from stixpy.map.stix import STIXMap # noqa __all__ = ['FlarelistSDC', 'FlarePositionMixin', 'FlareSOOPMixin', - 'FlareListProduct', 'FlarelistSDCLocImg', 'FlarePeekPreviewMixin'] + 'FlareList', 'FlarelistSDCLocImg', 'FlarePeekPreviewMixin', + 'FlarelistSC', 'FlarelistSCLoc', 'FlarelistSCLocImg'] logger = get_logger(__name__) @@ -80,7 +81,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, data['flare_position'] = [SkyCoord(0, 0, frame="icrs", unit="deg") for i in range(0, len(data))] - data['aux_ephemeris_path'] = Column(" " * 500, dtype=str, + data['anc_ephemeris_path'] = Column(" " * 500, dtype=str, description='TDB') data['cpd_path'] = Column(" " * 500, dtype=str, description='TDB') @@ -124,7 +125,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, data[i]['_position_message'] = "no ephemeris data found" no_ephemeris += 1 continue - data[i]['aux_ephemeris_path'] = anc_res["path"][0] + data[i]['anc_ephemeris_path'] = anc_res["path"][0] if start_time.datetime.hour < 2: start_time = start_time - 2 * u.hour @@ -227,14 +228,23 @@ def add_soop(self, data, *, class FlarePeekPreviewMixin: - """_summary_ + """ Mixin class to add peek preview images to flare list products. + This class provides a method to generate and add peek preview images + to the flare list data. The images are generated based on the + flare's peak time, start time, and end time, using the STIXPy library + for visibility calculations and image reconstruction. + The generated images are stored in the 'peek_preview_path' column of the data. + The method also updates the status and message columns to indicate + the success or failure of the image generation process. + + Currently the images are created for two energy ranges: 4-20 keV and 20-120 keV. """ @classmethod def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, peek_time_colname='peak_UTC', start_time_colname='start_UTC', end_time_colname='end_UTC', - aux_ephemeris_path_colname='aux_ephemeris_path', + anc_ephemeris_path_colname='anc_ephemeris_path', cpd_path_colname='cpd_path', product_name_suffix='fl', keep_all_flares=True, @@ -255,19 +265,21 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p images = 0 for i, row in enumerate(data): + peak_time = row[peek_time_colname] row[start_time_colname] row[end_time_colname] - aux_ephemeris_path = Path(row[aux_ephemeris_path_colname]) + anc_ephemeris_path = Path(row[anc_ephemeris_path_colname]) cpd_path = Path(row[cpd_path_colname]) status = False + message = "" peek_preview_start = row[peek_time_colname] peek_preview_end = row[peek_time_colname] - if aux_ephemeris_path.exists() and cpd_path.exists(): + if anc_ephemeris_path.exists() and cpd_path.exists(): try: status = True # do the imaging with stixpy @@ -286,7 +298,7 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p cpd_sci = STIXPYProduct(cpd_path) time_range_sci = [peek_preview_start, peek_preview_end] maps = [] - for energy_range in [[4, 18], [18, 120]] * u.keV: + for energy_range in [[4, 20], [20, 120]] * u.keV: # flare_position = preview_data['flare_position'][0] # flare_position = [0, 0] * u.arcsec comments = [] @@ -345,7 +357,7 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p ppi = PeekPreviewImage(control=QTable(), data=preview_data, month=month, energy=energies, maps=maps, product_name_suffix=product_name_suffix, - parents=[parent, aux_ephemeris_path.name, cpd_path.name]) + parents=[parent, anc_ephemeris_path.name, cpd_path.name]) for f in img_processor.write_fits(ppi): products.append(f) @@ -371,36 +383,61 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p return products -class FlareListProduct(CountDataMixin, GenericProduct, L2Mixin): +class FlareList(CountDataMixin, GenericProduct, L2Mixin): + """FlareList product class. + This class represents a flare list product in the STIX data processing pipeline. + It inherits from GenericProduct and L2Mixin, and provides methods to handle flare data. + It is used to store flare data, and can be enhanced + with flare positions and SOOP information.""" - LEVEL = 'ANC' + LEVEL = 'L3' TYPE = 'flarelist' def __init__(self, *, service_type=0, service_subtype=0, ssid, data, **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=ssid, data=data, **kwargs) - self.level = FlareListProduct.LEVEL - self.type = FlareListProduct.TYPE + self.level = FlareList.LEVEL + self.type = FlareList.TYPE self.service_subtype = 0 self.service_type = 0 self._parent = set() @property def parent(self): + """ Returns the parent(s) of the flare list product. + + Returns + ------- + list + A list of parent product names. + """ return list(self._parent) @parent.setter def parent(self, value): + """ Sets the parent of the flare list product. + + Parameters + ---------- + value : str + The name of the parent product to be added. + """ self._parent.add(value) def enhance_from_product(self, in_prod: GenericProduct): - pass + """ Enhances the flare list product by adding more data. + + Parameters + ---------- + in_prod : GenericProduct + The input product from which to enhance the flare list product. + """ -class FlarelistSDC(FlareListProduct, FlareSOOPMixin): +class FlarelistSDC(FlareList, FlareSOOPMixin): """Flarelist product class for StixDataCenter flares. - In ANC product format. + In L3 product format. """ PRODUCT_PROCESSING_VERSION = 2 NAME = 'sdc' @@ -448,7 +485,7 @@ def max_exposure(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'ANC' and service_type == 0 + return (kwargs['level'] == 'L3' and service_type == 0 and service_subtype == 0 and ssid == 2) @@ -487,7 +524,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'ANC' and service_type == 0 + return (kwargs['level'] == 'L3' and service_type == 0 and service_subtype == 0 and ssid == 3) @@ -517,12 +554,143 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p peek_time_colname='peak_UTC', start_time_colname='start_UTC', end_time_colname='end_UTC', - aux_ephemeris_path_colname='aux_ephemeris_path', + anc_ephemeris_path_colname='anc_ephemeris_path', cpd_path_colname='cpd_path', product_name_suffix=FlarelistSDC.NAME, keep_all_flares=False, month=month) @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'ANC' and service_type == 0 + return (kwargs['level'] == 'L3' and service_type == 0 and service_subtype == 0 and ssid == 4) + + +class FlarelistSC(FlareList, FlareSOOPMixin): + """Flarelist product class for STIXCore flares. + + In L3 product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'sc' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=6, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=6, data=data, **kwargs) + + self.name = FlarelistSC.NAME + self.ssid = 6 + + self._start_datetime = datetime.combine(month, datetime.min.time()) + self._end_datetime = self.data['end_UTC'].max() if len(self.data) > 0 \ + else self._start_datetime + + @property + def utc_timerange(self): + return TimeRange(self._start_datetime, self._end_datetime) + + @property + def scet_timerange(self): + tr = self.utc_timerange + start = SCETime.from_string(Spice.instance.datetime_to_scet(tr.start)[2:]) + end = SCETime.from_string(Spice.instance.datetime_to_scet(tr.end)[2:]) + return SCETimeRange(start=start, end=end) + + def split_to_files(self): + return [self] + + @property + def dmin(self): + return (self.data['lc_peak'].sum(axis=1)).min().value if len(self.data) > 0 else np.nan + + @property + def dmax(self): + return (self.data['lc_peak'].sum(axis=1)).max().value if len(self.data) > 0 else np.nan + + @property + def exposure(self): + return self.data['duration'].min().to_value('s') if len(self.data) > 0 else np.nan + + @property + def max_exposure(self): + return self.data['duration'].max().to_value('s') if len(self.data) > 0 else np.nan + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'L3' and service_type == 0 + and service_subtype == 0 and ssid == 6) + + +class FlarelistSCLoc(FlarelistSC, FlarePositionMixin): + """Flarelist product class for STIXCore flares. + + In L3 product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'scloc' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=7, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=7, + data=data, month=month, **kwargs) + + self.name = FlarelistSCLoc.NAME + self.ssid = 7 + + def enhance_from_product(self, in_prod: GenericProduct): + pass + + @classmethod + def filter_flare_function(cls, col): + return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', + fallback=1000) + + @classmethod + def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): + super().add_flare_position(data, fido_client, + filter_function=cls.filter_flare_function, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + keep_all_flares=False, month=month) + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'L3' and service_type == 0 + and service_subtype == 0 and ssid == 7) + + +class FlarelistSCLocImg(FlarelistSCLoc, FlarePeekPreviewMixin): + """Flarelist product class for StixCore flares. + + In ANC product format. + """ + PRODUCT_PROCESSING_VERSION = 2 + NAME = 'sclocimg' + + def __init__(self, *, service_type=0, service_subtype=0, ssid=8, data, month, **kwargs): + + super().__init__(service_type=0, service_subtype=0, ssid=8, + data=data, month=month, **kwargs) + + self.name = FlarelistSCLocImg.NAME + self.ssid = 8 + + def enhance_from_product(self, in_prod: GenericProduct): + pass + + @classmethod + def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, + *, month=None): + super().add_peek_preview(data, energies, parent, fido_client, img_processor, + peek_time_colname='peak_UTC', + start_time_colname='start_UTC', + end_time_colname='end_UTC', + anc_ephemeris_path_colname='anc_ephemeris_path', + cpd_path_colname='cpd_path', + product_name_suffix=FlarelistSC.NAME, + keep_all_flares=False, month=month) + + @classmethod + def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + return (kwargs['level'] == 'L3' and service_type == 0 + and service_subtype == 0 and ssid == 8) diff --git a/stixcore/products/level3/flarelistproduct.py b/stixcore/products/level3/flarelistproduct.py index 5b90d175..0330c72e 100644 --- a/stixcore/products/level3/flarelistproduct.py +++ b/stixcore/products/level3/flarelistproduct.py @@ -26,7 +26,7 @@ class PeekPreviewImage(FlareListProduct): Name = "peekpreviewimg" def __init__(self, control, data, energy, maps, parents, *, product_name_suffix="", **kwargs): - super().__init__(service_type=0, service_subtype=0, ssid=4, control=control, + super().__init__(service_type=0, service_subtype=0, ssid=5, control=control, data=data, energy=energy, **kwargs) self.name = f"{PeekPreviewImage.Name}-{product_name_suffix}" self.level = PeekPreviewImage.Level @@ -59,4 +59,4 @@ def split_to_files(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == PeekPreviewImage.Level and service_type == 0 - and service_subtype == 0 and ssid == 4) + and service_subtype == 0 and ssid == 5) diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 371105ae..d067f828 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -270,7 +270,7 @@ def __call__(self, *args, **kwargs): data["time"] = offset + data["time"] energies = None - if level in ["L1", "ANC"] and "ENERGIES" in hdul: + if level in ["L1", "L2", "L3", "ANC"] and "ENERGIES" in hdul: try: energies = read_qtable(file_path, hdu="ENERGIES") except KeyError: From 28d1ac5a44108338275d38408c4fb4caf51b2dc9 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 1 Sep 2025 17:38:35 +0200 Subject: [PATCH 05/11] new naming for LL3 QL products --- stixcore/io/fits/processors.py | 8 +++++++- stixcore/processing/pipeline_daily.py | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 53c15f64..9e24405b 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -752,7 +752,13 @@ def generate_filename(cls, product, *, version=0, status='', header=True): date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT%H%M%S")}-' +\ f'{product.utc_timerange.end.strftime("%Y%m%dT%H%M%S")}' - if product.type not in ['sci', 'flarelist'] or product.name == 'burst-aspect': + if (product.LEVEL in ['LL03']): + # for daily LL03 products use fake daily file naming convention all day from 0-24h + # this is to avoid multiple files per day and to be able to add + date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT000000")}-' +\ + f'{product.utc_timerange.start.strftime("%Y%m%dT235959")}' + elif (product.type not in ['sci', 'flarelist'] + or product.name == 'burst-aspect'): date_range = product.utc_timerange.center.strftime("%Y%m%d") return FitsProcessor.generate_filename( diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index 3960df5c..b43cc50f 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -269,16 +269,17 @@ def run_daily_pipeline(args): hk_in_files = [] fl_sdc_months = flarelist_sdc.find_processing_months(phs) - # fl_sdc_months = [] + fl_sdc_months = [] # fl_sc_months = flarelist_sc.find_processing_months(phs) fl_sc_months = [] - # ll_candidates = ll03ql.get_processing_files(phs) - ll_candidates = [] + ll_candidates = ll03ql.get_processing_files(phs) + # ll_candidates = [] + # ll_candidates = ll_candidates[0:40] fl_to_fl_files = fl_to_fl.get_processing_files(phs) - # fl_to_fl_files = [] + fl_to_fl_files = [] # all processing files should be terminated before the next step as the different # processing steeps might create new candidates From 9d131446fa7d1424ac22f4ca9b02b8d9d0c708d1 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 2 Sep 2025 19:46:48 +0200 Subject: [PATCH 06/11] fixes after python3.11 upgrade --- .gitignore | 1 + pyproject.toml | 7 +- stixcore/data/test/stixcore.ini | 1 + stixcore/io/FlareListManager.py | 424 ++++++++-------- stixcore/io/fits/processors.py | 101 ++-- stixcore/processing/FLtoFL.py | 89 ++-- stixcore/processing/FlareListL3.py | 57 ++- stixcore/processing/LL.py | 68 +-- stixcore/processing/pipeline_daily.py | 137 ++++-- stixcore/processing/publish.py | 6 +- stixcore/processing/tests/test_processing.py | 3 + stixcore/processing/tests/test_publish.py | 1 + stixcore/products/ANC/aspect.py | 26 +- stixcore/products/level1/quicklookL1.py | 48 +- stixcore/products/level1/scienceL1.py | 50 +- stixcore/products/level3/flarelist.py | 486 ++++++++++--------- stixcore/products/level3/flarelistproduct.py | 11 +- stixcore/products/lowlatency/quicklookLL.py | 4 +- stixcore/products/product.py | 35 +- stixcore/util/util.py | 9 +- 20 files changed, 877 insertions(+), 687 deletions(-) diff --git a/.gitignore b/.gitignore index 9047cc90..c942678b 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,4 @@ monitor_status.json stixcore/data/test/idb/v2.26.38/idb.sqlite .python-version sunpy/_version.py +stixcore/_version.py diff --git a/pyproject.toml b/pyproject.toml index 8bb71c52..e1691c13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ license = "BSD-3-Clause" license-files = ["LICENSE.rst"] authors = [ {name="Shane Maloney", email = "shane.maloney@dias.ie"}, - {name="Nicky Maloney", email = "shane.maloney@dias.ie"} + {name="Nicky Hochmuth", email = "nicky.hochmuth@ateleris.ch"} ] dependencies = [ "bitstring~=4.2", @@ -30,9 +30,10 @@ dependencies = [ "roentgen~=2.1", "scp", "spiceypy~=6.0", - "sunpy~=5.1", + "sunpy~=6.0.0", "watchdog~=6.0", - "stixdcpy==3.0" + "stixdcpy==3.0", + "stixpy@git+https://github.com/TCDSolar/stixpy.git@main", ] dynamic = ["version"] keywords = ["solar physics", "solar", "sun", "solarorbiter", "stix"] diff --git a/stixcore/data/test/stixcore.ini b/stixcore/data/test/stixcore.ini index 3ddcb1b7..d98f5ce2 100644 --- a/stixcore/data/test/stixcore.ini +++ b/stixcore/data/test/stixcore.ini @@ -20,6 +20,7 @@ error_mail_sender = test@test.de parallel_batchsize_L0 = 3 parallel_batchsize_L1 = 3 parallel_batchsize_L2 = 3 +status_server_port = 12345 [SOOP] endpoint = https://solarorbiter.esac.esa.int/soopkitchen/api user = smaloney diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index e01c6149..f4c06d23 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -65,8 +65,7 @@ def __init__(self, file, fido_client, update=False): self._flarelist = self.read_flarelist() def __str__(self) -> str: - return (f"{self.flarelistname}: file: {self.file} update: " - f"{self.update} size: {len(self.flarelist)}") + return f"{self.flarelistname}: file: {self.file} update: {self.update} size: {len(self.flarelist)}" def update_list(self): """Updates the flarelist file via api request. @@ -101,45 +100,47 @@ def read_flarelist(self): today = datetime.now() - timedelta(days=7) flare_df_lists = [] if self.file.exists(): - old_list = pd.read_csv(self.file, keep_default_na=True, na_values=['None']) - mds = old_list['start_UTC'].max() + old_list = pd.read_csv(self.file, keep_default_na=True, na_values=["None"]) + mds = old_list["start_UTC"].max() try: - last_date = datetime.strptime(mds, '%Y-%m-%dT%H:%M:%S') + last_date = datetime.strptime(mds, "%Y-%m-%dT%H:%M:%S") except ValueError: - last_date = datetime.strptime(mds, '%Y-%m-%dT%H:%M:%S.%f') + last_date = datetime.strptime(mds, "%Y-%m-%dT%H:%M:%S.%f") flare_df_lists = [old_list] last_date = last_date.replace(hour=0, minute=0, second=0, microsecond=0) if not self.file.parent.exists(): - logger.info(f'path not found: {self.file.parent} creating dir') + logger.info(f"path not found: {self.file.parent} creating dir") self.file.parent.mkdir(parents=True, exist_ok=True) try: - while (last_date < today): + while last_date < today: # run flare detection for batches of 7 days # 2 days overlap to ensure no flares are missed start = last_date - timedelta(days=2) end = last_date + timedelta(days=7) - ql_lc_files = self.fido_client.search(a.Time(start, end), - a.Instrument.stix, - a.stix.DataProduct.ql_lightcurve) + ql_lc_files = self.fido_client.search( + a.Time(start, end), a.Instrument.stix, a.stix.DataProduct.ql_lightcurve + ) try: if len(ql_lc_files) > 0: ql_lc_files.filter_for_latest_version() url_to_path(ql_lc_files) except Exception as e: - logger.error(f'Error filtering for latest version of lightcurve files: {e}') + logger.error(f"Error filtering for latest version of lightcurve files: {e}") ql_lc_files = [] - ql_bg_files = self.fido_client.search(a.Time(start, end), - a.Instrument.stix, - a.stix.DataProduct.ql_background) + ql_bg_files = self.fido_client.search( + a.Time(start, end), a.Instrument.stix, a.stix.DataProduct.ql_background + ) if len(ql_bg_files) > 0: ql_bg_files.filter_for_latest_version() url_to_path(ql_bg_files) - logger.info(f'flare detection chunk: {start.isoformat()}/{end.isoformat()} ' - f'with {len(ql_lc_files)} lightcurve files and ' - f'{len(ql_bg_files)} background files') + logger.info( + f"flare detection chunk: {start.isoformat()}/{end.isoformat()} " + f"with {len(ql_lc_files)} lightcurve files and " + f"{len(ql_bg_files)} background files" + ) # flares = stixpy.detect_flares(start, end, # ql_lc_files=ql_lc_files, # ql_bg_files=ql_bg_files) @@ -147,7 +148,7 @@ def read_flarelist(self): flares = [] if len(flares) > 0: flare_df_lists.append(pd.DataFrame(flares)) - logger.info(f'found {len(flares)} flares') + logger.info(f"found {len(flares)} flares") last_date += timedelta(days=7) except Exception: @@ -159,22 +160,22 @@ def read_flarelist(self): # full_flare_list.drop_duplicates(inplace=True) # full_flare_list.sort_values(by="peak_UTC", inplace=True) # full_flare_list.reset_index(inplace=True, drop=True) - logger.info(f'write total {len(full_flare_list)} flares to local storage') + logger.info(f"write total {len(full_flare_list)} flares to local storage") # full_flare_list.to_csv(self.file, index_label=False) else: logger.info(f"read flare list from {self.file}") - full_flare_list = pd.read_csv(self.file, keep_default_na=True, na_values=['None']) + full_flare_list = pd.read_csv(self.file, keep_default_na=True, na_values=["None"]) return full_flare_list @staticmethod def filter_flare_function(col): - return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sc_min_count', - fallback=1000) + return col["lc_peak"][0].value > CONFIG.getint("Processing", "flarelist_sc_min_count", fallback=1000) def get_data(self, *, start, end, fido_client): - month_data = self.flarelist[(self.flarelist['start_UTC'] >= start.isoformat()) & - (self.flarelist['start_UTC'] < end.isoformat())] + month_data = self.flarelist[ + (self.flarelist["start_UTC"] >= start.isoformat()) & (self.flarelist["start_UTC"] < end.isoformat()) + ] if len(month_data) == 0: return None, None, None @@ -184,79 +185,97 @@ def get_data(self, *, start, end, fido_client): control = QTable() energy = QTable() - data['flare_id'] = Column(mt['flare_id'].astype(int), - description=f'unique flare id for flarelist {self.flarelistname}') - data['start_UTC'] = Column(0, description="start time of flare") - data['start_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['start_UTC']] - data['duration'] = Column(mt['duration'].astype(float) * u.s, - description="duration of flare") - data['end_UTC'] = Column(0, description="end time of flare") - data['end_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['end_UTC']] - data['peak_UTC'] = Column(0, description="flare peak time") - data['peak_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['peak_UTC']] - data['att_in'] = Column(mt['att_in'].astype(bool), - description="was attenuator in during flare") - data['bkg_baseline'] = Column(mt['LC0_BKG'] * u.ct, - description='background baseline at 4-10 keV') - data['GOES_class'] = Column(mt['GOES_class'].astype(str), - description="GOES class of the GOES XRS data at time of flare" - " - not derived from STIX data. Do not use when " - "flare isn't visible to Earth") - data['goes_min_class_est'] = Column(mt['goes_estimated_min_class'].astype(str), - description="min GOES class estimate derived " - "from STIX data") - data['goes_max_class_est'] = Column(mt['goes_estimated_max_class'].astype(str), - description="max GOES class estimate derived " - "from STIX data") - data['goes_mean_class_est'] = Column(mt['goes_estimated_mean_class'].astype(str), - description="mean GOES class estimate derived " - "from STIX data") - - data['GOES_flux'] = Column(mt['GOES_flux'].astype(float) * u.W / u.m**2, - description="GOES flux of the GOES XRS data at time of flare" - "- not derived from STIX data. Do not use when the " - "flare isn't visible to Earth") - data['goes_min_flux_est'] = Column(mt['goes_estimated_min_flux'].astype(float) * - u.W / u.m**2, - description="min GOES flux estimate " - "derived from STIX data") - data['goes_max_flux_est'] = Column(mt['goes_estimated_max_flux'].astype(float) * - u.W / u.m**2, - description="max GOES flux estimate " - "derived from STIX data") - data['goes_mean_flux_est'] = Column(mt['goes_estimated_mean_flux'].astype(float) * - u.W / u.m**2, - description="mean GOES flux estimate " - "derived from STIX data") + data["flare_id"] = Column( + mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}" + ) + data["start_UTC"] = Column(0, description="start time of flare") + data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]] + data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare") + data["end_UTC"] = Column(0, description="end time of flare") + data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]] + data["peak_UTC"] = Column(0, description="flare peak time") + data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]] + data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") + data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") + data["GOES_class"] = Column( + mt["GOES_class"].astype(str), + description="GOES class of the GOES XRS data at time of flare" + " - not derived from STIX data. Do not use when " + "flare isn't visible to Earth", + ) + data["goes_min_class_est"] = Column( + mt["goes_estimated_min_class"].astype(str), description="min GOES class estimate derived from STIX data" + ) + data["goes_max_class_est"] = Column( + mt["goes_estimated_max_class"].astype(str), description="max GOES class estimate derived from STIX data" + ) + data["goes_mean_class_est"] = Column( + mt["goes_estimated_mean_class"].astype(str), description="mean GOES class estimate derived from STIX data" + ) + + data["GOES_flux"] = Column( + mt["GOES_flux"].astype(float) * u.W / u.m**2, + description="GOES flux of the GOES XRS data at time of flare" + "- not derived from STIX data. Do not use when the " + "flare isn't visible to Earth", + ) + data["goes_min_flux_est"] = Column( + mt["goes_estimated_min_flux"].astype(float) * u.W / u.m**2, + description="min GOES flux estimate derived from STIX data", + ) + data["goes_max_flux_est"] = Column( + mt["goes_estimated_max_flux"].astype(float) * u.W / u.m**2, + description="max GOES flux estimate derived from STIX data", + ) + data["goes_mean_flux_est"] = Column( + mt["goes_estimated_mean_flux"].astype(float) * u.W / u.m**2, + description="mean GOES flux estimate derived from STIX data", + ) # data['cfl_x'] = Column(mt['CFL_X_arcsec'].astype(float) * u.arcsec, - # description="corse flare location in x direction provided by" + # description="coarse flare location in x direction provided by" # "onboard algorithm. (0,0) represents disk center") # data['cfl_y'] = Column(mt['CFL_Y_arcsec'].astype(float) * u.arcsec, - # description="corse flare location in y direction provided by" + # description="coarse flare location in y direction provided by" # "onboard algorithm. (0,0) represents disk center") - data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, - mt['LC1_PEAK_COUNTS_4S'].value, - mt['LC2_PEAK_COUNTS_4S'].value, - mt['LC3_PEAK_COUNTS_4S'].value, - mt['LC4_PEAK_COUNTS_4S'].value)).T * u.ct).astype(int), - description="counts in 4s peak window from quicklook lightcurve", - dtype=np.int64) - - data['lc_bgk_peak'] = Column((np.vstack((mt['LC0_BKG_COUNTS_4S'].value, - mt['LC1_BKG_COUNTS_4S'].value, - mt['LC2_BKG_COUNTS_4S'].value, - mt['LC3_BKG_COUNTS_4S'].value, - mt['LC4_BKG_COUNTS_4S'].value)) - .T * u.ct).astype(int), - description="background counts in 4s peak window" - "from quicklook lightcurve", - dtype=np.int64) - - data['energy_index'] = Column(0, description="energy band index", dtype=np.int8) - - data.add_index('flare_id') + data["lc_peak"] = Column( + ( + np.vstack( + ( + mt["LC0_PEAK_COUNTS_4S"].value, + mt["LC1_PEAK_COUNTS_4S"].value, + mt["LC2_PEAK_COUNTS_4S"].value, + mt["LC3_PEAK_COUNTS_4S"].value, + mt["LC4_PEAK_COUNTS_4S"].value, + ) + ).T + * u.ct + ).astype(int), + description="counts in 4s peak window from quicklook lightcurve", + dtype=np.int64, + ) + + data["lc_bgk_peak"] = Column( + ( + np.vstack( + ( + mt["LC0_BKG_COUNTS_4S"].value, + mt["LC1_BKG_COUNTS_4S"].value, + mt["LC2_BKG_COUNTS_4S"].value, + mt["LC3_BKG_COUNTS_4S"].value, + mt["LC4_BKG_COUNTS_4S"].value, + ) + ).T + * u.ct + ).astype(int), + description="background counts in 4s peak windowfrom quicklook lightcurve", + dtype=np.int64, + ) + + data["energy_index"] = Column(0, description="energy band index", dtype=np.int8) + + data.add_index("flare_id") # add energy axis for the lightcurve peek time data for each flare # the energy bins are taken from the daily ql-lightcurve products @@ -265,21 +284,19 @@ def get_data(self, *, start, end, fido_client): # So we do the lookup also just grouped by peak day in order to save file lookups energy_look_up = {} - data['peak_day'] = [d.datetime.day for d in data['peak_UTC']] - data_by_day = data.group_by('peak_day') + data["peak_day"] = [d.datetime.day for d in data["peak_UTC"]] + data_by_day = data.group_by("peak_day") for day, flares in zip(data_by_day.groups.keys, data_by_day.groups): - time = flares['peak_UTC'][0] - lc_data = fido_client.search(a.Time(time, time), - a.Instrument.stix, - a.stix.DataProduct.ql_lightcurve) + time = flares["peak_UTC"][0] + lc_data = fido_client.search(a.Time(time, time), a.Instrument.stix, a.stix.DataProduct.ql_lightcurve) lc_data.filter_for_latest_version() url_to_path(lc_data) if len(lc_data) == 0: - logger.warning(f'No lightcurve data found for flare at time {time}') + logger.warning(f"No lightcurve data found for flare at time {time}") continue - lc = Product(lc_data['path'][0]) + lc = Product(lc_data["path"][0]) energy_table_hash = frozenset(pd.core.util.hashing.hash_array(lc.energies.as_array())) @@ -288,19 +305,18 @@ def get_data(self, *, start, end, fido_client): if energy_table_hash not in energy_look_up: e_idx = len(energy_look_up.keys()) energy_look_up[energy_table_hash] = e_idx - lc.energies["index"] = Column(e_idx, description='energy edge table index', - dtype=np.int8) + lc.energies["index"] = Column(e_idx, description="energy edge table index", dtype=np.int8) energy = vstack([energy, lc.energies]) if e_idx > 0: - logger.warning(f'multiple energy ql-lc tables found for month {start}') + logger.warning(f"multiple energy ql-lc tables found for month {start}") # add the energy index to the flare data to all flares of the same day # https://docs.astropy.org/en/latest/table/modify_table.html#caveats - replace = data.loc[flares['flare_id']] - replace['energy_index'] = energy_look_up[energy_table_hash] - data.loc[flares['flare_id']] = replace + replace = data.loc[flares["flare_id"]] + replace["energy_index"] = energy_look_up[energy_table_hash] + data.loc[flares["flare_id"]] = replace - del data['peak_day'] + del data["peak_day"] return data, control, energy @@ -364,8 +380,8 @@ def read_flarelist(cls, file, update=False): today = datetime.now() # - timedelta(days=60) flare_df_lists = [] if file.exists(): - old_list = pd.read_csv(file, keep_default_na=True, na_values=['None']) - mds = old_list['start_UTC'].max() + old_list = pd.read_csv(file, keep_default_na=True, na_values=["None"]) + mds = old_list["start_UTC"].max() try: last_date = datetime.strptime(mds, "%Y-%m-%dT%H:%M:%S") except ValueError: @@ -399,18 +415,18 @@ def read_flarelist(cls, file, update=False): full_flare_list.to_csv(file, index_label=False) else: logger.info(f"read flare list from {file}") - full_flare_list = pd.read_csv(file, keep_default_na=True, na_values=['None']) + full_flare_list = pd.read_csv(file, keep_default_na=True, na_values=["None"]) return full_flare_list @staticmethod def filter_flare_function(col): - return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', - fallback=1000) + return col["lc_peak"][0].value > CONFIG.getint("Processing", "flarelist_sdc_min_count", fallback=1000) def get_data(self, *, start, end, fido_client): - month_data = self.flarelist[(self.flarelist['start_UTC'] >= start.isoformat()) & - (self.flarelist['start_UTC'] < end.isoformat())] + month_data = self.flarelist[ + (self.flarelist["start_UTC"] >= start.isoformat()) & (self.flarelist["start_UTC"] < end.isoformat()) + ] if len(month_data) == 0: return None, None, None @@ -420,80 +436,97 @@ def get_data(self, *, start, end, fido_client): control = QTable() energy = QTable() - data['flare_id'] = Column(mt['flare_id'].astype(int), - description=f'unique flare id for flarelist {self.flarelistname}') - data['start_UTC'] = Column(0, description="start time of flare") - data['start_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['start_UTC']] - data['duration'] = Column(mt['duration'].astype(float) * u.s, - description="duration of flare") - data['end_UTC'] = Column(0, description="end time of flare") - data['end_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['end_UTC']] - data['peak_UTC'] = Column(0, description="flare peak time") - data['peak_UTC'] = [Time(d, format='isot', scale='utc') for d in mt['peak_UTC']] - data['att_in'] = Column(mt['att_in'].astype(bool), - description="was attenuator in during flare") - data['bkg_baseline'] = Column(mt['LC0_BKG'] * u.ct, - description='background baseline at 4-10 keV') - data['GOES_class'] = Column(mt['GOES_class'].astype(str), - description="GOES class of the GOES XRS data at time of flare" - " - not derived from STIX data. Do not use when " - "flare isn't visible to Earth") - data['goes_min_class_est'] = Column(mt['goes_estimated_min_class'].astype(str), - description="min GOES class estimate derived " - "from STIX data") - data['goes_max_class_est'] = Column(mt['goes_estimated_max_class'].astype(str), - description="max GOES class estimate derived " - "from STIX data") - data['goes_mean_class_est'] = Column(mt['goes_estimated_mean_class'].astype(str), - description="mean GOES class estimate derived " - "from STIX data") - - data['GOES_flux'] = Column(mt['GOES_flux'].astype(float) * u.W / u.m**2, - description="GOES flux of the GOES XRS data at time of flare" - "- not derived from STIX data. Do not use when the " - "flare isn't visible to Earth") - data['goes_min_flux_est'] = Column(mt['goes_estimated_min_flux'].astype(float) * - u.W / u.m**2, - description="min GOES flux estimate " - "derived from STIX data") - data['goes_max_flux_est'] = Column(mt['goes_estimated_max_flux'].astype(float) * - u.W / u.m**2, - description="max GOES flux estimate " - "derived from STIX data") - data['goes_mean_flux_est'] = Column(mt['goes_estimated_mean_flux'].astype(float) * - u.W / u.m**2, - description="mean GOES flux estimate " - "derived from STIX data") + data["flare_id"] = Column( + mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}" + ) + data["start_UTC"] = Column(0, description="start time of flare") + data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]] + data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare") + data["end_UTC"] = Column(0, description="end time of flare") + data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]] + data["peak_UTC"] = Column(0, description="flare peak time") + data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]] + data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") + data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") + data["GOES_class"] = Column( + mt["GOES_class"].astype(str), + description="GOES class of the GOES XRS data at time of flare" + " - not derived from STIX data. Do not use when " + "flare isn't visible to Earth", + ) + data["goes_min_class_est"] = Column( + mt["goes_estimated_min_class"].astype(str), description="min GOES class estimate derived from STIX data" + ) + data["goes_max_class_est"] = Column( + mt["goes_estimated_max_class"].astype(str), description="max GOES class estimate derived from STIX data" + ) + data["goes_mean_class_est"] = Column( + mt["goes_estimated_mean_class"].astype(str), description="mean GOES class estimate derived from STIX data" + ) + + data["GOES_flux"] = Column( + mt["GOES_flux"].astype(float) * u.W / u.m**2, + description="GOES flux of the GOES XRS data at time of flare" + "- not derived from STIX data. Do not use when the " + "flare isn't visible to Earth", + ) + data["goes_min_flux_est"] = Column( + mt["goes_estimated_min_flux"].astype(float) * u.W / u.m**2, + description="min GOES flux estimate derived from STIX data", + ) + data["goes_max_flux_est"] = Column( + mt["goes_estimated_max_flux"].astype(float) * u.W / u.m**2, + description="max GOES flux estimate derived from STIX data", + ) + data["goes_mean_flux_est"] = Column( + mt["goes_estimated_mean_flux"].astype(float) * u.W / u.m**2, + description="mean GOES flux estimate derived from STIX data", + ) # data['cfl_x'] = Column(mt['CFL_X_arcsec'].astype(float) * u.arcsec, - # description="corse flare location in x direction provided by" + # description="coarse flare location in x direction provided by" # "onboard algorithm. (0,0) represents disk center") # data['cfl_y'] = Column(mt['CFL_Y_arcsec'].astype(float) * u.arcsec, - # description="corse flare location in y direction provided by" + # description="coarse flare location in y direction provided by" # "onboard algorithm. (0,0) represents disk center") - data['lc_peak'] = Column((np.vstack((mt['LC0_PEAK_COUNTS_4S'].value, - mt['LC1_PEAK_COUNTS_4S'].value, - mt['LC2_PEAK_COUNTS_4S'].value, - mt['LC3_PEAK_COUNTS_4S'].value, - mt['LC4_PEAK_COUNTS_4S'].value)) - .T * u.ct).astype(int), - description="counts in 4s peak window from quicklook lightcurve", - dtype=np.int64) - - data['lc_bgk_peak'] = Column((np.vstack((mt['LC0_BKG_COUNTS_4S'].value, - mt['LC1_BKG_COUNTS_4S'].value, - mt['LC2_BKG_COUNTS_4S'].value, - mt['LC3_BKG_COUNTS_4S'].value, - mt['LC4_BKG_COUNTS_4S'].value)) - .T * u.ct).astype(int), - description="background counts in 4s peak window" - "from quicklook lightcurve", - dtype=np.int64) - - data['energy_index'] = Column(0, description="energy band index", dtype=np.int8) - - data.add_index('flare_id') + data["lc_peak"] = Column( + ( + np.vstack( + ( + mt["LC0_PEAK_COUNTS_4S"].value, + mt["LC1_PEAK_COUNTS_4S"].value, + mt["LC2_PEAK_COUNTS_4S"].value, + mt["LC3_PEAK_COUNTS_4S"].value, + mt["LC4_PEAK_COUNTS_4S"].value, + ) + ).T + * u.ct + ).astype(int), + description="counts in 4s peak window from quicklook lightcurve", + dtype=np.int64, + ) + + data["lc_bgk_peak"] = Column( + ( + np.vstack( + ( + mt["LC0_BKG_COUNTS_4S"].value, + mt["LC1_BKG_COUNTS_4S"].value, + mt["LC2_BKG_COUNTS_4S"].value, + mt["LC3_BKG_COUNTS_4S"].value, + mt["LC4_BKG_COUNTS_4S"].value, + ) + ).T + * u.ct + ).astype(int), + description="background counts in 4s peak windowfrom quicklook lightcurve", + dtype=np.int64, + ) + + data["energy_index"] = Column(0, description="energy band index", dtype=np.int8) + + data.add_index("flare_id") # add energy axis for the lightcurve peek time data for each flare # the energy bins are taken from the daily ql-lightcurve products @@ -502,21 +535,19 @@ def get_data(self, *, start, end, fido_client): # So we do the lookup also just grouped by peak day in order to save file lookups energy_look_up = {} - data['peak_day'] = [d.datetime.day for d in data['peak_UTC']] - data_by_day = data.group_by('peak_day') + data["peak_day"] = [d.datetime.day for d in data["peak_UTC"]] + data_by_day = data.group_by("peak_day") for day, flares in zip(data_by_day.groups.keys, data_by_day.groups): - time = flares['peak_UTC'][0] - lc_data = fido_client.search(a.Time(time, time), - a.Instrument.stix, - a.stix.DataProduct.ql_lightcurve) + time = flares["peak_UTC"][0] + lc_data = fido_client.search(a.Time(time, time), a.Instrument.stix, a.stix.DataProduct.ql_lightcurve) lc_data.filter_for_latest_version() url_to_path(lc_data) if len(lc_data) == 0: - logger.warning(f'No lightcurve data found for flare at time {time}') + logger.warning(f"No lightcurve data found for flare at time {time}") continue - lc = Product(lc_data['path'][0]) + lc = Product(lc_data["path"][0]) energy_table_hash = frozenset(pd.core.util.hashing.hash_array(lc.energies.as_array())) @@ -525,19 +556,18 @@ def get_data(self, *, start, end, fido_client): if energy_table_hash not in energy_look_up: e_idx = len(energy_look_up.keys()) energy_look_up[energy_table_hash] = e_idx - lc.energies["index"] = Column(e_idx, description='energy edge table index', - dtype=np.int8) + lc.energies["index"] = Column(e_idx, description="energy edge table index", dtype=np.int8) energy = vstack([energy, lc.energies]) if e_idx > 0: - logger.warning(f'multiple energy ql-lc tables found for month {start}') + logger.warning(f"multiple energy ql-lc tables found for month {start}") # add the energy index to the flare data to all flares of the same day # https://docs.astropy.org/en/latest/table/modify_table.html#caveats - replace = data.loc[flares['flare_id']] - replace['energy_index'] = energy_look_up[energy_table_hash] - data.loc[flares['flare_id']] = replace + replace = data.loc[flares["flare_id"]] + replace["energy_index"] = energy_look_up[energy_table_hash] + data.loc[flares["flare_id"]] = replace - del data['peak_day'] + del data["peak_day"] return data, control, energy diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 9e24405b..25d458dd 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -1,6 +1,5 @@ """Module for the different processing levels.""" -from math import isnan from pathlib import Path from datetime import datetime @@ -21,8 +20,15 @@ from stixcore.util.logging import get_logger from stixcore.util.util import get_complete_file_name_and_path -__all__ = ['SEC_IN_DAY', 'FitsProcessor', 'FitsLBProcessor', 'FitsL0Processor', - 'FitsL1Processor', 'FitsL2Processor', 'PlotProcessor'] +__all__ = [ + "SEC_IN_DAY", + "FitsProcessor", + "FitsLBProcessor", + "FitsL0Processor", + "FitsL1Processor", + "FitsL2Processor", + "PlotProcessor", +] logger = get_logger(__name__) @@ -36,6 +42,7 @@ def empty_if_nan(val): return "" if isnan(val) else val + def version_format(version): # some very strange work around for direct use of format '{0:02d}'.format(version) # as this is not supported by magicMoc @@ -98,14 +105,13 @@ def generate_filename(cls, product, *, version, date_range, status="", header=Tr The filename """ - user_req = '' - if product.control and 'request_id' in product.control.colnames: + user_req = "" + if product.control and "request_id" in product.control.colnames: user_req = f"_{product.control['request_id'][0]:010d}" - tc_control = '' - if product.control and 'tc_packet_seq_control' in product.control.colnames \ - and user_req != '': - tc_control = f'-{product.control["tc_packet_seq_control"][0]:05d}' + tc_control = "" + if product.control and "tc_packet_seq_control" in product.control.colnames and user_req != "": + tc_control = f"-{product.control['tc_packet_seq_control'][0]:05d}" if status == "" and (not header) and (product.level != "LB" and product.fits_daily_file is True): status = "U" @@ -748,17 +754,19 @@ def __init__(self, archive_path): self.archive_path = archive_path @classmethod - def generate_filename(cls, product, *, version=0, status='', header=True): - - date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT%H%M%S")}-' +\ - f'{product.utc_timerange.end.strftime("%Y%m%dT%H%M%S")}' - if (product.LEVEL in ['LL03']): + def generate_filename(cls, product, *, version=0, status="", header=True): + date_range = ( + f"{product.utc_timerange.start.strftime('%Y%m%dT%H%M%S')}-" + + f"{product.utc_timerange.end.strftime('%Y%m%dT%H%M%S')}" + ) + if product.LEVEL in ["LL03"]: # for daily LL03 products use fake daily file naming convention all day from 0-24h # this is to avoid multiple files per day and to be able to add - date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT000000")}-' +\ - f'{product.utc_timerange.start.strftime("%Y%m%dT235959")}' - elif (product.type not in ['sci', 'flarelist'] - or product.name == 'burst-aspect'): + date_range = ( + f"{product.utc_timerange.start.strftime('%Y%m%dT000000')}-" + + f"{product.utc_timerange.start.strftime('%Y%m%dT235959')}" + ) + elif product.type not in ["sci", "flarelist"] or product.name == "burst-aspect": date_range = product.utc_timerange.center.strftime("%Y%m%d") return FitsProcessor.generate_filename( @@ -846,7 +854,7 @@ def write_fits(self, product, *, version=0): # day boundary parts = [prod.level, prod.utc_timerange.center.strftime("%Y/%m/%d"), prod.type.upper()] # for science data use start date - if prod.type in ["sci", 'flarelist']: + if prod.type in ["sci", "flarelist"]: parts[1] = prod.utc_timerange.start.strftime("%Y/%m/%d") path = self.archive_path.joinpath(*[str(x) for x in parts]) path.mkdir(parents=True, exist_ok=True) @@ -875,8 +883,7 @@ def write_fits(self, product, *, version=0): if col.endswith("_comp_err"): data[col].description = "Error due only to integer compression" - primary_header, header_override = self.generate_primary_header(filename, prod, - version=version) + primary_header, header_override = self.generate_primary_header(filename, prod, version=version) primary_hdu = fits.PrimaryHDU() primary_hdu.header.update(primary_header) primary_hdu.header.update(header_override) @@ -918,9 +925,13 @@ def write_fits(self, product, *, version=0): hdul = [primary_hdu, control_hdu, data_hdu] - idb_versions = QTable(rows=[(version, range.start.as_float(), range.end.as_float()) - for version, range in prod.idb_versions.items()], - names=["version", "obt_start", "obt_end"]) + idb_versions = QTable( + rows=[ + (version, range.start.as_float(), range.end.as_float()) + for version, range in prod.idb_versions.items() + ], + names=["version", "obt_start", "obt_end"], + ) idb_enc = fits.connect._encode_mixins(idb_versions) idb_hdu = table_to_hdu(idb_enc) idb_hdu = add_default_tuint(idb_hdu) @@ -1023,10 +1034,10 @@ def write_fits(self, prod, *, version=0): filename = self.generate_filename(product=prod, version=version, header=False) - parts = [prod.level, prod.utc_timerange.center.strftime('%Y/%m/%d'), prod.type.upper()] + parts = [prod.level, prod.utc_timerange.center.strftime("%Y/%m/%d"), prod.type.upper()] # for science data use start date - if prod.type in ['flarelist']: - parts[1] = prod.utc_timerange.start.strftime('%Y/%m/%d') + if prod.type in ["flarelist"]: + parts[1] = prod.utc_timerange.start.strftime("%Y/%m/%d") path = self.archive_path.joinpath(*[str(x) for x in parts]) path.mkdir(parents=True, exist_ok=True) @@ -1034,14 +1045,13 @@ def write_fits(self, prod, *, version=0): fitspath_complete = get_complete_file_name_and_path(fitspath) if fitspath.exists(): - logger.warning('Fits file %s exists data will be overriden', fitspath.name) + logger.warning("Fits file %s exists data will be overridden", fitspath.name) elif fitspath_complete.exists(): - logger.warning('Complete Fits file %s exists will be overriden', fitspath.name) + logger.warning("Complete Fits file %s exists will be overridden", fitspath.name) data = prod.data - primary_header, header_override = self.generate_primary_header(filename, prod, - version=version) + primary_header, header_override = self.generate_primary_header(filename, prod, version=version) primary_hdu = fits.PrimaryHDU() primary_hdu.header.update(primary_header) primary_hdu.header.update(header_override) @@ -1050,7 +1060,7 @@ def write_fits(self, prod, *, version=0): # Add comment and history [primary_hdu.header.add_comment(com) for com in prod.comment] [primary_hdu.header.add_history(com) for com in prod.history] - primary_hdu.header.update({'HISTORY': 'Processed by STIXCore ANC'}) + primary_hdu.header.update({"HISTORY": "Processed by STIXCore ANC"}) data_enc = fits.connect._encode_mixins(data) data_hdu = table_to_hdu(data_enc) @@ -1065,7 +1075,7 @@ def write_fits(self, prod, *, version=0): energy_hdu = table_to_hdu(energy_enc) energy_hdu = add_default_tuint(energy_hdu) energy_hdu.name = "ENERGIES" - hdul.append((energy_hdu)) + hdul.append(energy_hdu) hdul = fits.HDUList(hdul) @@ -1077,6 +1087,7 @@ def write_fits(self, prod, *, version=0): class PlotProcessor(FitsL2Processor): """A file product processor for plot images""" + def __init__(self, archive_path): """Creates a new PlotProcessor object. @@ -1087,7 +1098,7 @@ def __init__(self, archive_path): """ super().__init__(archive_path) - def generate_filename(self, product, *, version=0, suffix='.svg'): + def generate_filename(self, product, *, version=0, suffix=".svg"): """Generates a SOAR conform filename based on product characteristics. Parameters @@ -1104,8 +1115,7 @@ def generate_filename(self, product, *, version=0, suffix='.svg'): Path a Path object with full name and path """ - p = Path(super().generate_filename(product=product, version=version, - header=True, status='C')) + p = Path(super().generate_filename(product=product, version=version, header=True, status="C")) return p.with_suffix(suffix).name def generate_primary_header(self, filename, product, *, version=0): @@ -1119,7 +1129,7 @@ def generate_primary_header(self, filename, product, *, version=0): product : Product The products holding the data version : int, optional - teh processing version, by default 0 = detect from codebase + the processing version, by default 0 = detect from codebase Returns ------- @@ -1160,8 +1170,7 @@ def write_plot(self, product, *, version=0): # headers = self.generate_primary_header(filename, product, version=version) # headers['parent'] = get_complete_file_name(product.parent_file_path.name) - parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), - product.type.upper()] + parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), product.type.upper()] # for science data use start date if product.type in ["sci", "flarelist"]: parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d") @@ -1213,14 +1222,13 @@ def write_fits(self, prod, *, version=0): fitspath_complete = get_complete_file_name_and_path(fitspath) if fitspath.exists(): - logger.warning('Fits file %s exists data will be overriden', fitspath.name) + logger.warning("Fits file %s exists data will be overridden", fitspath.name) elif fitspath_complete.exists(): - logger.warning('Complete Fits file %s exists will be overriden', fitspath.name) + logger.warning("Complete Fits file %s exists will be overridden", fitspath.name) data = prod.data - primary_header, header_override = self.generate_primary_header(filename, prod, - version=version) + primary_header, header_override = self.generate_primary_header(filename, prod, version=version) primary_hdu = fits.PrimaryHDU() primary_hdu.header.update(primary_header) @@ -1231,7 +1239,7 @@ def write_fits(self, prod, *, version=0): # Add comment and history [primary_hdu.header.add_comment(com) for com in prod.comment] [primary_hdu.header.add_history(com) for com in prod.history] - primary_hdu.header.update({'HISTORY': 'Processed by STIXCore ANC'}) + primary_hdu.header.update({"HISTORY": "Processed by STIXCore ANC"}) if hasattr(prod, "maps") and len(prod.maps) > 0: # fig = plt.figure(figsize=(12, 6)) @@ -1255,7 +1263,7 @@ def write_fits(self, prod, *, version=0): for idx, map in enumerate(prod.maps): map_data, header = map image_hdu = fits.ImageHDU(map_data, header=header) - image_hdu.name = f'MAP_{idx}' + image_hdu.name = f"MAP_{idx}" hdul.append(image_hdu) data_enc = fits.connect._encode_mixins(data) @@ -1276,12 +1284,11 @@ def write_fits(self, prod, *, version=0): hdul = fits.HDUList(hdul) filetowrite = path / filename - logger.info(f'Writing fits file to {filetowrite}') + logger.info(f"Writing fits file to {filetowrite}") hdul.writeto(filetowrite, overwrite=True, checksum=True) return [filetowrite] def generate_primary_header(self, filename, product, *, version=0): - if product.fits_header is None: L2, o = super().generate_primary_header(filename, product, version=version) L2headers = L2 + o diff --git a/stixcore/processing/FLtoFL.py b/stixcore/processing/FLtoFL.py index a6cd9b88..087f0d36 100644 --- a/stixcore/processing/FLtoFL.py +++ b/stixcore/processing/FLtoFL.py @@ -24,19 +24,25 @@ from stixcore.util.logging import get_logger from stixcore.util.util import get_complete_file_name_and_path -__all__ = ['FLtoFL'] +__all__ = ["FLtoFL"] logger = get_logger(__name__) class FLtoFL(SingleProductProcessingStepMixin): - """Processing step from enhance monthly flare list files to next processing steep - """ + """Processing step from enhance monthly flare list files to next processing steep""" + INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits" - def __init__(self, source_dir: Path, output_dir: Path, *, - input_pattern: str = '', products_in_out, - cadence: timedelta = timedelta(hours=5)): + def __init__( + self, + source_dir: Path, + output_dir: Path, + *, + input_pattern: str = "", + products_in_out, + cadence: timedelta = timedelta(hours=5), + ): """Crates a new Processor. Parameters @@ -63,8 +69,7 @@ def find_processing_candidates(self): a list of fits files candidates """ for product_in, product_out in self.in_products.items(): - pattern = (f"solo_{product_in.LEVEL}_stix-{product_in.TYPE}-" - f"{product_in.NAME}_*.fits") + pattern = f"solo_{product_in.LEVEL}_stix-{product_in.TYPE}-{product_in.NAME}_*.fits" for c in self.source_dir.rglob(pattern): yield product_in, product_out, c @@ -83,10 +88,9 @@ def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]: fl_to_process.append((product_in, product_out, fl_can)) return fl_to_process - def test_for_processing(self, candidate: Path, - product_in: FlareList, - product_out: FlareList, - phm: ProcessingHistoryStorage) -> TestForProcessingResult: + def test_for_processing( + self, candidate: Path, product_in: FlareList, product_out: FlareList, phm: ProcessingHistoryStorage + ) -> TestForProcessingResult: """_summary_ Parameters @@ -103,16 +107,19 @@ def test_for_processing(self, candidate: Path, """ try: c_header = fits.getheader(candidate) - f_data_end = datetime.fromisoformat(c_header['DATE-END']) - f_create_date = datetime.fromisoformat(c_header['DATE']) + f_data_end = datetime.fromisoformat(c_header["DATE-END"]) + f_create_date = datetime.fromisoformat(c_header["DATE"]) cfn = get_complete_file_name_and_path(candidate) - wp = phm.has_processed_fits_products(product_out.NAME, - product_out.LEVEL, - product_out.TYPE, - product_out.get_cls_processing_version(), - str(cfn), f_create_date) + wp = phm.has_processed_fits_products( + product_out.NAME, + product_out.LEVEL, + product_out.TYPE, + product_out.get_cls_processing_version(), + str(cfn), + f_create_date, + ) # found already in the processing history if wp: @@ -121,12 +128,11 @@ def test_for_processing(self, candidate: Path, # safety margin of 1day until we process higher products with position and pointing # only use flown spice kernels not predicted once as pointing information # can be "very off" - if (f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - - timedelta(hours=24))): + if f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - timedelta(hours=24)): return TestForProcessingResult.NotSuitable # safety margin of x until we start with processing the list files - if (f_create_date >= (datetime.now() - self.cadence)): + if f_create_date >= (datetime.now() - self.cadence): return TestForProcessingResult.NotSuitable return TestForProcessingResult.Suitable @@ -134,9 +140,9 @@ def test_for_processing(self, candidate: Path, logger.error(e) return TestForProcessingResult.NotSuitable - def process_fits_files(self, flarelists, *, soopmanager: SOOPManager, - spice_kernel_path: Path, fl_processor, - img_processor, config) -> list[Path]: + def process_fits_files( + self, flarelists, *, soopmanager: SOOPManager, spice_kernel_path: Path, fl_processor, img_processor, config + ) -> list[Path]: """Performs the processing (expected to run in a dedicated python process) from a list flare list product to an enhanced flare list product. @@ -161,8 +167,7 @@ def process_fits_files(self, flarelists, *, soopmanager: SOOPManager, CONFIG = config SOOPManager.instance = soopmanager Spice.instance = Spice(spice_kernel_path) - fido_client = STIXClient(source=CONFIG.get('Paths', 'fido_search_url', - fallback=STIXClient.baseurl)) + fido_client = STIXClient(source=CONFIG.get("Paths", "fido_search_url", fallback=STIXClient.baseurl)) all_files = list() for in_product, out_product, file_path in flarelists: @@ -175,20 +180,16 @@ def process_fits_files(self, flarelists, *, soopmanager: SOOPManager, month = prod.utc_timerange.start.datetime.date() # add flare position if not already present - if issubclass(out_product, FlarePositionMixin) and \ - not issubclass(in_product, FlarePositionMixin): + if issubclass(out_product, FlarePositionMixin) and not issubclass(in_product, FlarePositionMixin): out_product.add_flare_position(data, fido_client, month=month) # add soop information if not already present - if issubclass(out_product, FlareSOOPMixin) and \ - not issubclass(in_product, FlareSOOPMixin): + if issubclass(out_product, FlareSOOPMixin) and not issubclass(in_product, FlareSOOPMixin): out_product.add_soop(data) # add peek preview images if not already present - if issubclass(out_product, FlarePeekPreviewMixin) and \ - not issubclass(in_product, FlarePeekPreviewMixin): - out_product.add_peek_preview(data, energy, file_path.name, fido_client, - img_processor, month=month) + if issubclass(out_product, FlarePeekPreviewMixin) and not issubclass(in_product, FlarePeekPreviewMixin): + out_product.add_peek_preview(data, energy, file_path.name, fido_client, img_processor, month=month) out_prod = out_product(control=control, data=data, month=month, energy=energy) out_prod.parent = file_path.name @@ -196,12 +197,18 @@ def process_fits_files(self, flarelists, *, soopmanager: SOOPManager, # call upgrade method to enhance the product out_prod.enhance_from_product(prod) - new_f = [SingleProcessingStepResult(out_product.NAME, out_product.LEVEL, - out_product.TYPE, - out_product.get_cls_processing_version(), - fop, get_complete_file_name_and_path(file_path), - datetime.now()) - for fop in fl_processor.write_fits(out_prod)] + new_f = [ + SingleProcessingStepResult( + out_product.NAME, + out_product.LEVEL, + out_product.TYPE, + out_product.get_cls_processing_version(), + fop, + get_complete_file_name_and_path(file_path), + datetime.now(), + ) + for fop in fl_processor.write_fits(out_prod) + ] all_files.extend(new_f) except Exception as e: diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index 97cbe9a0..89e6d5c8 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -16,14 +16,13 @@ from stixcore.soop.manager import SOOPManager from stixcore.util.logging import get_logger -__all__ = ['FlareListL3'] +__all__ = ["FlareListL3"] logger = get_logger(__name__) class FlareListL3(SingleProductProcessingStepMixin): - """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file. - """ + """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.""" STARTDATE = date(2024, 1, 1) @@ -49,13 +48,12 @@ def find_processing_months(self, phs: ProcessingHistoryStorage) -> list[date]: a list of fits files candidates """ fl_months = list() - for month in pd.date_range(FlareListL3.STARTDATE, date.today(), freq='MS').date: + for month in pd.date_range(FlareListL3.STARTDATE, date.today(), freq="MS").date: if self.test_for_processing(month, phs) == TestForProcessingResult.Suitable: fl_months.append(month) return fl_months - def test_for_processing(self, month: date, - phm: ProcessingHistoryStorage) -> TestForProcessingResult: + def test_for_processing(self, month: date, phm: ProcessingHistoryStorage) -> TestForProcessingResult: """_summary_ Parameters @@ -71,18 +69,21 @@ def test_for_processing(self, month: date, what should happen with the candidate in the next processing step """ try: - wp = phm.has_processed_fits_products(self.flm.productCls.NAME, - self.flm.productCls.LEVEL, - self.flm.productCls.TYPE, - self.flm.productCls.get_cls_processing_version(), - str(month), FlareListL3.STARTDATE) + wp = phm.has_processed_fits_products( + self.flm.productCls.NAME, + self.flm.productCls.LEVEL, + self.flm.productCls.TYPE, + self.flm.productCls.get_cls_processing_version(), + str(month), + FlareListL3.STARTDATE, + ) # found already in the processing history if wp: return TestForProcessingResult.NotSuitable # safety margin of 2 month until we start with processing of monthly flare list files - if (month >= (date.today() - timedelta(days=50)).replace(day=1)): + if month >= (date.today() - timedelta(days=50)).replace(day=1): return TestForProcessingResult.NotSuitable return TestForProcessingResult.Suitable @@ -90,8 +91,9 @@ def test_for_processing(self, month: date, logger.error(e) return TestForProcessingResult.NotSuitable - def process_fits_files(self, months: list[date], *, soopmanager: SOOPManager, - spice_kernel_path: Path, processor, config) -> list[Path]: + def process_fits_files( + self, months: list[date], *, soopmanager: SOOPManager, spice_kernel_path: Path, processor, config + ) -> list[Path]: """Performs the processing (expected to run in a dedicated python process) from a list of time periods (months) and creates a FITS file for each one. @@ -117,29 +119,34 @@ def process_fits_files(self, months: list[date], *, soopmanager: SOOPManager, SOOPManager.instance = soopmanager Spice.instance = Spice(spice_kernel_path) all_files = list() - fido_url = CONFIG.get('Paths', 'fido_search_url', - fallback='https://pub099.cs.technik.fhnw.ch/data/fits') + fido_url = CONFIG.get("Paths", "fido_search_url", fallback="https://pub099.cs.technik.fhnw.ch/data/fits") fido_client = STIXClient(source=fido_url) for month in months: - data, control, energy = self.flm.get_data(start=month, - end=(month + pd.DateOffset(months=1)).date(), - fido_client=fido_client) + data, control, energy = self.flm.get_data( + start=month, end=(month + pd.DateOffset(months=1)).date(), fido_client=fido_client + ) if not data: - logger.warning(f'No data found for month {month}') + logger.warning(f"No data found for month {month}") continue if issubclass(self.flm.productCls, FlarePositionMixin): - self.flm.productCls.add_flare_position(data, fido_client, filter_function=self.flm.filter_flare_function) # noqa + self.flm.productCls.add_flare_position( + data, fido_client, filter_function=self.flm.filter_flare_function + ) # noqa if issubclass(self.flm.productCls, FlareSOOPMixin): self.flm.productCls.add_soop(data) anc = self.flm.productCls(control=control, data=data, month=month, energy=energy) - all_files.extend([SingleProcessingStepResult(anc.name, anc.level, anc.type, - anc.get_processing_version(), - fop, str(month), datetime.now()) - for fop in processor.write_fits(anc)]) + all_files.extend( + [ + SingleProcessingStepResult( + anc.name, anc.level, anc.type, anc.get_processing_version(), fop, str(month), datetime.now() + ) + for fop in processor.write_fits(anc) + ] + ) return all_files diff --git a/stixcore/processing/LL.py b/stixcore/processing/LL.py index 8b69b751..9c99d7d3 100644 --- a/stixcore/processing/LL.py +++ b/stixcore/processing/LL.py @@ -17,19 +17,26 @@ from stixcore.util.logging import get_logger from stixcore.util.util import get_complete_file_name_and_path -__all__ = ['LL03QL'] +__all__ = ["LL03QL"] logger = get_logger(__name__) class LL03QL(SingleProductProcessingStepMixin): - """Processing step from a ql ligtcurve L1 fits file to LL level 3 plot chart image. - """ + """Processing step from a ql ligtcurve L1 fits file to LL level 3 plot chart image.""" + INPUT_PATTERN = "solo_ANC_stix-flarelist-sdc*.fits" - def __init__(self, source_dir: Path, output_dir: Path, *, - input_pattern: str = '', in_product: LightCurve, out_product: LightCurveL3, - cadence: timedelta = timedelta(minutes=1)): + def __init__( + self, + source_dir: Path, + output_dir: Path, + *, + input_pattern: str = "", + in_product: LightCurve, + out_product: LightCurveL3, + cadence: timedelta = timedelta(minutes=1), + ): """Crates a new Processor. Parameters @@ -71,8 +78,7 @@ def get_processing_files(self, phs: ProcessingHistoryStorage) -> list[Path]: ql_to_process.append(ql_can) return ql_to_process - def test_for_processing(self, candidate: Path, - phm: ProcessingHistoryStorage) -> TestForProcessingResult: + def test_for_processing(self, candidate: Path, phm: ProcessingHistoryStorage) -> TestForProcessingResult: """Performs a check on each found candidate if it should be processed or skipped Parameters @@ -89,23 +95,26 @@ def test_for_processing(self, candidate: Path, """ try: c_header = fits.getheader(candidate) - f_data_end = datetime.fromisoformat(c_header['DATE-END']) - f_create_date = datetime.fromisoformat(c_header['DATE']) + f_data_end = datetime.fromisoformat(c_header["DATE-END"]) + f_create_date = datetime.fromisoformat(c_header["DATE"]) cfn = get_complete_file_name_and_path(candidate) - wp = phm.has_processed_fits_products(self.out_product.NAME, - self.out_product.LEVEL, - self.out_product.TYPE, - self.out_product.get_cls_processing_version(), - str(cfn), f_create_date) + wp = phm.has_processed_fits_products( + self.out_product.NAME, + self.out_product.LEVEL, + self.out_product.TYPE, + self.out_product.get_cls_processing_version(), + str(cfn), + f_create_date, + ) # found already in the processing history if wp: return TestForProcessingResult.NotSuitable # safety margin of x until we start with processing the list files - if (f_create_date >= (datetime.now() - self.cadence)): + if f_create_date >= (datetime.now() - self.cadence): return TestForProcessingResult.NotSuitable # only process QL files with data for full 24 hours no created of half a day plots @@ -114,7 +123,7 @@ def test_for_processing(self, candidate: Path, # small rounding margin of 4 minutes is allowed (-60) # input files oldr then a month will be processed also as for not full 24 hours file_age = (datetime.now() - f_data_end).days - if (tbins < 60/4 * 60 * 24 - 60) and file_age < 30: + if (tbins < 60 / 4 * 60 * 24 - 60) and file_age < 30: return TestForProcessingResult.NotSuitable return TestForProcessingResult.Suitable @@ -122,8 +131,9 @@ def test_for_processing(self, candidate: Path, logger.error(e) return TestForProcessingResult.NotSuitable - def process_fits_files(self, files: list[Path], *, soopmanager: SOOPManager, - spice_kernel_path: Path, processor: PlotProcessor, config) -> list[Path]: + def process_fits_files( + self, files: list[Path], *, soopmanager: SOOPManager, spice_kernel_path: Path, processor: PlotProcessor, config + ) -> list[Path]: """Performs the processing (expected to run in a dedicated python process) from a list of ql lightcurve products into LL plots. @@ -152,18 +162,22 @@ def process_fits_files(self, files: list[Path], *, soopmanager: SOOPManager, data = in_prod.data energy = in_prod.energies - out_prod = self.out_product(control=control, data=data, energy=energy, - parent_file_path=file_path, header=in_prod.fits_header) + out_prod = self.out_product( + control=control, data=data, energy=energy, parent_file_path=file_path, header=in_prod.fits_header + ) plot_file = processor.write_plot(out_prod) logger.info(f"Generated plot file: {plot_file}") - new_f = SingleProcessingStepResult(self.out_product.NAME, self.out_product.LEVEL, - self.out_product.TYPE, - self.out_product.get_cls_processing_version(), - plot_file, - get_complete_file_name_and_path(file_path), - datetime.now()) + new_f = SingleProcessingStepResult( + self.out_product.NAME, + self.out_product.LEVEL, + self.out_product.TYPE, + self.out_product.get_cls_processing_version(), + plot_file, + get_complete_file_name_and_path(file_path), + datetime.now(), + ) all_files.append(new_f) except Exception as e: diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index b43cc50f..e91bfff1 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -176,9 +176,13 @@ def run_daily_pipeline(args): dest="log_level", ) - parser.add_argument("-r", "--rid_lut_file", - help=("Path to the rid LUT file"), - default=CONFIG.get('Publish', 'rid_lut_file'), type=str) + parser.add_argument( + "-r", + "--rid_lut_file", + help=("Path to the rid LUT file"), + default=CONFIG.get("Publish", "rid_lut_file"), + type=str, + ) args = parser.parse_args(args) @@ -202,22 +206,20 @@ def run_daily_pipeline(args): Spice.instance = Spice(spicemeta) - SOOPManager.instance = SOOPManager(Path(CONFIG.get('Paths', 'soop_files'))) + SOOPManager.instance = SOOPManager(Path(CONFIG.get("Paths", "soop_files"))) Path(CONFIG.get("Paths", "fits_archive")) - fido_url = CONFIG.get('Paths', 'fido_search_url', - fallback='https://pub099.cs.technik.fhnw.ch/data/fits') + fido_url = CONFIG.get("Paths", "fido_search_url", fallback="https://pub099.cs.technik.fhnw.ch/data/fits") fido_client = STIXClient(source=fido_url) - flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sc_lut_file')) + flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sc_lut_file")) SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True) - flare_lut_file = Path(CONFIG.get('Pipeline', 'flareid_sdc_lut_file')) + flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sdc_lut_file")) SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) - RidLutManager.instance = RidLutManager(Path(CONFIG.get('Publish', 'rid_lut_file')), - update=False) + RidLutManager.instance = RidLutManager(Path(CONFIG.get("Publish", "rid_lut_file")), update=False) db_file = Path(args.db_file) fits_in_dir = Path(args.fits_in_dir) @@ -249,16 +251,21 @@ def run_daily_pipeline(args): flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir) - fl_to_fl = FLtoFL(fits_in_dir, - fits_out_dir, - products_in_out=[(FlarelistSDC, FlarelistSDCLoc), - (FlarelistSDCLoc, FlarelistSDCLocImg), - (FlarelistSC, FlarelistSCLoc), - (FlarelistSCLoc, FlarelistSCLocImg)], - cadence=timedelta(seconds=1)) - - ll03ql = LL03QL(fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, - cadence=timedelta(seconds=1)) + fl_to_fl = FLtoFL( + fits_in_dir, + fits_out_dir, + products_in_out=[ + (FlarelistSDC, FlarelistSDCLoc), + (FlarelistSDCLoc, FlarelistSDCLocImg), + (FlarelistSC, FlarelistSCLoc), + (FlarelistSCLoc, FlarelistSCLocImg), + ], + cadence=timedelta(seconds=1), + ) + + ll03ql = LL03QL( + fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, cadence=timedelta(seconds=1) + ) plot_writer = PlotProcessor(fits_out_dir) l2_fits_writer = FitsL2Processor(fits_out_dir) @@ -286,39 +293,63 @@ def run_daily_pipeline(args): # let each processing "task" run in its own process jobs = [] with ProcessPoolExecutor() as executor: - jobs.append(executor.submit(aspect_anc_processor.process_fits_files, hk_in_files, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l2_fits_writer, - config=CONFIG)) - - jobs.append(executor.submit(flarelist_sdc.process_fits_files, fl_sdc_months, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l3_fits_writer, - config=CONFIG)) - - jobs.append(executor.submit(flarelist_sc.process_fits_files, fl_sc_months, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l3_fits_writer, - config=CONFIG)) + jobs.append( + executor.submit( + aspect_anc_processor.process_fits_files, + hk_in_files, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l2_fits_writer, + config=CONFIG, + ) + ) + + jobs.append( + executor.submit( + flarelist_sdc.process_fits_files, + fl_sdc_months, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l3_fits_writer, + config=CONFIG, + ) + ) + + jobs.append( + executor.submit( + flarelist_sc.process_fits_files, + fl_sc_months, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l3_fits_writer, + config=CONFIG, + ) + ) # TODO a owen processing step for each flarelist file? # for fl_to_fl_file in fl_to_fl_files: - jobs.append(executor.submit(fl_to_fl.process_fits_files, - fl_to_fl_files, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - fl_processor=anc_fits_writer, - img_processor=l3_fits_writer, - config=CONFIG)) - - jobs.append(executor.submit(ll03ql.process_fits_files, ll_candidates, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=plot_writer, - config=CONFIG)) + jobs.append( + executor.submit( + fl_to_fl.process_fits_files, + fl_to_fl_files, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + fl_processor=anc_fits_writer, + img_processor=l3_fits_writer, + config=CONFIG, + ) + ) + + jobs.append( + executor.submit( + ll03ql.process_fits_files, + ll_candidates, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=plot_writer, + config=CONFIG, + ) + ) # wait for all processes to end all_files = [] @@ -333,9 +364,9 @@ def run_daily_pipeline(args): # create an entry for each generated file in the ProcessingHistoryStorage for pr in all_files: if isinstance(pr, SingleProcessingStepResult): - phs.add_processed_fits_products(pr.name, pr.level, pr.type, - pr.version, pr.in_path, - pr.out_path, pr.date) + phs.add_processed_fits_products( + pr.name, pr.level, pr.type, pr.version, pr.in_path, pr.out_path, pr.date + ) phs.close() diff --git a/stixcore/processing/publish.py b/stixcore/processing/publish.py index 54d1d478..f381af1e 100644 --- a/stixcore/processing/publish.py +++ b/stixcore/processing/publish.py @@ -667,7 +667,7 @@ def publish_fits_to_esa(args): wait_period = u.Quantity(args.waiting_period) wait_period_s = wait_period.to(u.s).value - LL_wait_period = u.Quantity(CONFIG.get('Publish', 'LL_waiting_period', fallback="1min")) + LL_wait_period = u.Quantity(CONFIG.get("Publish", "LL_waiting_period", fallback="1min")) LL_wait_period_s = LL_wait_period.to(u.s).value target_dir = Path(args.target_dir) @@ -782,7 +782,7 @@ def publish_fits_to_esa(args): if level not in include_levels: continue - wp_s = LL_wait_period_s if level.startswith('ll') else wait_period_s + wp_s = LL_wait_period_s if level.startswith("ll") else wait_period_s # is the waiting time after last modification done if (now - last_mod) < wp_s: @@ -816,7 +816,7 @@ def publish_fits_to_esa(args): for p in to_publish: try: - isFits = p.suffix == '.fits' + isFits = p.suffix == ".fits" if isFits: comment = add_BSD_comment(p) diff --git a/stixcore/processing/tests/test_processing.py b/stixcore/processing/tests/test_processing.py index dbdb5e1c..d65238a4 100644 --- a/stixcore/processing/tests/test_processing.py +++ b/stixcore/processing/tests/test_processing.py @@ -265,11 +265,13 @@ def test_pipeline_logging(spicekernelmanager, out_dir): FITS_ARCHIVE = CONFIG.get("Paths", "fits_archive") LOG_LEVEL = CONFIG.get("Pipeline", "log_level") LOG_DIR = CONFIG.get("Pipeline", "log_dir") + SERVER_PORT = CONFIG.getint("Pipeline", "status_server_port") try: CONFIG.set("Logging", "stop_on_error", str(False)) CONFIG.set("Paths", "fits_archive", str(out_dir / "fits")) CONFIG.set("Pipeline", "log_level", "DEBUG") CONFIG.set("Pipeline", "log_dir", str(out_dir / "logging")) + CONFIG.set("Pipeline", "status_server_port", "12346") log_dir = Path(CONFIG.get("Pipeline", "log_dir")) log_dir.mkdir(parents=True, exist_ok=True) @@ -290,6 +292,7 @@ def test_pipeline_logging(spicekernelmanager, out_dir): CONFIG.set("Paths", "fits_archive", str(FITS_ARCHIVE)) CONFIG.set("Pipeline", "log_level", str(LOG_LEVEL)) CONFIG.set("Pipeline", "log_dir", str(LOG_DIR)) + CONFIG.set("Pipeline", "status_server_port", str(SERVER_PORT)) @patch("smtplib.SMTP") diff --git a/stixcore/processing/tests/test_publish.py b/stixcore/processing/tests/test_publish.py index 05ff0dcd..fbb27465 100644 --- a/stixcore/processing/tests/test_publish.py +++ b/stixcore/processing/tests/test_publish.py @@ -232,6 +232,7 @@ def test_fits_incomplete_switch_over(out_dir): product.raw = ["packet1.xml", "packet2.xml"] product.parent = ["packet1.xml", "packet2.xml"] product.level = "L1" + product.LEVEL = "L1" product.service_type = 21 product.service_subtype = 6 product.ssid = 31 diff --git a/stixcore/products/ANC/aspect.py b/stixcore/products/ANC/aspect.py index 011b1e04..0d0c5c6c 100644 --- a/stixcore/products/ANC/aspect.py +++ b/stixcore/products/ANC/aspect.py @@ -208,15 +208,23 @@ def postprocessing(self, result, fits_processor): aux = Ephemeris(control=control, data=data, idb_versions=HK.idb_versions) aux.add_additional_header_keyword( - ("STX_GSW", result.idlgswversion[0].decode(), - "Version of STX-GSW that provided data")) - aux.add_additional_header_keyword( - ("HISTORY", "aspect data processed by STX-GSW", "")) - files.extend([SingleProcessingStepResult(aux.name, aux.level, aux.type, - aux.get_processing_version(), fop, - get_complete_file_name_and_path(file_path), - datetime.now()) - for fop in fits_processor.write_fits(aux)]) + ("STX_GSW", result.idlgswversion[0].decode(), "Version of STX-GSW that provided data") + ) + aux.add_additional_header_keyword(("HISTORY", "aspect data processed by STX-GSW", "")) + files.extend( + [ + SingleProcessingStepResult( + aux.name, + aux.level, + aux.type, + aux.get_processing_version(), + fop, + get_complete_file_name_and_path(file_path), + datetime.now(), + ) + for fop in fits_processor.write_fits(aux) + ] + ) else: logger.error("IDL ERROR") diff --git a/stixcore/products/level1/quicklookL1.py b/stixcore/products/level1/quicklookL1.py index 5d1ebb3e..8ab36106 100644 --- a/stixcore/products/level1/quicklookL1.py +++ b/stixcore/products/level1/quicklookL1.py @@ -22,6 +22,8 @@ class LightCurve(QLProduct, L1Mixin): In level 1 format. """ + NAME = "lightcurve" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -35,12 +37,12 @@ def __init__( **kwargs, ) - self.name = "lightcurve" - self.level = "L1" + self.name = LightCurve.NAME + self.level = LightCurve.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 30 + return kwargs["level"] == LightCurve.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 30 class Background(QLProduct, L1Mixin): @@ -49,6 +51,8 @@ class Background(QLProduct, L1Mixin): In level 1 format. """ + NAME = "background" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -62,12 +66,12 @@ def __init__( **kwargs, ) - self.name = "background" - self.level = "L1" + self.name = Background.NAME + self.level = Background.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 31 + return kwargs["level"] == Background.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 31 class Spectra(QLProduct, L1Mixin): @@ -76,6 +80,8 @@ class Spectra(QLProduct, L1Mixin): In level 1 format. """ + NAME = "spectra" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -89,8 +95,8 @@ def __init__( **kwargs, ) - self.name = "spectra" - self.level = "L1" + self.name = Spectra.NAME + self.level = Spectra.LEVEL @property def dmin(self): @@ -102,7 +108,7 @@ def dmax(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 32 + return kwargs["level"] == Spectra.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 32 class Variance(QLProduct, L1Mixin): @@ -151,6 +157,8 @@ class FlareFlag(QLProduct, L1Mixin): In level 1 format. """ + NAME = "flareflag" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -164,8 +172,8 @@ def __init__( **kwargs, ) - self.name = "flareflag" - self.level = "L1" + self.name = FlareFlag.NAME + self.level = FlareFlag.LEVEL @property def dmin(self): @@ -182,7 +190,7 @@ def bunit(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 34 + return kwargs["level"] == FlareFlag.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 34 class EnergyCalibration(QLProduct, L1Mixin): @@ -191,6 +199,8 @@ class EnergyCalibration(QLProduct, L1Mixin): In level 1 format. """ + NAME = "energy" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -204,13 +214,13 @@ def __init__( **kwargs, ) - self.name = "energy" - self.level = "L1" + self.name = EnergyCalibration.NAME + self.level = EnergyCalibration.LEVEL self.type = "cal" @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 41 + return kwargs["level"] == EnergyCalibration.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 41 class TMStatusFlareList(QLProduct, L1Mixin): @@ -219,6 +229,8 @@ class TMStatusFlareList(QLProduct, L1Mixin): In level 1 format. """ + NAME = "tmstatusflarelist" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -232,8 +244,8 @@ def __init__( **kwargs, ) - self.name = "tmstatusflarelist" - self.level = "L1" + self.name = TMStatusFlareList.NAME + self.level = TMStatusFlareList.LEVEL @property def dmin(self): @@ -252,4 +264,4 @@ def bunit(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 43 + return kwargs["level"] == TMStatusFlareList.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 43 diff --git a/stixcore/products/level1/scienceL1.py b/stixcore/products/level1/scienceL1.py index 62960e8b..08448553 100644 --- a/stixcore/products/level1/scienceL1.py +++ b/stixcore/products/level1/scienceL1.py @@ -15,6 +15,8 @@ class RawPixelData(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "xray-rpd" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -27,12 +29,12 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "xray-rpd" - self.level = "L1" + self.name = RawPixelData.NAME + self.level = RawPixelData.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 20 + return kwargs["level"] == RawPixelData.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 20 class CompressedPixelData(ScienceProduct, L1Mixin): @@ -41,6 +43,8 @@ class CompressedPixelData(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "xray-cpd" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -53,12 +57,14 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "xray-cpd" - self.level = "L1" + self.name = CompressedPixelData.NAME + self.level = CompressedPixelData.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 21 + return ( + kwargs["level"] == CompressedPixelData.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 21 + ) class SummedPixelData(ScienceProduct, L1Mixin): @@ -67,6 +73,8 @@ class SummedPixelData(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "xray-scpd" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -79,12 +87,12 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "xray-scpd" - self.level = "L1" + self.name = SummedPixelData.NAME + self.level = SummedPixelData.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 22 + return kwargs["level"] == SummedPixelData.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 22 class Visibility(ScienceProduct, L1Mixin): @@ -94,6 +102,8 @@ class Visibility(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "xray-vis" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -106,8 +116,8 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "xray-vis" - self.level = "L1" + self.name = Visibility.NAME + self.level = Visibility.LEVEL @property def dmin(self): @@ -126,7 +136,7 @@ def bunit(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 23 + return kwargs["level"] == Visibility.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 23 class Spectrogram(ScienceProduct, L1Mixin): @@ -136,6 +146,8 @@ class Spectrogram(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "xray-spec" + PRODUCT_PROCESSING_VERSION = 4 def __init__( @@ -150,12 +162,12 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "xray-spec" - self.level = "L1" + self.name = Spectrogram.NAME + self.level = Spectrogram.LEVEL @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 24 + return kwargs["level"] == Spectrogram.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 24 class Aspect(ScienceProduct, L1Mixin): @@ -164,6 +176,8 @@ class Aspect(ScienceProduct, L1Mixin): In level 1 format. """ + NAME = "aspect-burst" + def __init__( self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs ): @@ -176,8 +190,8 @@ def __init__( idb_versions=idb_versions, **kwargs, ) - self.name = "aspect-burst" - self.level = "L1" + self.name = Aspect.NAME + self.level = Aspect.LEVEL @property def dmin(self): @@ -207,4 +221,4 @@ def bunit(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == "L1" and service_type == 21 and service_subtype == 6 and ssid == 42 + return kwargs["level"] == Aspect.LEVEL and service_type == 21 and service_subtype == 6 and ssid == 42 diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index f4a73a5c..aef66704 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -33,22 +33,32 @@ from stixpy.map.stix import STIXMap # noqa -__all__ = ['FlarelistSDC', 'FlarePositionMixin', 'FlareSOOPMixin', - 'FlareList', 'FlarelistSDCLocImg', 'FlarePeekPreviewMixin', - 'FlarelistSC', 'FlarelistSCLoc', 'FlarelistSCLocImg'] +__all__ = [ + "FlarelistSDC", + "FlarePositionMixin", + "FlareSOOPMixin", + "FlareList", + "FlarelistSDCLocImg", + "FlarePeekPreviewMixin", + "FlarelistSC", + "FlarelistSCLoc", + "FlarelistSCLocImg", +] logger = get_logger(__name__) -def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_angle, - energy_range): +def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_angle, energy_range): """Create a FITS WCS header for the given data.""" - header = make_fitswcs_header(data, flare_position, - telescope="STIX", - observatory="Solar Orbiter", - scale=scale, - exposure=exposure, - rotation_angle=rotation_angle) + header = make_fitswcs_header( + data, + flare_position, + telescope="STIX", + observatory="Solar Orbiter", + scale=scale, + exposure=exposure, + rotation_angle=rotation_angle, + ) if energy_range is not None and header["wcsaxes"] == 2: # add energy range to WCS header @@ -66,29 +76,27 @@ def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_ class FlarePositionMixin: - """_summary_ - """ + """_summary_""" + @classmethod - def add_flare_position(cls, data, fido_client: STIXClient, *, - filter_function=lambda x: True, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - keep_all_flares=True, - month=None - ): - - data['flare_position'] = [SkyCoord(0, 0, frame="icrs", unit="deg") - for i in range(0, len(data))] - - data['anc_ephemeris_path'] = Column(" " * 500, dtype=str, - description='TDB') - data['cpd_path'] = Column(" " * 500, dtype=str, - description='TDB') - data['_position_status'] = Column(False, dtype=bool, - description='TDB') - data['_position_message'] = Column(" " * 500, dtype=str, - description='TDB') + def add_flare_position( + cls, + data, + fido_client: STIXClient, + *, + filter_function=lambda x: True, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + keep_all_flares=True, + month=None, + ): + data["flare_position"] = [SkyCoord(0, 0, frame="icrs", unit="deg") for i in range(0, len(data))] + + data["anc_ephemeris_path"] = Column(" " * 500, dtype=str, description="TDB") + data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") + data["_position_status"] = Column(False, dtype=bool, description="TDB") + data["_position_message"] = Column(" " * 500, dtype=str, description="TDB") to_remove = [] pass_filter = 0 no_ephemeris = 0 @@ -111,41 +119,41 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, if day in day_asp_ephemeris_cache: anc_res = day_asp_ephemeris_cache[day] else: - anc_res = fido_client.search(a.Time(peak_time, peak_time), - a.Instrument.stix, - a.stix.DataProduct.asp_ephemeris) + anc_res = fido_client.search( + a.Time(peak_time, peak_time), a.Instrument.stix, a.stix.DataProduct.asp_ephemeris + ) if anc_res: anc_res.filter_for_latest_version() url_to_path(anc_res) day_asp_ephemeris_cache[day] = anc_res if len(anc_res) < 1: - logger.warning(f'No ephemeris data found for flare at time ' - f'{start_time} : {end_time}') - data[i]['_position_message'] = "no ephemeris data found" + logger.warning(f"No ephemeris data found for flare at time {start_time} : {end_time}") + data[i]["_position_message"] = "no ephemeris data found" no_ephemeris += 1 continue - data[i]['anc_ephemeris_path'] = anc_res["path"][0] + data[i]["anc_ephemeris_path"] = anc_res["path"][0] if start_time.datetime.hour < 2: start_time = start_time - 2 * u.hour - cpd_res = fido_client.search(a.Time(start_time, end_time), - a.Instrument.stix, - a.stix.DataProduct.sci_xray_cpd) + cpd_res = fido_client.search( + a.Time(start_time, end_time), a.Instrument.stix, a.stix.DataProduct.sci_xray_cpd + ) if cpd_res: cpd_res.filter_for_latest_version() url_to_path(cpd_res) if len(cpd_res) < 1: - logger.warning(f'No CPD data found for flare at time {start_time} : {end_time}') - data[i]['_position_message'] = "no CPD data found" + logger.warning(f"No CPD data found for flare at time {start_time} : {end_time}") + data[i]["_position_message"] = "no CPD data found" no_cpd += 1 continue if len(cpd_res) > 1: - logger.debug(f'Many CPD data found for flare at time {start_time} : {end_time}') + logger.debug(f"Many CPD data found for flare at time {start_time} : {end_time}") # select the best available CPD data file - cpd_res["inc_peak"] = np.logical_and(peak_time >= cpd_res["Start Time"], - peak_time <= cpd_res["End Time"]) + cpd_res["inc_peak"] = np.logical_and( + peak_time >= cpd_res["Start Time"], peak_time <= cpd_res["End Time"] + ) cpd_res["file_size"] = [path.stat().st_size for path in cpd_res["path"]] cpd_res["exposure"] = 0.0 cpd_res["duration"] = 0.0 @@ -165,9 +173,10 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, cpd_res["tbins"][i] = header_data["NAXIS2"] cpd_res["ebins"][i] = len(energies) cpd_res["estart"][i] = energies["e_low"][0].value - cpd_res["eend"][i] = energies["e_high"][-1].value if len(energies) < 31 \ - else energies["e_high"][-2].value - cpd_res["duration"][i] = header["OBT_END"]-header["OBT_BEG"] + cpd_res["eend"][i] = ( + energies["e_high"][-1].value if len(energies) < 31 else energies["e_high"][-2].value + ) + cpd_res["duration"][i] = header["OBT_END"] - header["OBT_BEG"] # TODO: add more criteria to select the best CPD file cpd_res.sort(["tbins", "duration"]) @@ -176,13 +185,13 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, else: one_cpd += 1 best_cpd_idx = 0 - data[i]['cpd_path'] = cpd_res["path"][best_cpd_idx] + data[i]["cpd_path"] = cpd_res["path"][best_cpd_idx] # do the calculations with stixpy - data[i]['flare_position'] = SkyCoord(1, 1, frame="icrs", unit="deg") - data[i]['_position_status'] = True - data[i]['_position_message'] = "OK" + data[i]["flare_position"] = SkyCoord(1, 1, frame="icrs", unit="deg") + data[i]["_position_status"] = True + data[i]["_position_message"] = "OK" else: to_remove.append(i) @@ -190,22 +199,21 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, if not keep_all_flares: data.remove_rows(to_remove) - logger.info(f'Flare position calculated for month {month} with {total_flares} flares, ' - f'passed filter: {pass_filter} no ephemeris data found for {no_ephemeris} ' - f'flares, no CPD data found for {no_cpd} flares, many CPD data found for ' - f'{many_cpd} flares, one CPD data found for {one_cpd} flares') + logger.info( + f"Flare position calculated for month {month} with {total_flares} flares, " + f"passed filter: {pass_filter} no ephemeris data found for {no_ephemeris} " + f"flares, no CPD data found for {no_cpd} flares, many CPD data found for " + f"{many_cpd} flares, one CPD data found for {one_cpd} flares" + ) class FlareSOOPMixin: - """_summary_ - """ + """_summary_""" @classmethod - def add_soop(self, data, *, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC'): - + def add_soop( + self, data, *, peek_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC" + ): soop_encoded_type = list() soop_id = list() soop_type = list() @@ -222,13 +230,13 @@ def add_soop(self, data, *, soop_id.append(None) soop_type.append(None) - data['soop_encoded_type'] = Column(soop_encoded_type, dtype=str, description='campaign ID') - data['soop_id'] = Column(soop_id, dtype=str, description='SOOP ID') - data['soop_type'] = Column(soop_type, dtype=str, description='name of the SOOP campaign') + data["soop_encoded_type"] = Column(soop_encoded_type, dtype=str, description="campaign ID") + data["soop_id"] = Column(soop_id, dtype=str, description="SOOP ID") + data["soop_type"] = Column(soop_type, dtype=str, description="name of the SOOP campaign") class FlarePeekPreviewMixin: - """ Mixin class to add peek preview images to flare list products. + """Mixin class to add peek preview images to flare list products. This class provides a method to generate and add peek preview images to the flare list data. The images are generated based on the flare's peak time, start time, and end time, using the STIXPy library @@ -239,33 +247,35 @@ class FlarePeekPreviewMixin: Currently the images are created for two energy ranges: 4-20 keV and 20-120 keV. """ + @classmethod - def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - anc_ephemeris_path_colname='anc_ephemeris_path', - cpd_path_colname='cpd_path', - product_name_suffix='fl', - keep_all_flares=True, - month=None - ): - data['peek_preview_path'] = Column(" " * 500, dtype=str, - description='TDB') - data['preview_start_UTC'] = [Time(d, format='isot', scale='utc') - for d in data[peek_time_colname]] - data['preview_end_UTC'] = [Time(d, format='isot', scale='utc') - for d in data[peek_time_colname]] - data['_peek_preview_status'] = Column(False, dtype=bool, - description='TDB') - data['_peek_preview_message'] = Column(" " * 500, dtype=str, - description='TDB') + def add_peek_preview( + cls, + data, + energies, + parent, + fido_client: STIXClient, + img_processor, + *, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + anc_ephemeris_path_colname="anc_ephemeris_path", + cpd_path_colname="cpd_path", + product_name_suffix="fl", + keep_all_flares=True, + month=None, + ): + data["peek_preview_path"] = Column(" " * 500, dtype=str, description="TDB") + data["preview_start_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peek_time_colname]] + data["preview_end_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peek_time_colname]] + data["_peek_preview_status"] = Column(False, dtype=bool, description="TDB") + data["_peek_preview_message"] = Column(" " * 500, dtype=str, description="TDB") to_remove = [] products = [] images = 0 for i, row in enumerate(data): - peak_time = row[peek_time_colname] row[start_time_colname] row[end_time_colname] @@ -284,16 +294,16 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p status = True # do the imaging with stixpy - preview_data = data[i:i+1] - del preview_data['peek_preview_path'] - del preview_data['_peek_preview_status'] - del preview_data['_peek_preview_message'] + preview_data = data[i : i + 1] + del preview_data["peek_preview_path"] + del preview_data["_peek_preview_status"] + del preview_data["_peek_preview_message"] peek_preview_start = row[peek_time_colname] - 10 * u.s peek_preview_end = row[peek_time_colname] + 10 * u.s - preview_data['preview_start_UTC'] = peek_preview_start - preview_data['preview_end_UTC'] = peek_preview_end + preview_data["preview_start_UTC"] = peek_preview_start + preview_data["preview_end_UTC"] = peek_preview_end cpd_sci = STIXPYProduct(cpd_path) time_range_sci = [peek_preview_start, peek_preview_end] @@ -302,19 +312,44 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p # flare_position = preview_data['flare_position'][0] # flare_position = [0, 0] * u.arcsec comments = [] - helio_frame = Helioprojective(observer='earth', obstime=peak_time) - flare_position = SkyCoord(0*u.deg, 0*u.deg, frame=helio_frame) - - meta_pixels_sci = create_meta_pixels(cpd_sci, - time_range=time_range_sci, - energy_range=energy_range, - flare_location=flare_position, - no_shadowing=True - ) + helio_frame = Helioprojective(observer="earth", obstime=peak_time) + flare_position = SkyCoord(0 * u.deg, 0 * u.deg, frame=helio_frame) + + meta_pixels_sci = create_meta_pixels( + cpd_sci, + time_range=time_range_sci, + energy_range=energy_range, + flare_location=flare_position, + no_shadowing=True, + ) vis = create_visibility(meta_pixels_sci) cal_vis = calibrate_visibility(vis, flare_location=flare_position) - isc_10_3 = [3, 20, 22, 16, 14, 32, 21, 26, 4, 24, 8, 28, 15, 27, - 31, 6, 30, 2, 25, 5, 23, 7, 29, 1] + isc_10_3 = [ + 3, + 20, + 22, + 16, + 14, + 32, + 21, + 26, + 4, + 24, + 8, + 28, + 15, + 27, + 31, + 6, + 30, + 2, + 25, + 5, + 23, + 7, + 29, + 1, + ] col_idx = np.argwhere(np.isin(cal_vis.meta["isc"], isc_10_3)).ravel() cal_vis.meta["offset"] = flare_position vis10_3 = cal_vis[col_idx] @@ -324,27 +359,26 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p vis_tr = TimeRange(vis.meta["time_range"]) roll, solo_xyz, pointing = get_hpc_info(vis_tr.start, vis_tr.end) - solo = HeliographicStonyhurst(*solo_xyz, obstime=vis_tr.center, - representation_type="cartesian") + solo = HeliographicStonyhurst(*solo_xyz, obstime=vis_tr.center, representation_type="cartesian") clean_map, model_map, resid_map = vis_clean( - vis10_3, imsize, pixel_size=pixel, gain=0.1, niter=200, - clean_beam_width=20 * u.arcsec) + vis10_3, imsize, pixel_size=pixel, gain=0.1, niter=200, clean_beam_width=20 * u.arcsec + ) comments.append(f"clean map with {len(col_idx)} visibilities") - comments.append(f"clean gain: {0.1}, niter: {200}, " - f"clean beam width: {20 * u.arcsec}") + comments.append(f"clean gain: {0.1}, niter: {200}, clean beam width: {20 * u.arcsec}") comments.append(f"det: {', '.join(sorted(vis10_3.meta['vis_labels']))}") map_with_erange = clean_map.data[np.newaxis, ...] map_with_erange[0, :, :] = clean_map.data - fp_hp = flare_position.transform_to(Helioprojective(obstime=vis_tr.center, - observer=solo)) - header = make_stix_fitswcs_header(map_with_erange, fp_hp, - scale=pixel, - exposure=vis_tr.seconds, - rotation_angle=90 * u.deg + roll, - energy_range=energy_range, - ) + fp_hp = flare_position.transform_to(Helioprojective(obstime=vis_tr.center, observer=solo)) + header = make_stix_fitswcs_header( + map_with_erange, + fp_hp, + scale=pixel, + exposure=vis_tr.seconds, + rotation_angle=90 * u.deg + roll, + energy_range=energy_range, + ) header = fits.Header(header) # Add comments @@ -354,10 +388,15 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p maps.append((map_with_erange, header)) - ppi = PeekPreviewImage(control=QTable(), data=preview_data, month=month, - energy=energies, maps=maps, - product_name_suffix=product_name_suffix, - parents=[parent, anc_ephemeris_path.name, cpd_path.name]) + ppi = PeekPreviewImage( + control=QTable(), + data=preview_data, + month=month, + energy=energies, + maps=maps, + product_name_suffix=product_name_suffix, + parents=[parent, anc_ephemeris_path.name, cpd_path.name], + ) for f in img_processor.write_fits(ppi): products.append(f) @@ -368,17 +407,19 @@ def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_p status = False message = str(e) - data[i]['preview_start_UTC'] = peek_preview_start - data[i]['preview_end_UTC'] = peek_preview_end - data[i]['peek_preview_path'] = "test" - data[i]['_peek_preview_status'] = status - data[i]['_peek_preview_message'] = message + data[i]["preview_start_UTC"] = peek_preview_start + data[i]["preview_end_UTC"] = peek_preview_end + data[i]["peek_preview_path"] = "test" + data[i]["_peek_preview_status"] = status + data[i]["_peek_preview_message"] = message if not keep_all_flares: data.remove_rows(to_remove) - logger.info(f'Flare images created for month {month} with {len(data)} flares, ' - f'{len(products)} peek previews created, with total {images} images') + logger.info( + f"Flare images created for month {month} with {len(data)} flares, " + f"{len(products)} peek previews created, with total {images} images" + ) return products @@ -390,12 +431,11 @@ class FlareList(CountDataMixin, GenericProduct, L2Mixin): It is used to store flare data, and can be enhanced with flare positions and SOOP information.""" - LEVEL = 'L3' - TYPE = 'flarelist' + LEVEL = "L3" + TYPE = "flarelist" def __init__(self, *, service_type=0, service_subtype=0, ssid, data, **kwargs): - super().__init__(service_type=0, service_subtype=0, - ssid=ssid, data=data, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=ssid, data=data, **kwargs) self.level = FlareList.LEVEL self.type = FlareList.TYPE self.service_subtype = 0 @@ -404,7 +444,7 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid, data, **kwargs): @property def parent(self): - """ Returns the parent(s) of the flare list product. + """Returns the parent(s) of the flare list product. Returns ------- @@ -415,7 +455,7 @@ def parent(self): @parent.setter def parent(self, value): - """ Sets the parent of the flare list product. + """Sets the parent of the flare list product. Parameters ---------- @@ -425,7 +465,7 @@ def parent(self, value): self._parent.add(value) def enhance_from_product(self, in_prod: GenericProduct): - """ Enhances the flare list product by adding more data. + """Enhances the flare list product by adding more data. Parameters ---------- @@ -439,19 +479,18 @@ class FlarelistSDC(FlareList, FlareSOOPMixin): In L3 product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'sdc' + NAME = "sdc" def __init__(self, *, service_type=0, service_subtype=0, ssid=2, data, month, **kwargs): - super().__init__(service_type=0, service_subtype=0, ssid=2, data=data, **kwargs) self.name = FlarelistSDC.NAME self.ssid = 2 self._start_datetime = datetime.combine(month, datetime.min.time()) - self._end_datetime = self.data['end_UTC'].max() if len(self.data) > 0 \ - else self._start_datetime + self._end_datetime = self.data["end_UTC"].max() if len(self.data) > 0 else self._start_datetime @property def utc_timerange(self): @@ -469,24 +508,23 @@ def split_to_files(self): @property def dmin(self): - return (self.data['lc_peak'].sum(axis=1)).min().value if len(self.data) > 0 else np.nan + return (self.data["lc_peak"].sum(axis=1)).min().value if len(self.data) > 0 else np.nan @property def dmax(self): - return (self.data['lc_peak'].sum(axis=1)).max().value if len(self.data) > 0 else np.nan + return (self.data["lc_peak"].sum(axis=1)).max().value if len(self.data) > 0 else np.nan @property def exposure(self): - return self.data['duration'].min().to_value('s') if len(self.data) > 0 else np.nan + return self.data["duration"].min().to_value("s") if len(self.data) > 0 else np.nan @property def max_exposure(self): - return self.data['duration'].max().to_value('s') if len(self.data) > 0 else np.nan + return self.data["duration"].max().to_value("s") if len(self.data) > 0 else np.nan @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 2) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 2 class FlarelistSDCLoc(FlarelistSDC, FlarePositionMixin): @@ -494,13 +532,12 @@ class FlarelistSDCLoc(FlarelistSDC, FlarePositionMixin): In ANC product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'sdcloc' + NAME = "sdcloc" def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, **kwargs): - - super().__init__(service_type=0, service_subtype=0, ssid=3, - data=data, month=month, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=3, data=data, month=month, **kwargs) self.name = FlarelistSDCLoc.NAME self.ssid = 3 @@ -510,22 +547,24 @@ def enhance_from_product(self, in_prod: GenericProduct): @classmethod def filter_flare_function(cls, col): - return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', - fallback=1000) + return col["lc_peak"][0].value > CONFIG.getint("Processing", "flarelist_sdc_min_count", fallback=1000) @classmethod def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): - super().add_flare_position(data, fido_client, - filter_function=cls.filter_flare_function, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - keep_all_flares=False, month=month) + super().add_flare_position( + data, + fido_client, + filter_function=cls.filter_flare_function, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + keep_all_flares=False, + month=month, + ) @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 3) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 3 class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeekPreviewMixin): @@ -533,13 +572,12 @@ class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeekPreviewMixin): In ANC product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'sdclocimg' + NAME = "sdclocimg" def __init__(self, *, service_type=0, service_subtype=0, ssid=4, data, month, **kwargs): - - super().__init__(service_type=0, service_subtype=0, ssid=4, - data=data, month=month, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=4, data=data, month=month, **kwargs) self.name = FlarelistSDCLocImg.NAME self.ssid = 4 @@ -548,21 +586,26 @@ def enhance_from_product(self, in_prod: GenericProduct): pass @classmethod - def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, - *, month=None): - super().add_peek_preview(data, energies, parent, fido_client, img_processor, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - anc_ephemeris_path_colname='anc_ephemeris_path', - cpd_path_colname='cpd_path', - product_name_suffix=FlarelistSDC.NAME, - keep_all_flares=False, month=month) + def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): + super().add_peek_preview( + data, + energies, + parent, + fido_client, + img_processor, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + anc_ephemeris_path_colname="anc_ephemeris_path", + cpd_path_colname="cpd_path", + product_name_suffix=FlarelistSDC.NAME, + keep_all_flares=False, + month=month, + ) @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 4) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 4 class FlarelistSC(FlareList, FlareSOOPMixin): @@ -570,19 +613,18 @@ class FlarelistSC(FlareList, FlareSOOPMixin): In L3 product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'sc' + NAME = "sc" def __init__(self, *, service_type=0, service_subtype=0, ssid=6, data, month, **kwargs): - super().__init__(service_type=0, service_subtype=0, ssid=6, data=data, **kwargs) self.name = FlarelistSC.NAME self.ssid = 6 self._start_datetime = datetime.combine(month, datetime.min.time()) - self._end_datetime = self.data['end_UTC'].max() if len(self.data) > 0 \ - else self._start_datetime + self._end_datetime = self.data["end_UTC"].max() if len(self.data) > 0 else self._start_datetime @property def utc_timerange(self): @@ -600,24 +642,23 @@ def split_to_files(self): @property def dmin(self): - return (self.data['lc_peak'].sum(axis=1)).min().value if len(self.data) > 0 else np.nan + return (self.data["lc_peak"].sum(axis=1)).min().value if len(self.data) > 0 else np.nan @property def dmax(self): - return (self.data['lc_peak'].sum(axis=1)).max().value if len(self.data) > 0 else np.nan + return (self.data["lc_peak"].sum(axis=1)).max().value if len(self.data) > 0 else np.nan @property def exposure(self): - return self.data['duration'].min().to_value('s') if len(self.data) > 0 else np.nan + return self.data["duration"].min().to_value("s") if len(self.data) > 0 else np.nan @property def max_exposure(self): - return self.data['duration'].max().to_value('s') if len(self.data) > 0 else np.nan + return self.data["duration"].max().to_value("s") if len(self.data) > 0 else np.nan @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 6) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 6 class FlarelistSCLoc(FlarelistSC, FlarePositionMixin): @@ -625,13 +666,12 @@ class FlarelistSCLoc(FlarelistSC, FlarePositionMixin): In L3 product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'scloc' + NAME = "scloc" def __init__(self, *, service_type=0, service_subtype=0, ssid=7, data, month, **kwargs): - - super().__init__(service_type=0, service_subtype=0, ssid=7, - data=data, month=month, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=7, data=data, month=month, **kwargs) self.name = FlarelistSCLoc.NAME self.ssid = 7 @@ -641,22 +681,24 @@ def enhance_from_product(self, in_prod: GenericProduct): @classmethod def filter_flare_function(cls, col): - return col['lc_peak'][0].value > CONFIG.getint('Processing', 'flarelist_sdc_min_count', - fallback=1000) + return col["lc_peak"][0].value > CONFIG.getint("Processing", "flarelist_sdc_min_count", fallback=1000) @classmethod def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): - super().add_flare_position(data, fido_client, - filter_function=cls.filter_flare_function, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - keep_all_flares=False, month=month) + super().add_flare_position( + data, + fido_client, + filter_function=cls.filter_flare_function, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + keep_all_flares=False, + month=month, + ) @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 7) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 7 class FlarelistSCLocImg(FlarelistSCLoc, FlarePeekPreviewMixin): @@ -664,13 +706,12 @@ class FlarelistSCLocImg(FlarelistSCLoc, FlarePeekPreviewMixin): In ANC product format. """ + PRODUCT_PROCESSING_VERSION = 2 - NAME = 'sclocimg' + NAME = "sclocimg" def __init__(self, *, service_type=0, service_subtype=0, ssid=8, data, month, **kwargs): - - super().__init__(service_type=0, service_subtype=0, ssid=8, - data=data, month=month, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=8, data=data, month=month, **kwargs) self.name = FlarelistSCLocImg.NAME self.ssid = 8 @@ -679,18 +720,23 @@ def enhance_from_product(self, in_prod: GenericProduct): pass @classmethod - def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, - *, month=None): - super().add_peek_preview(data, energies, parent, fido_client, img_processor, - peek_time_colname='peak_UTC', - start_time_colname='start_UTC', - end_time_colname='end_UTC', - anc_ephemeris_path_colname='anc_ephemeris_path', - cpd_path_colname='cpd_path', - product_name_suffix=FlarelistSC.NAME, - keep_all_flares=False, month=month) + def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): + super().add_peek_preview( + data, + energies, + parent, + fido_client, + img_processor, + peek_time_colname="peak_UTC", + start_time_colname="start_UTC", + end_time_colname="end_UTC", + anc_ephemeris_path_colname="anc_ephemeris_path", + cpd_path_colname="cpd_path", + product_name_suffix=FlarelistSC.NAME, + keep_all_flares=False, + month=month, + ) @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L3' and service_type == 0 - and service_subtype == 0 and ssid == 8) + return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 8 diff --git a/stixcore/products/level3/flarelistproduct.py b/stixcore/products/level3/flarelistproduct.py index 0330c72e..4a3fbdaa 100644 --- a/stixcore/products/level3/flarelistproduct.py +++ b/stixcore/products/level3/flarelistproduct.py @@ -5,7 +5,7 @@ from stixcore.products.product import GenericProduct, L3Mixin from stixcore.time.datetime import SCETime, SCETimeRange -__all__ = ["FlareListProduct", "FlareOverviewImage"] +__all__ = ["FlareListProduct", "PeekPreviewImage"] class FlareListProduct(GenericProduct, L3Mixin): @@ -26,8 +26,7 @@ class PeekPreviewImage(FlareListProduct): Name = "peekpreviewimg" def __init__(self, control, data, energy, maps, parents, *, product_name_suffix="", **kwargs): - super().__init__(service_type=0, service_subtype=0, ssid=5, control=control, - data=data, energy=energy, **kwargs) + super().__init__(service_type=0, service_subtype=0, ssid=5, control=control, data=data, energy=energy, **kwargs) self.name = f"{PeekPreviewImage.Name}-{product_name_suffix}" self.level = PeekPreviewImage.Level self.type = PeekPreviewImage.Type @@ -35,8 +34,7 @@ def __init__(self, control, data, energy, maps, parents, *, product_name_suffix= self.maps = maps self.parents = parents - self.add_additional_header_keyword(('NR_MAPS', len(maps) if maps else 0, - 'number of maps in file')) + self.add_additional_header_keyword(("NR_MAPS", len(maps) if maps else 0, "number of maps in file")) @property def parent(self): @@ -58,5 +56,4 @@ def split_to_files(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == PeekPreviewImage.Level and service_type == 0 - and service_subtype == 0 and ssid == 5) + return kwargs["level"] == PeekPreviewImage.Level and service_type == 0 and service_subtype == 0 and ssid == 5 diff --git a/stixcore/products/lowlatency/quicklookLL.py b/stixcore/products/lowlatency/quicklookLL.py index db5b1645..e151cf22 100644 --- a/stixcore/products/lowlatency/quicklookLL.py +++ b/stixcore/products/lowlatency/quicklookLL.py @@ -50,9 +50,9 @@ class LightCurveL3(QLProduct, L1Mixin): for Low Latency Processing Level 3 format - svg chart """ + def __init__(self, *, control, data, parent_file_path: Path, **kwargs): - super().__init__(service_type=21, service_subtype=6, - ssid=34, control=control, data=data, **kwargs) + super().__init__(service_type=21, service_subtype=6, ssid=34, control=control, data=data, **kwargs) self.name = LightCurveL3.NAME self.level = LightCurveL3.LEVEL self.type = LightCurveL3.TYPE diff --git a/stixcore/products/product.py b/stixcore/products/product.py index d067f828..ae9ba16f 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -89,7 +89,7 @@ def read_qtable(file, hdu, hdul=None): if hasattr(dtype, "subdtype"): dtype = dtype.base - if col.coord_type != 'UTC': + if col.coord_type != "UTC": qtable[col.name] = qtable[col.name].astype(dtype) else: qtable[col.name].format = "isot" @@ -257,8 +257,7 @@ def __call__(self, *args, **kwargs): service_subtype = 6 ssid = 34 - if level not in ["LB", "LL01"] and 'timedel' in data.colnames \ - and 'time' in data.colnames: + if level not in ["LB", "LL01"] and "timedel" in data.colnames and "time" in data.colnames: data["timedel"] = SCETimeDelta(data["timedel"]) offset = SCETime.from_float(pri_header["OBT_BEG"] * u.s) @@ -300,11 +299,21 @@ def __call__(self, *args, **kwargs): if hasattr(Product, "TYPE") and Product.TYPE == "flarelist": month = datetime.fromisoformat(pri_header["DATE-BEG"]).date() - p = Product(level=level, service_type=service_type, - service_subtype=service_subtype, - ssid=ssid, control=control, - data=data, energies=energies, idb_versions=idb_versions, - raw=raw, parent=parent, comment=comment, history=history, month=month) + p = Product( + level=level, + service_type=service_type, + service_subtype=service_subtype, + ssid=ssid, + control=control, + data=data, + energies=energies, + idb_versions=idb_versions, + raw=raw, + parent=parent, + comment=comment, + history=history, + month=month, + ) # store the old fits header for later reuse if isinstance(p, (L1Mixin, L2Mixin)): @@ -522,13 +531,11 @@ def scet_timerange(self): @property def raw(self): - return np.unique(self.control["raw_file"]).tolist() \ - if "raw_file" in self.control.colnames else [] + return np.unique(self.control["raw_file"]).tolist() if "raw_file" in self.control.colnames else [] @property def parent(self): - return np.unique(self.control["parent"]).tolist() \ - if "parent" in self.control.colnames else [] + return np.unique(self.control["parent"]).tolist() if "parent" in self.control.colnames else [] @property def dmin(self): @@ -555,7 +562,7 @@ def max_exposure(self): # default for FITS HEADER return 0.0 - def find_parent_products(self, root): + def find_parent_products(self, root): """ Convenient way to get access to the parent products. @@ -895,6 +902,8 @@ def add_additional_header_keywords(self, keywords): class L1Mixin(FitsHeaderMixin): + LEVEL = "L1" + @property def utc_timerange(self): return self.scet_timerange.to_timerange() diff --git a/stixcore/util/util.py b/stixcore/util/util.py index 8dd2446a..499e38fe 100644 --- a/stixcore/util/util.py +++ b/stixcore/util/util.py @@ -9,7 +9,7 @@ "get_complete_file_name_and_path", "get_incomplete_file_name_and_path", "is_incomplete_file_name", - "url_to_path" + "url_to_path", ] @@ -39,6 +39,7 @@ def get_incomplete_file_name(name): def url_to_path(fido_res: StixQueryResponse): - if 'url' in fido_res.columns: - fido_res['path'] = [Path(url.replace("file:", "")) - if url.startswith("file:") else None for url in fido_res['url']] + if "url" in fido_res.columns: + fido_res["path"] = [ + Path(url.replace("file:", "")) if url.startswith("file:") else None for url in fido_res["url"] + ] From 34f27b810fa8e5d7083167f3741f8f76177ddb97 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 30 Sep 2025 14:07:54 +0200 Subject: [PATCH 07/11] fix missing isnan from merge conflicts fix tests fix tests fix tests --- stixcore/io/fits/processors.py | 3 +-- stixcore/io/fits/tests/test_processors.py | 7 +++++++ stixcore/processing/tests/test_publish.py | 11 +++++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 25d458dd..ebb3ed79 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -39,8 +39,7 @@ def empty_if_nan(val): - return "" if isnan(val) else val - + return "" if np.isnan(val) else val def version_format(version): diff --git a/stixcore/io/fits/tests/test_processors.py b/stixcore/io/fits/tests/test_processors.py index 0b8b80e7..b8840d1c 100644 --- a/stixcore/io/fits/tests/test_processors.py +++ b/stixcore/io/fits/tests/test_processors.py @@ -244,8 +244,15 @@ def test_level1_processor_generate_primary_header(product, soop_manager): product.utc_timerange = product.scet_timerange.to_timerange() product.raw = ["packet1.xml", "packet2.xml"] product.parent = ["l01.fits", "l02.fts"] + product.LEVEL = "L1" product.level = "L1" product.type = "ql" + product.TYPE = "ql" + product.dmin = 1 + product.dmax = 1 + product.dunit = "" + product.max_exposure = 1 + product.exposure = 1 product.service_type = 1 product.service_subtype = 2 product.ssid = 3 diff --git a/stixcore/processing/tests/test_publish.py b/stixcore/processing/tests/test_publish.py index fbb27465..53bea6bf 100644 --- a/stixcore/processing/tests/test_publish.py +++ b/stixcore/processing/tests/test_publish.py @@ -125,12 +125,14 @@ def test_publish_fits_to_esa_incomplete(product, out_dir): product.raw = ["packet1.xml", "packet2.xml"] product.parent = ["packet1.xml", "packet2.xml"] product.level = "L1" + product.LEVEL = "L1" product.service_type = 21 product.service_subtype = 6 product.ssid = 31 product.fits_daily_file = True product.type = "ql" product.name = "background" + product.NAME = "background" product.obt_beg = beg product.obt_end = end product.date_obs = beg @@ -233,6 +235,7 @@ def test_fits_incomplete_switch_over(out_dir): product.parent = ["packet1.xml", "packet2.xml"] product.level = "L1" product.LEVEL = "L1" + product.NAME = "background" product.service_type = 21 product.service_subtype = 6 product.ssid = 31 @@ -316,16 +319,16 @@ def test_fits_incomplete_switch_over(out_dir): @pytest.mark.skipif(sys.platform.startswith("win"), reason="does not run on windows") -@pytest.mark.skipif(sys.platform != "Darwin", reason="does not run on mac") +@pytest.mark.skipif(sys.platform == "Darwin", reason="does not run on mac") def test_fits_incomplete_switch_over_remove_dup_files(out_dir): test_fits_incomplete_switch_over(out_dir) fits_dir = out_dir / "fits" target_dir = out_dir / "move" - ufiles = list(fits_dir.rglob("*_V*U.fits")) + ufiles = sorted(list(fits_dir.rglob("*_V*U.fits"))) p = re.compile(r".*_V[0-9]+\.fits") - cfiles = [f for f in fits_dir.rglob("*.fits") if p.match(f.name)] + cfiles = sorted([f for f in fits_dir.rglob("*.fits") if p.match(f.name)]) assert len(ufiles) == 3 assert len(cfiles) == 4 @@ -340,7 +343,7 @@ def test_fits_incomplete_switch_over_remove_dup_files(out_dir): assert len(res) == 1 moved = list(target_dir.rglob("*.fits")) assert len(moved) == 1 - assert moved[0].name == cfiles[3].name + assert moved[0].name == cfiles[2].name @patch("stixcore.products.level1.scienceL1.Spectrogram") From aa50fdb618fb5cba740efcc42aef41b8a66f0256 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 14 Oct 2025 13:26:47 +0200 Subject: [PATCH 08/11] add future migration logic to ProcessingHistoryStorage --- stixcore/io/ProcessingHistoryStorage.py | 74 +++++++++++++------------ 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/stixcore/io/ProcessingHistoryStorage.py b/stixcore/io/ProcessingHistoryStorage.py index 56acab0a..4d593971 100644 --- a/stixcore/io/ProcessingHistoryStorage.py +++ b/stixcore/io/ProcessingHistoryStorage.py @@ -10,6 +10,8 @@ class ProcessingHistoryStorage: """Persistent handler for meta data on already processed Products""" + DB_VERSION = 1 + def __init__(self, filename): """Create a new persistent handler. Will open or create the given sqlite DB file. @@ -22,6 +24,7 @@ def __init__(self, filename): self.cur = None self.filename = filename self._connect_database() + self._migrate_database() def _connect_database(self): """Connects to the sqlite file or creates an empty one if not present.""" @@ -30,45 +33,48 @@ def _connect_database(self): self.cur = self.conn.cursor() logger.info(f"ProcessingHistoryStorage DB loaded from {self.filename}") - # TODO reactivate later - # self.cur.execute('''CREATE TABLE if not exists processed_flare_products ( - # id INTEGER PRIMARY KEY AUTOINCREMENT, - # flareid TEXT NOT NULL, - # flarelist TEXT NOT NULL, - # version INTEGER NOT NULL, - # name TEXT NOT NULL, - # level TEXT NOT NULL, - # type TEXT NOT NULL, - # fitspath TEXT NOT NULL, - # p_date FLOAT NOT NULL - # ) - # ''') - # self.cur.execute('''CREATE INDEX if not exists processed_flare_products_idx ON - # processed_flare_products (flareid, flarelist, version, name, - # level, type)''') - - self.cur.execute("""CREATE TABLE if not exists processed_fits_products ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - level TEXT NOT NULL, - type TEXT NOT NULL, - version INTEGER NOT NULL, - fits_in_path TEXT NOT NULL, - fits_out_path TEXT NOT NULL, - p_date TEXT NOT NULL - ) - """) - - self.cur.execute("""CREATE INDEX if not exists processed_fits_products_idx ON - processed_fits_products - (name, level, type, version, fits_in_path)""") - - self.conn.commit() except sqlite3.Error: logger.error(f"Failed load DB from {self.filename}") self.close() raise + def _migrate_database(self): + """Migrate the database to the latest version if needed.""" + try: + curent_DB_version = self.cur.execute("PRAGMA user_version;").fetchone()[0] + + if curent_DB_version < self.DB_VERSION: + logger.info(f"Migrating DB from version {curent_DB_version} to {self.DB_VERSION}") + + if curent_DB_version < 1: + self.cur.execute("""CREATE TABLE if not exists processed_fits_products ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + level TEXT NOT NULL, + type TEXT NOT NULL, + version INTEGER NOT NULL, + fits_in_path TEXT NOT NULL, + fits_out_path TEXT NOT NULL, + p_date TEXT NOT NULL + ) + """) + + self.cur.execute("""CREATE INDEX if not exists processed_fits_products_idx ON + processed_fits_products + (name, level, type, version, fits_in_path)""") + if curent_DB_version < 2: + # future migrations here + pass + self.cur.execute(f"PRAGMA user_version = {self.DB_VERSION};") + self.conn.commit() + logger.info(f"DB migration done up to version {self.DB_VERSION}") + else: + logger.info(f"DB is already at version {curent_DB_version} no migration to do") + except sqlite3.Error: + logger.error(f"Failed to migrate DB to version {self.DB_VERSION}") + self.close() + raise + def close(self): """Close the IDB connection.""" if self.conn: From 99a0b6130d65ea6cefcf7dbc5ca201dc23f57dcc Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 14 Oct 2025 16:58:09 +0200 Subject: [PATCH 09/11] preperation for merge and release --- docs/code_ref/io.rst | 4 +- processed.sqlite | Bin 24576 -> 0 bytes stixcore/io/FlareListManager.py | 4 +- .../{ => product_processors}/fits/__init__.py | 0 .../fits/processors.py | 103 ----------- .../io/product_processors/plots/processors.py | 115 ++++++++++++ .../tests/__init__.py | 0 .../tests/test_processors.py | 23 ++- stixcore/processing/L0toL1.py | 2 +- stixcore/processing/L1toL2.py | 2 +- stixcore/processing/LBtoL0.py | 2 +- stixcore/processing/LL.py | 2 +- stixcore/processing/TMTCtoLB.py | 2 +- stixcore/processing/TMtoLL01.py | 2 +- stixcore/processing/pipeline_daily.py | 174 +++++++++--------- stixcore/processing/publish.py | 11 -- stixcore/processing/tests/test_publish.py | 2 +- stixcore/processing/tests/test_sswidl.py | 2 +- stixcore/products/tests/test_housekeeping.py | 2 +- 19 files changed, 236 insertions(+), 216 deletions(-) delete mode 100644 processed.sqlite rename stixcore/io/{ => product_processors}/fits/__init__.py (100%) rename stixcore/io/{ => product_processors}/fits/processors.py (93%) create mode 100644 stixcore/io/product_processors/plots/processors.py rename stixcore/io/{fits => product_processors}/tests/__init__.py (100%) rename stixcore/io/{fits => product_processors}/tests/test_processors.py (92%) diff --git a/docs/code_ref/io.rst b/docs/code_ref/io.rst index 71775a1e..5962f501 100644 --- a/docs/code_ref/io.rst +++ b/docs/code_ref/io.rst @@ -6,8 +6,8 @@ to certain directories. .. automodapi:: stixcore.io -.. automodapi:: stixcore.io.fits -.. automodapi:: stixcore.io.fits.processors +.. automodapi:: stixcore.io.product_processors.fits.processors +.. automodapi:: stixcore.io.product_processors.plots.processors .. automodapi:: stixcore.io.soc .. automodapi:: stixcore.io.soc.manager diff --git a/processed.sqlite b/processed.sqlite deleted file mode 100644 index 9c99a5ac2e3ea722f0283d14532dc885a1bced40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24576 zcmeI2O>YxN7{_fVvEwARAprs;63d|yh-5u)`x+~4N?nv<0u82sda&HYsFp$!ViQ3R zm26c-YCk{^{RBO5w7a*n zQK{7`o27fzdaVSfn-4eOO?9*1SEHBL^DD)?wz~FS{*E@_dhPn!gf`^pbNSHgE^L?Y zR~EFb%6?^QL90LBg_HftUah*by`Z(6RJTjJ<@%==Z^YuWmo7!`*2|x4Rca5mAZsht zDi0o3w!ydmbmYKUapm1YUJHEL{ztg-;uo=KVs>dM`ed&4n|&9KoL*`6ahs*5l{pa0YyL&_`eevo96fCqnTeL@KA>$pa>`eihv@Z z2q*%IfFhs>C<2OrBA^Joxdi5-BTLafJ2d7ez}Ng(Ba-mX1e^39E zewzL|T}@v}zcu+Zes27mvBt!Mq>=o2G82C>bv5yE;?LBRso&mQEK0c|pa>`e|1p8a z{P>yU=c3I_w6?jiwX;#6_qNJ=m91*69?QZmu5z|kuRhAc4zO&!QmbbxwQ>o*u9Rz) z690f^3p7g*VOazZoPEErvYs!j-YjNoJ6k)Y0xj|D7rh@{guf~Ac*GDcqSmq4IyYM9 zj8R(p2oarcJe!xOj)P8q?Tx;csJ95Ho{UmZqmb{0mSy0h}sVJxH$ltqr?N-rlCAha$%t};eim5aG zC5y~<$(qRVO7@gQb(WYqE1=G9m#l*I^Y;?f$J>vIfP_Dh%3F?@mZmXN#$8 zv+All=wT=(9=jQd>co1KgErb#hg<9jGP>%f<*CC+h}}W&>#6JXe+Dx>cE@F@gGwjV z-x(;_-w}+9*uf_2_QuB~suL>NPVd`c*HKF>Y>_Aj{gNb`2GLs$bKaAcKY+wi>be+mL@-&c%J+tX^rnE zp2c6r>Db;VjlGOMgCJ=~GdDdMX-|8bCs+RQ#8;@w!+;(brVi=M1ZUo%1;7NqJ^4OV z1A?)r+w9uS%Ur~$h^dPV7g;(X21B%&J1!C55%jiV_`(b5;0x2^2k=ps*gB?;P3&gw zm^6GxG8~BD6A^HLuhYr*bozWqvI!yG^raUcaShvLhHg=t+N7Dw$ix@A?eK;ZYP=G} z7g~Ua&);}EIDKPg0H5I!q%+g7F>NfTCnBx!w3*>u-{B*wpa=O=62AcgRexhxVt~))1WQ1E(hx9!T*J{} z*$gpg#L2-1T9ZvN14#ZFl-~d{T}E|S$}lV_f$<>*_*_`ffc%s6U3A3XOPlhyk7?2^x^U2p=%OTm7ZzyRUB z5cEL~u^Y>=AqIFBD`-G$Bn~XkKz7y24t+No0tTQNVBNvUFrW%WhZqnaiNP>D!r#a7 z&qg7=eSK}tZ-CjZ0poySz(~yV|A~>WBI)l_&8f?iqsbq}zf4?-kB|NcAF4wUPz3%Z z0?pjKw6yw2ZUzf>!=VSl&Z1738`8ZU8HHZVHDFc3q7FgL+?-5&@&^$?e4zz+`26=7 zh7M$WF6}a#C-sa1zOyp%$xmej@r4%P;R|s~-(UNf=~AR)!W@R1xib>+9YHQGhA+H; z4nEQK0H}S~#g^_kg!0CFS|+~GZKt&sA7(8^5MO8k9zK8L?JRTpHbwYrAGKUUb=bvY zJFp@27i diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index f4c06d23..38359be6 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -95,8 +95,7 @@ def read_flarelist(self): """ if self.update or not self.file.exists(): last_date = datetime(2020, 1, 1, 0, 0, 0) - last_date = datetime(2025, 4, 12, 0, 0, 0) - # only run flare detection for time oldet than 7 days + # only run flare detection for time older than 7 days today = datetime.now() - timedelta(days=7) flare_df_lists = [] if self.file.exists(): @@ -141,6 +140,7 @@ def read_flarelist(self): f"with {len(ql_lc_files)} lightcurve files and " f"{len(ql_bg_files)} background files" ) + # TODO re-enable flare-list creation # flares = stixpy.detect_flares(start, end, # ql_lc_files=ql_lc_files, # ql_bg_files=ql_bg_files) diff --git a/stixcore/io/fits/__init__.py b/stixcore/io/product_processors/fits/__init__.py similarity index 100% rename from stixcore/io/fits/__init__.py rename to stixcore/io/product_processors/fits/__init__.py diff --git a/stixcore/io/fits/processors.py b/stixcore/io/product_processors/fits/processors.py similarity index 93% rename from stixcore/io/fits/processors.py rename to stixcore/io/product_processors/fits/processors.py index ebb3ed79..a3bbfd3d 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/product_processors/fits/processors.py @@ -1,10 +1,8 @@ """Module for the different processing levels.""" -from pathlib import Path from datetime import datetime import numpy as np -from matplotlib import pyplot as plt import astropy.units as u from astropy.io import fits @@ -27,7 +25,6 @@ "FitsL0Processor", "FitsL1Processor", "FitsL2Processor", - "PlotProcessor", ] @@ -1084,106 +1081,6 @@ def write_fits(self, prod, *, version=0): return [filetowrite] -class PlotProcessor(FitsL2Processor): - """A file product processor for plot images""" - - def __init__(self, archive_path): - """Creates a new PlotProcessor object. - - Parameters - ---------- - archive_path : Path - the output root path where the files should be created - """ - super().__init__(archive_path) - - def generate_filename(self, product, *, version=0, suffix=".svg"): - """Generates a SOAR conform filename based on product characteristics. - - Parameters - ---------- - product : Product - The data product the file name should be generated for - version : int, optional - The file version, by default 0 = detect from codebase - suffix : str, optional - file name suffix like svg, png, ..., by default '.svg' - - Returns - ------- - Path - a Path object with full name and path - """ - p = Path(super().generate_filename(product=product, version=version, header=True, status="C")) - return p.with_suffix(suffix).name - - def generate_primary_header(self, filename, product, *, version=0): - """Transforms the fits header into a more generic header dict - that might be used in other output file formats - - Parameters - ---------- - filename : Path - The envisoned file name and path for the product - product : Product - The products holding the data - version : int, optional - the processing version, by default 0 = detect from codebase - - Returns - ------- - dict - a dict of header keywords and values - """ - l1, l2 = super().generate_primary_header(filename, product, version=version) - header = dict() - for k, v in l1: - header[k] = v - for kv in l2: - header[kv[0]] = kv[1] - return header - - def write_plot(self, product, *, version=0): - """ - Write products into a plot image file. - - Parameters - ---------- - product : `stixcore.product.level2` - - version : `int` - the version modifier for the filename - default 0 = detect from codebase. - - Returns - ------- - list - of created file as `pathlib.Path` - - """ - if version == 0: - version = product.get_processing_version() - - filename = self.generate_filename(product=product, version=version) - - # headers = self.generate_primary_header(filename, product, version=version) - # headers['parent'] = get_complete_file_name(product.parent_file_path.name) - - parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), product.type.upper()] - # for science data use start date - if product.type in ["sci", "flarelist"]: - parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d") - path = self.archive_path.joinpath(*[str(x) for x in parts]) - path.mkdir(parents=True, exist_ok=True) - - plot_path = path / filename - - fig = product.get_plot() - fig.savefig(plot_path, format="svg") - plt.close(fig) - return plot_path - - class FitsL3Processor(FitsL2Processor): def __init__(self, archive_path): super().__init__(archive_path) diff --git a/stixcore/io/product_processors/plots/processors.py b/stixcore/io/product_processors/plots/processors.py new file mode 100644 index 00000000..3461a016 --- /dev/null +++ b/stixcore/io/product_processors/plots/processors.py @@ -0,0 +1,115 @@ +"""Module for the different processing levels.""" + +from pathlib import Path + +from matplotlib import pyplot as plt + +from stixcore.io.product_processors.fits.processors import FitsL2Processor +from stixcore.util.logging import get_logger + +__all__ = [ + "PlotProcessor", +] + + +logger = get_logger(__name__) + + +class PlotProcessor(FitsL2Processor): + """A file product processor for plot images""" + + def __init__(self, archive_path): + """Creates a new PlotProcessor object. + + Parameters + ---------- + archive_path : Path + the output root path where the files should be created + """ + super().__init__(archive_path) + + def generate_filename(self, product, *, version=0, suffix=".svg"): + """Generates a SOAR conform filename based on product characteristics. + + Parameters + ---------- + product : Product + The data product the file name should be generated for + version : int, optional + The file version, by default 0 = detect from codebase + suffix : str, optional + file name suffix like svg, png, ..., by default '.svg' + + Returns + ------- + Path + a Path object with full name and path + """ + p = Path(super().generate_filename(product=product, version=version, header=True, status="C")) + return p.with_suffix(suffix).name + + def generate_primary_header(self, filename, product, *, version=0): + """Transforms the fits header into a more generic header dict + that might be used in other output file formats + + Parameters + ---------- + filename : Path + The envisoned file name and path for the product + product : Product + The products holding the data + version : int, optional + the processing version, by default 0 = detect from codebase + + Returns + ------- + dict + a dict of header keywords and values + """ + l1, l2 = super().generate_primary_header(filename, product, version=version) + header = dict() + for k, v in l1: + header[k] = v + for kv in l2: + header[kv[0]] = kv[1] + return header + + def write_plot(self, product, *, version=0): + """ + Write products into a plot image file. + + Parameters + ---------- + product : `stixcore.product.level2` + + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + + Returns + ------- + list + of created file as `pathlib.Path` + + """ + if version == 0: + version = product.get_processing_version() + + filename = self.generate_filename(product=product, version=version) + + # headers = self.generate_primary_header(filename, product, version=version) + # headers['parent'] = get_complete_file_name(product.parent_file_path.name) + + parts = [product.level, product.utc_timerange.center.strftime("%Y/%m/%d"), product.type.upper()] + # for science data use start date + if product.type in ["sci", "flarelist"]: + parts[1] = product.utc_timerange.start.strftime("%Y/%m/%d") + path = self.archive_path.joinpath(*[str(x) for x in parts]) + path.mkdir(parents=True, exist_ok=True) + + plot_path = path / filename + + fig = product.get_plot() + fig.savefig(plot_path, format="svg") + plt.close(fig) + return plot_path diff --git a/stixcore/io/fits/tests/__init__.py b/stixcore/io/product_processors/tests/__init__.py similarity index 100% rename from stixcore/io/fits/tests/__init__.py rename to stixcore/io/product_processors/tests/__init__.py diff --git a/stixcore/io/fits/tests/test_processors.py b/stixcore/io/product_processors/tests/test_processors.py similarity index 92% rename from stixcore/io/fits/tests/test_processors.py rename to stixcore/io/product_processors/tests/test_processors.py index b8840d1c..bb0fe4ec 100644 --- a/stixcore/io/fits/tests/test_processors.py +++ b/stixcore/io/product_processors/tests/test_processors.py @@ -6,7 +6,12 @@ from astropy.table import QTable from stixcore.data.test import test_data -from stixcore.io.fits.processors import FitsL0Processor, FitsL1Processor, FitsLBProcessor +from stixcore.io.product_processors.fits.processors import ( + FitsL0Processor, + FitsL1Processor, + FitsLBProcessor, +) +from stixcore.io.product_processors.plots.processors import PlotProcessor from stixcore.products.product import Product from stixcore.soop.manager import SOOPManager from stixcore.time import SCETime, SCETimeRange @@ -51,7 +56,7 @@ def test_levelb_processor_generate_filename_without_rid(): @patch("stixcore.products.level0.quicklookL0.QLProduct") -@patch("stixcore.io.fits.processors.datetime") +@patch("stixcore.io.product_processors.fits.processors.datetime") def test_levelb_processor_generate_primary_header(datetime, product): processor = FitsLBProcessor("some/path") datetime.now().isoformat.return_value = "1234-05-07T01:02:03.346" @@ -136,7 +141,7 @@ def test_level0_processor_generate_filename(): @patch("stixcore.products.level0.quicklookL0.QLProduct") -@patch("stixcore.io.fits.processors.datetime") +@patch("stixcore.io.product_processors.fits.processors.datetime") def test_level0_processor_generate_primary_header(datetime, product): processor = FitsL0Processor("some/path") datetime.now().isoformat.return_value = "1234-05-07T01:02:03.346" @@ -290,3 +295,15 @@ def test_level1_processor_generate_primary_header(product, soop_manager): assert np.allclose(test_data[name], value) else: assert value == test_data[name] + + +@patch("stixcore.products.lowlatency.quicklookLL.LightCurveL3") +def test_plot_processor_generate_filename(product): + processor = PlotProcessor("some/path") + product.type = "ql" + product.scet_timerange = SCETimeRange(start=SCETime(0, 0), end=SCETime(coarse=0, fine=2**16 - 1)) + product.utc_timerange = product.scet_timerange.to_timerange() + product.level = "LL03" + product.name = "lightcurve" + filename = processor.generate_filename(product, version=1) + assert filename == "solo_LL03_stix-ql-lightcurve_20000101_V01C.svg" diff --git a/stixcore/processing/L0toL1.py b/stixcore/processing/L0toL1.py index d1c280ce..51c19e76 100644 --- a/stixcore/processing/L0toL1.py +++ b/stixcore/processing/L0toL1.py @@ -8,7 +8,7 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager -from stixcore.io.fits.processors import FitsL1Processor +from stixcore.io.product_processors.fits.processors import FitsL1Processor from stixcore.products import Product from stixcore.products.level0.scienceL0 import NotCombineException from stixcore.soop.manager import SOOPManager diff --git a/stixcore/processing/L1toL2.py b/stixcore/processing/L1toL2.py index 4c834712..27b11787 100644 --- a/stixcore/processing/L1toL2.py +++ b/stixcore/processing/L1toL2.py @@ -8,7 +8,7 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice -from stixcore.io.fits.processors import FitsL2Processor +from stixcore.io.product_processors.fits.processors import FitsL2Processor from stixcore.processing.sswidl import SSWIDLProcessor from stixcore.products import Product from stixcore.products.level0.scienceL0 import NotCombineException diff --git a/stixcore/processing/LBtoL0.py b/stixcore/processing/LBtoL0.py index 9c771ca5..963748e9 100644 --- a/stixcore/processing/LBtoL0.py +++ b/stixcore/processing/LBtoL0.py @@ -7,7 +7,7 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager from stixcore.idb.manager import IDBManager -from stixcore.io.fits.processors import FitsL0Processor +from stixcore.io.product_processors.fits.processors import FitsL0Processor from stixcore.io.RidLutManager import RidLutManager from stixcore.products.level0.scienceL0 import NotCombineException from stixcore.products.product import Product diff --git a/stixcore/processing/LL.py b/stixcore/processing/LL.py index 9c99d7d3..7c4a1afd 100644 --- a/stixcore/processing/LL.py +++ b/stixcore/processing/LL.py @@ -3,8 +3,8 @@ from astropy.io import fits -from stixcore.io.fits.processors import PlotProcessor from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.io.product_processors.plots.processors import PlotProcessor from stixcore.processing.SingleStep import ( SingleProcessingStepResult, SingleProductProcessingStepMixin, diff --git a/stixcore/processing/TMTCtoLB.py b/stixcore/processing/TMTCtoLB.py index 39e023f6..a03bd2e7 100644 --- a/stixcore/processing/TMTCtoLB.py +++ b/stixcore/processing/TMTCtoLB.py @@ -4,7 +4,7 @@ from concurrent.futures import ProcessPoolExecutor from stixcore.config.config import CONFIG -from stixcore.io.fits.processors import FitsLBProcessor +from stixcore.io.product_processors.fits.processors import FitsLBProcessor from stixcore.io.soc.manager import SOCManager from stixcore.products.levelb.binary import LevelB from stixcore.tmtc.packets import TMTC diff --git a/stixcore/processing/TMtoLL01.py b/stixcore/processing/TMtoLL01.py index a4ca0374..50f4b2a9 100644 --- a/stixcore/processing/TMtoLL01.py +++ b/stixcore/processing/TMtoLL01.py @@ -10,7 +10,7 @@ from datetime import datetime from stixcore.config.config import CONFIG -from stixcore.io.fits.processors import FitsLL01Processor +from stixcore.io.product_processors.fits.processors import FitsLL01Processor from stixcore.io.soc.manager import SOCPacketFile from stixcore.products.levelb.binary import LevelB from stixcore.products.product import Product diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index e91bfff1..09b19181 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -7,38 +7,36 @@ from datetime import date, timedelta from concurrent.futures import ProcessPoolExecutor -from stixpy.net.client import STIXClient - from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager -from stixcore.io.fits.processors import ( - FitsANCProcessor, +from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.io.product_processors.fits.processors import ( # FitsANCProcessor,; FitsL3Processor, FitsL2Processor, - FitsL3Processor, - PlotProcessor, ) -from stixcore.io.FlareListManager import SCFlareListManager, SDCFlareListManager -from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage +from stixcore.io.product_processors.plots.processors import PlotProcessor from stixcore.io.RidLutManager import RidLutManager from stixcore.processing.AspectANC import AspectANC -from stixcore.processing.FlareListL3 import FlareListL3 -from stixcore.processing.FLtoFL import FLtoFL from stixcore.processing.LL import LL03QL from stixcore.processing.pipeline import PipelineStatus from stixcore.processing.SingleStep import SingleProcessingStepResult from stixcore.products.level1.quicklookL1 import LightCurve -from stixcore.products.level3.flarelist import ( - FlarelistSC, - FlarelistSCLoc, - FlarelistSCLocImg, - FlarelistSDC, - FlarelistSDCLoc, - FlarelistSDCLocImg, -) from stixcore.products.lowlatency.quicklookLL import LightCurveL3 from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger +# from stixpy.net.client import STIXClient +# from stixcore.io.FlareListManager import SCFlareListManager, SDCFlareListManager +# from stixcore.processing.FlareListL3 import FlareListL3 +# from stixcore.processing.FLtoFL import FLtoFL +# from stixcore.products.level3.flarelist import ( +# FlarelistSC, +# FlarelistSCLoc, +# FlarelistSCLocImg, +# FlarelistSDC, +# FlarelistSDCLoc, +# FlarelistSDCLocImg, +# ) + logger = get_logger(__name__) @@ -210,14 +208,15 @@ def run_daily_pipeline(args): Path(CONFIG.get("Paths", "fits_archive")) - fido_url = CONFIG.get("Paths", "fido_search_url", fallback="https://pub099.cs.technik.fhnw.ch/data/fits") - fido_client = STIXClient(source=fido_url) + # fido_url = CONFIG.get("Paths", "fido_search_url", fallback="https://pub099.cs.technik.fhnw.ch/data/fits") + # fido_client = STIXClient(source=fido_url) - flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sc_lut_file")) - SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True) + # flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sc_lut_file")) + # SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True) - flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sdc_lut_file")) - SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) + # TODO reactivate once flarelist processing is finalized + # flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sdc_lut_file")) + # SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) RidLutManager.instance = RidLutManager(Path(CONFIG.get("Publish", "rid_lut_file")), update=False) @@ -249,19 +248,20 @@ def run_daily_pipeline(args): aspect_anc_processor = AspectANC(fits_in_dir, fits_out_dir) - flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) - flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir) - fl_to_fl = FLtoFL( - fits_in_dir, - fits_out_dir, - products_in_out=[ - (FlarelistSDC, FlarelistSDCLoc), - (FlarelistSDCLoc, FlarelistSDCLocImg), - (FlarelistSC, FlarelistSCLoc), - (FlarelistSCLoc, FlarelistSCLocImg), - ], - cadence=timedelta(seconds=1), - ) + # TODO reactivate once flarelist processing is finalized + # flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) + # flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir) + # fl_to_fl = FLtoFL( + # fits_in_dir, + # fits_out_dir, + # products_in_out=[ + # (FlarelistSDC, FlarelistSDCLoc), + # (FlarelistSDCLoc, FlarelistSDCLocImg), + # (FlarelistSC, FlarelistSCLoc), + # (FlarelistSCLoc, FlarelistSCLocImg), + # ], + # cadence=timedelta(seconds=1), + # ) ll03ql = LL03QL( fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, cadence=timedelta(seconds=1) @@ -269,24 +269,26 @@ def run_daily_pipeline(args): plot_writer = PlotProcessor(fits_out_dir) l2_fits_writer = FitsL2Processor(fits_out_dir) - l3_fits_writer = FitsL3Processor(fits_out_dir) - anc_fits_writer = FitsANCProcessor(fits_out_dir) - - # hk_in_files = aspect_anc_processor.get_processing_files(phs) - hk_in_files = [] + # TODO reactivate once flarelist processing is finalized + # l3_fits_writer = FitsL3Processor(fits_out_dir) + # anc_fits_writer = FitsANCProcessor(fits_out_dir) - fl_sdc_months = flarelist_sdc.find_processing_months(phs) - fl_sdc_months = [] - - # fl_sc_months = flarelist_sc.find_processing_months(phs) - fl_sc_months = [] + hk_in_files = aspect_anc_processor.get_processing_files(phs) ll_candidates = ll03ql.get_processing_files(phs) # ll_candidates = [] - # ll_candidates = ll_candidates[0:40] - fl_to_fl_files = fl_to_fl.get_processing_files(phs) - fl_to_fl_files = [] + # TODO reactivate once flarelist processing is finalized + # fl_sdc_months = flarelist_sdc.find_processing_months(phs) + # fl_sdc_months = [] + + # TODO reactivate once flarelist processing is finalized + # fl_sc_months = flarelist_sc.find_processing_months(phs) + # fl_sc_months = [] + + # TODO reactivate once flarelist processing is finalized + # fl_to_fl_files = fl_to_fl.get_processing_files(phs) + # fl_to_fl_files = [] # all processing files should be terminated before the next step as the different # processing steeps might create new candidates @@ -303,42 +305,42 @@ def run_daily_pipeline(args): config=CONFIG, ) ) - - jobs.append( - executor.submit( - flarelist_sdc.process_fits_files, - fl_sdc_months, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l3_fits_writer, - config=CONFIG, - ) - ) - - jobs.append( - executor.submit( - flarelist_sc.process_fits_files, - fl_sc_months, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - processor=l3_fits_writer, - config=CONFIG, - ) - ) - - # TODO a owen processing step for each flarelist file? - # for fl_to_fl_file in fl_to_fl_files: - jobs.append( - executor.submit( - fl_to_fl.process_fits_files, - fl_to_fl_files, - soopmanager=SOOPManager.instance, - spice_kernel_path=Spice.instance.meta_kernel_path, - fl_processor=anc_fits_writer, - img_processor=l3_fits_writer, - config=CONFIG, - ) - ) + # TODO reactivate once flarelist processing is finalized + # jobs.append( + # executor.submit( + # flarelist_sdc.process_fits_files, + # fl_sdc_months, + # soopmanager=SOOPManager.instance, + # spice_kernel_path=Spice.instance.meta_kernel_path, + # processor=l3_fits_writer, + # config=CONFIG, + # ) + # ) + + # jobs.append( + # executor.submit( + # flarelist_sc.process_fits_files, + # fl_sc_months, + # soopmanager=SOOPManager.instance, + # spice_kernel_path=Spice.instance.meta_kernel_path, + # processor=l3_fits_writer, + # config=CONFIG, + # ) + # ) + + # # TODO a owen processing step for each flarelist file? + # # for fl_to_fl_file in fl_to_fl_files: + # jobs.append( + # executor.submit( + # fl_to_fl.process_fits_files, + # fl_to_fl_files, + # soopmanager=SOOPManager.instance, + # spice_kernel_path=Spice.instance.meta_kernel_path, + # fl_processor=anc_fits_writer, + # img_processor=l3_fits_writer, + # config=CONFIG, + # ) + # ) jobs.append( executor.submit( diff --git a/stixcore/processing/publish.py b/stixcore/processing/publish.py index f381af1e..7c3ffaef 100644 --- a/stixcore/processing/publish.py +++ b/stixcore/processing/publish.py @@ -25,7 +25,6 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager from stixcore.io.RidLutManager import RidLutManager -from stixcore.processing.pipeline_status import pipeline_status from stixcore.products.product import Product from stixcore.time.datetime import SCETime from stixcore.util.logging import get_logger @@ -297,12 +296,6 @@ def send_mail_report(files): """ if CONFIG.getboolean("Publish", "report_mail_send", fallback=False): try: - try: - with Capturing() as open_tm_files: - pipeline_status(["--next"]) - except Exception as se: - open_tm_files = str(se) - sender = CONFIG.get("Pipeline", "error_mail_sender", fallback="") receivers = CONFIG.get("Publish", "report_mail_receivers").split(",") host = CONFIG.get("Pipeline", "error_mail_smpt_host", fallback="localhost") @@ -317,10 +310,6 @@ def send_mail_report(files): error = "" if len(files[PublishResult.ERROR]) <= 0 else "ERROR-" message = f"""Subject: StixCore TMTC Publishing {error}Report {su} -OPEN TM FILES IN QUEUE -********************** - -{open_tm_files} ERRORS (E) ********** diff --git a/stixcore/processing/tests/test_publish.py b/stixcore/processing/tests/test_publish.py index 53bea6bf..fee75ec3 100644 --- a/stixcore/processing/tests/test_publish.py +++ b/stixcore/processing/tests/test_publish.py @@ -10,7 +10,7 @@ from astropy.io import fits from astropy.table import QTable -from stixcore.io.fits.processors import FitsL1Processor +from stixcore.io.product_processors.fits.processors import FitsL1Processor from stixcore.processing.publish import ( PublishConflicts, PublishHistoryStorage, diff --git a/stixcore/processing/tests/test_sswidl.py b/stixcore/processing/tests/test_sswidl.py index 776e0ca2..64dad4f1 100644 --- a/stixcore/processing/tests/test_sswidl.py +++ b/stixcore/processing/tests/test_sswidl.py @@ -1,6 +1,6 @@ from pathlib import Path -from stixcore.io.fits.processors import FitsL2Processor +from stixcore.io.product_processors.fits.processors import FitsL2Processor from stixcore.processing.sswidl import SSWIDLProcessor, SSWIDLTask diff --git a/stixcore/products/tests/test_housekeeping.py b/stixcore/products/tests/test_housekeeping.py index e1ed5237..c525051a 100644 --- a/stixcore/products/tests/test_housekeeping.py +++ b/stixcore/products/tests/test_housekeeping.py @@ -6,7 +6,7 @@ from stixcore.data.test import test_data from stixcore.idb.manager import IDBManager -from stixcore.io.fits.processors import FitsL0Processor, FitsL1Processor +from stixcore.io.product_processors.fits.processors import FitsL0Processor, FitsL1Processor from stixcore.io.soc.manager import SOCPacketFile from stixcore.processing.LBtoL0 import Level0 from stixcore.products.level0.housekeepingL0 import MaxiReport as MaxiReportL0 From 7f8bb72b2a25cd0ec1318347c1083258e47a79a2 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Wed, 22 Oct 2025 10:38:43 +0200 Subject: [PATCH 10/11] fix for #437 --- stixcore/processing/L0toL1.py | 4 ++-- stixcore/products/level0/scienceL0.py | 2 +- stixcore/products/product.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stixcore/processing/L0toL1.py b/stixcore/processing/L0toL1.py index 51c19e76..3e2d4f28 100644 --- a/stixcore/processing/L0toL1.py +++ b/stixcore/processing/L0toL1.py @@ -91,9 +91,9 @@ def process_type(files, *, processor, soopmanager, spice_kernel_path, config): CONFIG = config for file in files: - l0 = Product(file) - logger.info(f"processing file: {file}") try: + logger.info(f"processing file: {file} for L0 to L1") + l0 = Product(file) tmp = Product._check_registered_widget( level="L1", service_type=l0.service_type, diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py index 837c3323..abf5e2e3 100644 --- a/stixcore/products/level0/scienceL0.py +++ b/stixcore/products/level0/scienceL0.py @@ -968,7 +968,7 @@ def from_levelb(cls, levelb, parent=""): data.add_basic(name="cha_diode1", nix="NIX00091", packets=packets, dtype=np.uint16) data.add_basic(name="chb_diode0", nix="NIX00092", packets=packets, dtype=np.uint16) data.add_basic(name="chb_diode1", nix="NIX00093", packets=packets, dtype=np.uint16) - data["control_index"] = np.hstack([np.full(ns, i) for i, ns in enumerate(samples)]) + data["control_index"] = np.hstack([np.full(ns, i) for i, ns in enumerate(samples)]).astype(np.uint16) except ValueError as e: logger.warning(e) raise e diff --git a/stixcore/products/product.py b/stixcore/products/product.py index ae9ba16f..f902d909 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -71,7 +71,7 @@ def read_qtable(file, hdu, hdul=None): `astropy.table.QTable` The corrected QTable with correct data types """ - qtable = QTable.read(file, hdu, astropy_native=True) + qtable = QTable.read(file, hdu) if hdul is None: hdul = fits.open(file) From 56f1c030d6cf8153341b63d1961103bee5050cfd Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Wed, 22 Oct 2025 12:17:58 +0200 Subject: [PATCH 11/11] Update stixpy dependency to use GitHub source --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7918f968..5f5704ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "sunpy~=6.0.0", "watchdog~=6.0", "stixdcpy==3.0", - "stixpy>=0.1.3" + "stixpy@git+https://github.com/TCDSolar/stixpy.git@main" ] dynamic = ["version"] keywords = ["solar physics", "solar", "sun", "solarorbiter", "stix"]