diff --git a/docs/changes/323.datamodel.rst b/docs/changes/323.datamodel.rst new file mode 100644 index 00000000..d377f29f --- /dev/null +++ b/docs/changes/323.datamodel.rst @@ -0,0 +1 @@ +Added support for ctapipe HDF5 data format v8.0.0, using the updated DL0 trigger-table paths while preserving compatibility with earlier data formats. diff --git a/src/ctlearn/conftest.py b/src/ctlearn/conftest.py index 809acbe9..0991f5dc 100644 --- a/src/ctlearn/conftest.py +++ b/src/ctlearn/conftest.py @@ -24,6 +24,10 @@ from traitlets.config.loader import Config # TODO: Bump minimum ctapipe version to v0.33.0. Clean up imports! +try: + from ctapipe.io.hdf5dataformat import DL0_TEL_TRIGGER_TABLE +except ImportError: + DL0_TEL_TRIGGER_TABLE = "/dl0/event/telescope/trigger" try: from ctapipe.io.metadata import ProductMetadata as _ProductMetadata except ImportError: @@ -212,7 +216,19 @@ def dl1_obs_mock_file(dl1_tmp_path, dl1_gamma_file): output = dl1_tmp_path / "dl1_obs_mock_file.dl1.h5" shutil.copy(dl1_gamma_file, output) # Read the trigger table to get the unique telescope IDs and their pointing information - trigger_table = read_table(dl1_gamma_file, path=DL1_TEL_TRIGGER_TABLE) + # TODO: Bump minimum ctapipe version to v0.33.0. Clean up here! + with tables.open_file(output, mode="r") as h5_file: + file_attrs = h5_file.root._v_attrs + if "data.type" in file_attrs: + data_format_version = file_attrs["model.version"] + elif "CTA PROCESS TYPE" in file_attrs: + data_format_version = file_attrs["CTA PRODUCT DATA MODEL VERSION"] + else: + raise IOError("Unsupported or missing reference metadata") + if int(data_format_version.split(".")[0].replace("v", "")) >= 8: + trigger_table = read_table(dl1_gamma_file, path=DL0_TEL_TRIGGER_TABLE) + else: + trigger_table = read_table(dl1_gamma_file, path=DL1_TEL_TRIGGER_TABLE) tel_ids = np.unique(trigger_table["tel_id"]) time = trigger_table["time"][[0, -1]] # Mimic an observation of 20 mins duration by adjusting the time range for the tel pointing tables diff --git a/src/ctlearn/tools/predict_model.py b/src/ctlearn/tools/predict_model.py index 7d0a54c4..3ad67553 100644 --- a/src/ctlearn/tools/predict_model.py +++ b/src/ctlearn/tools/predict_model.py @@ -60,6 +60,14 @@ ) # TODO: Bump minimum ctapipe version to v0.33.0. Clean up imports! +try: + from ctapipe.io.hdf5dataformat import ( + DL0_SUBARRAY_TRIGGER_TABLE, + DL0_TEL_TRIGGER_TABLE, + ) +except ImportError: + DL0_SUBARRAY_TRIGGER_TABLE = "/dl0/event/subarray/trigger" + DL0_TEL_TRIGGER_TABLE = "/dl0/event/telescope/trigger" try: from ctao_datamodel.models.dataproducts import Software from ctapipe.io.metadata import ( @@ -472,6 +480,15 @@ def setup(self): input_url_signal=[self.input_url], parent=self, ) + if ( + int(self.dl1dh_reader.data_format_version.split(".")[0].replace("v", "")) + >= 8 + ): + self.subarray_trigger_table_path = DL0_SUBARRAY_TRIGGER_TABLE + self.tel_trigger_table_path = DL0_TEL_TRIGGER_TABLE + else: + self.subarray_trigger_table_path = DL1_SUBARRAY_TRIGGER_TABLE + self.tel_trigger_table_path = DL1_TEL_TRIGGER_TABLE self.log.info("Number of events loaded: %s", self.dl1dh_reader.get_n_events()) # Check if the number of events is enough to form a batch if self.dl1dh_reader.get_n_events() < self.batch_size: @@ -596,10 +613,9 @@ def _ensure_subarray_consistency(self): ) selected_subarray.to_hdf(self.output_path) self.log.info("SubarrayDescription was stored in '%s'", self.output_path) - tel_trigger_table = read_table( self.output_path, - DL1_TEL_TRIGGER_TABLE, + self.tel_trigger_table_path, ) mask = np.isin(tel_trigger_table["tel_id"], self.dl1dh_reader.tel_ids) tel_trigger_table = tel_trigger_table[mask] @@ -608,7 +624,7 @@ def _ensure_subarray_consistency(self): write_table( tel_trigger_table, self.output_path, - DL1_TEL_TRIGGER_TABLE, + self.tel_trigger_table_path, overwrite=True, ) @@ -639,7 +655,7 @@ def _ensure_subarray_consistency(self): write_table( subarray_trigger_table, self.output_path, - DL1_SUBARRAY_TRIGGER_TABLE, + self.subarray_trigger_table_path, overwrite=True, ) # Update the simulation shower table to keep only events present in the subarray trigger table diff --git a/src/ctlearn/tools/predict_mono_model.py b/src/ctlearn/tools/predict_mono_model.py index 3022d73b..dca6e69a 100644 --- a/src/ctlearn/tools/predict_mono_model.py +++ b/src/ctlearn/tools/predict_mono_model.py @@ -23,7 +23,6 @@ from ctapipe.io.hdf5dataformat import ( DL1_TEL_GROUP, DL1_TEL_POINTING_GROUP, - DL1_TEL_TRIGGER_TABLE, DL2_EVENT_STATISTICS_GROUP, DL2_SUBARRAY_ENERGY_GROUP, DL2_SUBARRAY_GEOMETRY_GROUP, @@ -126,7 +125,7 @@ def start(self): example_identifiers.keep_columns(TEL_EVENT_KEYS) all_identifiers = read_table( self.output_path, - DL1_TEL_TRIGGER_TABLE, + self.tel_trigger_table_path, ) all_identifiers.keep_columns(TEL_EVENT_KEYS + ["time"]) nonexample_identifiers = setdiff( diff --git a/src/ctlearn/tools/predict_stereo_model.py b/src/ctlearn/tools/predict_stereo_model.py index 300440c8..645a8ac0 100644 --- a/src/ctlearn/tools/predict_stereo_model.py +++ b/src/ctlearn/tools/predict_stereo_model.py @@ -23,7 +23,6 @@ from ctapipe.io.hdf5dataformat import ( DL1_SUBARRAY_GROUP, DL1_SUBARRAY_POINTING_GROUP, - DL1_TEL_TRIGGER_TABLE, DL2_EVENT_STATISTICS_GROUP, DL2_SUBARRAY_ENERGY_GROUP, DL2_SUBARRAY_GEOMETRY_GROUP, @@ -100,7 +99,7 @@ def start(self): example_identifiers.keep_columns(SUBARRAY_EVENT_KEYS) all_identifiers = read_table( self.output_path, - DL1_TEL_TRIGGER_TABLE, + self.tel_trigger_table_path, ) all_identifiers.keep_columns(SUBARRAY_EVENT_KEYS + ["time"]) # Unique example identifiers by events