Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/changes/323.datamodel.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added support for ctapipe HDF5 data format v8.0.0, using the updated DL0 trigger-table paths while preserving compatibility with earlier data formats.
18 changes: 17 additions & 1 deletion src/ctlearn/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
from traitlets.config.loader import Config

# TODO: Bump minimum ctapipe version to v0.33.0. Clean up imports!
try:
from ctapipe.io.hdf5dataformat import DL0_TEL_TRIGGER_TABLE
except ImportError:
DL0_TEL_TRIGGER_TABLE = "/dl0/event/telescope/trigger"
try:
from ctapipe.io.metadata import ProductMetadata as _ProductMetadata
except ImportError:
Expand Down Expand Up @@ -212,7 +216,19 @@ def dl1_obs_mock_file(dl1_tmp_path, dl1_gamma_file):
output = dl1_tmp_path / "dl1_obs_mock_file.dl1.h5"
shutil.copy(dl1_gamma_file, output)
# Read the trigger table to get the unique telescope IDs and their pointing information
trigger_table = read_table(dl1_gamma_file, path=DL1_TEL_TRIGGER_TABLE)
# TODO: Bump minimum ctapipe version to v0.33.0. Clean up here!
with tables.open_file(output, mode="r") as h5_file:
file_attrs = h5_file.root._v_attrs
if "data.type" in file_attrs:
data_format_version = file_attrs["model.version"]
elif "CTA PROCESS TYPE" in file_attrs:
data_format_version = file_attrs["CTA PRODUCT DATA MODEL VERSION"]
else:
raise IOError("Unsupported or missing reference metadata")
if int(data_format_version.split(".")[0].replace("v", "")) >= 8:
trigger_table = read_table(dl1_gamma_file, path=DL0_TEL_TRIGGER_TABLE)
else:
trigger_table = read_table(dl1_gamma_file, path=DL1_TEL_TRIGGER_TABLE)
tel_ids = np.unique(trigger_table["tel_id"])
time = trigger_table["time"][[0, -1]]
# Mimic an observation of 20 mins duration by adjusting the time range for the tel pointing tables
Expand Down
24 changes: 20 additions & 4 deletions src/ctlearn/tools/predict_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@
)

# TODO: Bump minimum ctapipe version to v0.33.0. Clean up imports!
try:
from ctapipe.io.hdf5dataformat import (
DL0_SUBARRAY_TRIGGER_TABLE,
DL0_TEL_TRIGGER_TABLE,
)
except ImportError:
DL0_SUBARRAY_TRIGGER_TABLE = "/dl0/event/subarray/trigger"
DL0_TEL_TRIGGER_TABLE = "/dl0/event/telescope/trigger"
try:
from ctao_datamodel.models.dataproducts import Software
from ctapipe.io.metadata import (
Expand Down Expand Up @@ -472,6 +480,15 @@ def setup(self):
input_url_signal=[self.input_url],
parent=self,
)
if (
int(self.dl1dh_reader.data_format_version.split(".")[0].replace("v", ""))
>= 8
):
self.subarray_trigger_table_path = DL0_SUBARRAY_TRIGGER_TABLE
self.tel_trigger_table_path = DL0_TEL_TRIGGER_TABLE
else:
self.subarray_trigger_table_path = DL1_SUBARRAY_TRIGGER_TABLE
self.tel_trigger_table_path = DL1_TEL_TRIGGER_TABLE
self.log.info("Number of events loaded: %s", self.dl1dh_reader.get_n_events())
# Check if the number of events is enough to form a batch
if self.dl1dh_reader.get_n_events() < self.batch_size:
Expand Down Expand Up @@ -596,10 +613,9 @@ def _ensure_subarray_consistency(self):
)
selected_subarray.to_hdf(self.output_path)
self.log.info("SubarrayDescription was stored in '%s'", self.output_path)

tel_trigger_table = read_table(
self.output_path,
DL1_TEL_TRIGGER_TABLE,
self.tel_trigger_table_path,
)
mask = np.isin(tel_trigger_table["tel_id"], self.dl1dh_reader.tel_ids)
tel_trigger_table = tel_trigger_table[mask]
Expand All @@ -608,7 +624,7 @@ def _ensure_subarray_consistency(self):
write_table(
tel_trigger_table,
self.output_path,
DL1_TEL_TRIGGER_TABLE,
self.tel_trigger_table_path,
overwrite=True,
)

Expand Down Expand Up @@ -639,7 +655,7 @@ def _ensure_subarray_consistency(self):
write_table(
subarray_trigger_table,
self.output_path,
DL1_SUBARRAY_TRIGGER_TABLE,
self.subarray_trigger_table_path,
overwrite=True,
)
# Update the simulation shower table to keep only events present in the subarray trigger table
Expand Down
3 changes: 1 addition & 2 deletions src/ctlearn/tools/predict_mono_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from ctapipe.io.hdf5dataformat import (
DL1_TEL_GROUP,
DL1_TEL_POINTING_GROUP,
DL1_TEL_TRIGGER_TABLE,
DL2_EVENT_STATISTICS_GROUP,
DL2_SUBARRAY_ENERGY_GROUP,
DL2_SUBARRAY_GEOMETRY_GROUP,
Expand Down Expand Up @@ -126,7 +125,7 @@ def start(self):
example_identifiers.keep_columns(TEL_EVENT_KEYS)
all_identifiers = read_table(
self.output_path,
DL1_TEL_TRIGGER_TABLE,
self.tel_trigger_table_path,
)
all_identifiers.keep_columns(TEL_EVENT_KEYS + ["time"])
nonexample_identifiers = setdiff(
Expand Down
3 changes: 1 addition & 2 deletions src/ctlearn/tools/predict_stereo_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from ctapipe.io.hdf5dataformat import (
DL1_SUBARRAY_GROUP,
DL1_SUBARRAY_POINTING_GROUP,
DL1_TEL_TRIGGER_TABLE,
DL2_EVENT_STATISTICS_GROUP,
DL2_SUBARRAY_ENERGY_GROUP,
DL2_SUBARRAY_GEOMETRY_GROUP,
Expand Down Expand Up @@ -100,7 +99,7 @@ def start(self):
example_identifiers.keep_columns(SUBARRAY_EVENT_KEYS)
all_identifiers = read_table(
self.output_path,
DL1_TEL_TRIGGER_TABLE,
self.tel_trigger_table_path,
)
all_identifiers.keep_columns(SUBARRAY_EVENT_KEYS + ["time"])
# Unique example identifiers by events
Expand Down
Loading