import hashlib import awkward as ak import numpy as np import pytest import torch import uproot from conftest import GAMGAM_FIXTURES, TTH_FIXTURE, fixture_config, random_events from gnn4colliders import data from gnn4colliders.data import ( apply_derived_branches, build_object_features, load_events, parse_derived_branches, prepare, save_events, ) from gnn4colliders.graphs import collate_graph_batch from gnn4colliders.loader import EventLoader BTAG_THRESHOLD = 0.8244273 def test_object_features_are_grouped_by_type_scaled_and_typed(): arrays = { "jet_pt": ak.Array([[30.0, 20.0], []]), "jet_eta": ak.Array([[0.5, -1.0], []]), "met": np.array([10.0, 40.0]), } features, counts = build_object_features( arrays, 2, [["jet_pt", "met"], ["jet_eta", 0], "CALC_E", "NODE_TYPE"], ["vector", "single"], [0.1, 1, 0.1, 1], ) assert counts.tolist() == [3, 1] torch.testing.assert_close(features[:, 0], torch.tensor([3.0, 2.0, 1.0, 4.0])) energy = torch.tensor([30 * np.cosh(0.5), 20 * np.cosh(-1.0), 10, 40]) * 0.1 torch.testing.assert_close(features[:, 2], energy.float()) assert features[:, 3].tolist() == [0, 0, 1, 1] def test_derived_branches_split_collections_and_threshold_scores(): arrays = ak.Array( { "lep_pt": [[1.0, 2.0, 3.0]], "lep_type": [[11, 13, 11]], "jet_score": [[0.9, 0.1]], } ) derived = parse_derived_branches( { "ele_pt": {"source": "lep_pt", "where": {"lep_type": 11}}, "jet_btag": {"source": "jet_score", "above": 0.5}, } ) result = apply_derived_branches(arrays, derived) assert result["ele_pt"].tolist() == [[1.0, 3.0]] assert result["jet_btag"].tolist() == [[1.0, 0.0]] with pytest.raises(ValueError, match="depends on a derived"): parse_derived_branches( {"a": {"source": "x"}, "b": {"source": "a"}}, ) def test_gamgam_preparation_follows_the_open_data_schema(gamgam): events = load_events(gamgam.data.cache) assert len(events) == 5 * 64 assert events.labels.tolist() == [label for label in range(5) for _ in range(64)] assert torch.equal(events.weight, torch.ones(len(events))) offset = 0 for path in GAMGAM_FIXTURES: raw = uproot.open(path)["mini"].arrays(library="ak") folds = events.fold[offset : offset + 64].tolist() assert folds == (np.asarray(raw.eventNumber, dtype=np.int64) % 10).tolist() for local in (0, 17, 63): event = raw[local] first = int(events.object_offsets[offset + local]) nodes = events.object_features[ first : int(events.object_offsets[offset + local + 1]) ] electron = np.asarray(event.lep_type) == 11 muon = np.asarray(event.lep_type) == 13 pt = np.concatenate( [ np.asarray(event.jet_pt), np.asarray(event.lep_pt)[electron], np.asarray(event.lep_pt)[muon], np.asarray(event.photon_pt), [event.met_et], ] ) sizes = [len(event.jet_pt), electron.sum(), muon.sum()] sizes += [len(event.photon_pt), 1] # MeV -> units of 10 GeV, as in pretraining. np.testing.assert_allclose(nodes[:, 0], pt * 1e-4, rtol=1e-6) assert nodes[:, 6].tolist() == np.repeat(np.arange(5), sizes).tolist() jets = len(event.jet_pt) btag = np.asarray(event.jet_MV2c10) > BTAG_THRESHOLD assert nodes[:jets, 4].tolist() == btag.astype(float).tolist() charge = np.concatenate( [ np.asarray(event.lep_charge)[electron], np.asarray(event.lep_charge)[muon], ] ) leptons = nodes[jets : jets + len(charge), 5] assert leptons.tolist() == charge.astype(float).tolist() assert nodes[-1, 1] == 0.0 # MET has no pseudorapidity offset += 64 def test_parallel_and_chunked_preparation_match_serial(gamgam, tmp_path, monkeypatch): serial = load_events(gamgam.data.cache) monkeypatch.setattr(data, "CHUNK_EVENTS", 24) gamgam.data.cache = str(tmp_path / "parallel.pt") parallel = load_events(prepare(gamgam.data, workers=2)) for name in ("object_features", "object_offsets", "labels", "fold", "weight"): assert torch.equal(getattr(parallel, name), getattr(serial, name)) assert torch.equal(parallel.entry_index, serial.entry_index) def test_preparation_reads_the_leading_events_of_each_file(gamgam, tmp_path): full = load_events(gamgam.data.cache) gamgam.data.cache, gamgam.data.max_events_per_file = str(tmp_path / "c.pt"), 5 capped = load_events(prepare(gamgam.data)) files = full.file_index.unique() assert len(capped) == 5 * len(files) assert (capped.entry_index < 5).all() assert torch.equal(capped.file_index.unique(), files) def test_delphes_preparation_reads_signed_weights_and_number_folds(tmp_path): config = fixture_config("tth_cp_even_odd", [TTH_FIXTURE], [0], tmp_path) events = load_events(prepare(config.data)) raw = uproot.open(TTH_FIXTURE)["output"].arrays(["Number", "weight"], library="np") assert events.fold.tolist() == (raw["Number"].astype(np.int64) % 10).tolist() np.testing.assert_array_equal(events.weight.numpy(), raw["weight"]) assert (events.weight < 0).any() def test_huggingface_files_are_downloaded_at_the_revision_and_verified( tmp_path, monkeypatch ): root_file = tmp_path / "sample.root" root_file.write_bytes(b"fixture") requests = [] def download(**kwargs): requests.append(kwargs) return str(root_file) monkeypatch.setattr("huggingface_hub.hf_hub_download", download) config = fixture_config("gamgam", GAMGAM_FIXTURES[:1], [0], tmp_path).data config.hf_repo, config.hf_revision = "org/data", "abc123" good = hashlib.sha256(b"fixture").hexdigest() config.files = [{"path": "sample.root", "label": 4, "sha256": good}] assert data.resolve_files(config) == [(root_file, 4)] assert requests[0]["revision"] == "abc123" config.files[0]["sha256"] = hashlib.sha256(b"wrong").hexdigest() with pytest.raises(ValueError, match="SHA-256 mismatch"): data.resolve_files(config) def test_cache_round_trip_and_version_check(tmp_path): events = random_events() loaded = load_events(save_events(events, tmp_path / "e.pt")) assert torch.equal(loaded.object_features, events.object_features) payload = torch.load(tmp_path / "e.pt") payload["cache_schema_version"] = 1 torch.save(payload, tmp_path / "old.pt") with pytest.raises(ValueError, match="cache version"): load_events(tmp_path / "old.pt") def test_shuffled_order_depends_only_on_seed_and_epoch(): events = random_events() loader = EventLoader(events, collate_graph_batch, 8, shuffle=True, seed=3) first = loader.order() assert torch.equal(first, loader.order()) loader.set_epoch(1) assert not torch.equal(first, loader.order()) assert sorted(loader.order().tolist()) == list(range(len(events))) def test_ranks_split_training_with_padding_and_evaluation_without(): events = random_events(41) def orders(shuffle): return [ EventLoader( events, collate_graph_batch, 8, shuffle=shuffle, rank=rank, world_size=2, ).order() for rank in range(2) ] train = orders(True) assert len(train[0]) == len(train[1]) == 21 assert set(torch.cat(train).tolist()) == set(range(41)) evaluation = orders(False) assert sorted(torch.cat(evaluation).tolist()) == list(range(41)) def test_loader_workers_yield_identical_pinned_batches(): events = random_events(seed=4) serial = EventLoader(events, collate_graph_batch, 7, shuffle=True, seed=3) workers = EventLoader( events, collate_graph_batch, 7, shuffle=True, seed=3, num_workers=2, pin_memory=torch.cuda.is_available(), ) pairs = list(zip(serial, workers, strict=True)) assert len(pairs) == len(serial) == 6 for left, right in pairs: assert torch.equal(left.graph.edge_features, right.graph.edge_features) assert torch.equal(left.labels, right.labels) assert torch.equal(left.event_index, right.event_index)