From b33675dc03933857604cc876b53db0700acd4acf Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:17:47 +0100 Subject: [PATCH 01/17] Fix output path of plot_embedding_edge_performance --- .../evaluation/embedding/plot_embedding_edge_performance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py b/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py index f8cc81b4..3ce244a5 100644 --- a/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py +++ b/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py @@ -1,6 +1,7 @@ """A script that plots the edge performance (edge purity and efficiency) as a function of the maximal squared distance. """ + from argparse import ArgumentParser import os.path as op @@ -29,6 +30,7 @@ if __name__ == "__main__": else op.join( cdirs.performance_directory, load_config(pipeline_config_path)["common"]["experiment_name"], + "embedding", ) ) -- GitLab From 7cc0b13d76f47e34e3f1a87beccee5acfd0e2a98 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:18:35 +0100 Subject: [PATCH 02/17] Save figures in svg and pdf by default --- etx4velo/pipeline/utils/plotutils/plotools.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/etx4velo/pipeline/utils/plotutils/plotools.py b/etx4velo/pipeline/utils/plotutils/plotools.py index 1c9e327e..ff5147bb 100644 --- a/etx4velo/pipeline/utils/plotutils/plotools.py +++ b/etx4velo/pipeline/utils/plotutils/plotools.py @@ -1,5 +1,6 @@ """Define some global utilies for plots. """ + from __future__ import annotations import typing import os @@ -15,22 +16,19 @@ from matplotlib.text import Text @typing.overload def get_figs_axes_on_grid( nrows: int, ncols: int, same_fig: typing.Literal[True] = True, **kwargs -) -> typing.Tuple[Figure, npt.NDArray]: - ... +) -> typing.Tuple[Figure, npt.NDArray]: ... @typing.overload def get_figs_axes_on_grid( nrows: int, ncols: int, same_fig: typing.Literal[False], **kwargs -) -> typing.Tuple[npt.NDArray, npt.NDArray]: - ... +) -> typing.Tuple[npt.NDArray, npt.NDArray]: ... @typing.overload def get_figs_axes_on_grid( nrows: int, ncols: int, same_fig: bool, **kwargs -) -> typing.Tuple[Figure | npt.NDArray, npt.NDArray]: - ... +) -> typing.Tuple[Figure | npt.NDArray, npt.NDArray]: ... def get_figs_axes_on_grid( @@ -183,7 +181,7 @@ def hide_repetitive_labels_in_grid(axes: typing.Collection[typing.Collection[Axe def save_fig( fig: Figure, path: str, - exts: typing.List[str] = [".pdf", ".png"], + exts: typing.List[str] = [".pdf", ".svg"], bbox_inches: str | None = "tight", **kwargs, ): @@ -198,10 +196,13 @@ def save_fig( fig.tight_layout() path_without_ext, ext = os.path.splitext(path) - if ext not in [".png", ".pdf"]: + if ext not in exts: path_without_ext = path for ext in exts: + assert ext.startswith(".") overall_path = path_without_ext + ext - fig.savefig(path_without_ext + ext, bbox_inches=bbox_inches, **kwargs) + fig.savefig( + path_without_ext + ext, format=ext[1:], bbox_inches=bbox_inches, **kwargs + ) print("Figure was saved in", overall_path) -- GitLab From 44292cdd063319ee8321682d53bc64de9ac4f074 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:19:03 +0100 Subject: [PATCH 03/17] Add matching condition to select best track to particle match --- montetracko | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/montetracko b/montetracko index 37f05c8c..742fa387 160000 --- a/montetracko +++ b/montetracko @@ -1 +1 @@ -Subproject commit 37f05c8c1162b204386b7fdf6fbc1ced8a802aa3 +Subproject commit 742fa3876253e3cae9f62432cd677fedc6c56340 -- GitLab From 181f11423d59c5e50e1b8f3a7870ff0025eb8d78 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:19:36 +0100 Subject: [PATCH 04/17] Format plot_embedding_best_tracking_performance.py --- .../embedding/plot_embedding_best_tracking_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py index 4851a418..53db8591 100644 --- a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py +++ b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py @@ -1,5 +1,6 @@ """A script that plots the metric saved during the learning. """ + from argparse import ArgumentParser import montetracko.lhcb as mtb @@ -42,7 +43,6 @@ if __name__ == "__main__": model=embedding_model, path_or_config=pipeline_config_path, partition=test_dataset_name, - # list_squared_distance_max=[0.015, 0.020], list_squared_distance_max=[ 0.005, 0.0075, -- GitLab From c285419a494c836ea613f94c9a9267540386768a Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:19:47 +0100 Subject: [PATCH 05/17] Put legend inside by default --- .../embedding/plot_embedding_best_tracking_performance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py index 53db8591..4c4889ab 100644 --- a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py +++ b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py @@ -74,4 +74,5 @@ if __name__ == "__main__": output_path=output_wpath, same_fig=False, lhcb=True, + legend_inside=True, ) -- GitLab From ddc65445fd77920ec760a2f564b88bc0e58cb821 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:20:44 +0100 Subject: [PATCH 06/17] Set color of test datasets to black in plot_embedding_performance_given_squared_distance_max_k_max --- etx4velo/pipeline/Embedding/embedding_plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etx4velo/pipeline/Embedding/embedding_plots.py b/etx4velo/pipeline/Embedding/embedding_plots.py index c50128cc..76b865e6 100644 --- a/etx4velo/pipeline/Embedding/embedding_plots.py +++ b/etx4velo/pipeline/Embedding/embedding_plots.py @@ -130,7 +130,7 @@ def plot_embedding_performance_given_squared_distance_max_k_max( if show_err else None ), - color=partition_to_color.get(partition_name), + color=partition_to_color.get(partition_name, "k"), label=partition_to_label.get(partition_name, partition_label), marker=".", ) -- GitLab From e013b0fa37086fd30c86c7745b9a427043eb9ff0 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:21:01 +0100 Subject: [PATCH 07/17] Fix typing and formatting of embedding_plots.py --- etx4velo/pipeline/Embedding/embedding_plots.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etx4velo/pipeline/Embedding/embedding_plots.py b/etx4velo/pipeline/Embedding/embedding_plots.py index 76b865e6..690ef7f1 100644 --- a/etx4velo/pipeline/Embedding/embedding_plots.py +++ b/etx4velo/pipeline/Embedding/embedding_plots.py @@ -1,5 +1,6 @@ """A module that handles the validation plots for the embedding phase specifically. """ + from __future__ import annotations import typing import os.path as op @@ -180,7 +181,9 @@ def plot_best_performances_squared_distance_max( ) -> typing.Tuple[ Figure | npt.NDArray, typing.List[Axes], - typing.Dict[float, typing.Dict[typing.Tuple[str | None, str], float]], + typing.Dict[ + float, typing.Dict[typing.Tuple[str | None, str], typing.Dict[str, float]] + ], ]: """Plot best performance for perfect inference as a function of the squared maximal distance. -- GitLab From b1d6c52df209585deb7a0a2bad9668d9cb436703 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:21:28 +0100 Subject: [PATCH 08/17] Use but comment out BestMatchingParticles matching condition --- etx4velo/pipeline/Evaluation/matching.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etx4velo/pipeline/Evaluation/matching.py b/etx4velo/pipeline/Evaluation/matching.py index d3eff864..e94a3ba7 100644 --- a/etx4velo/pipeline/Evaluation/matching.py +++ b/etx4velo/pipeline/Evaluation/matching.py @@ -28,7 +28,10 @@ def perform_matching( df_events_hits_particles=df_hits_particles, df_events_particles=df_particles, df_events_tracks_hits=df_tracks, - matching_condition=mt.matchcond.MinMatchingFraction(matching_fraction), + matching_condition=( + mt.matchcond.MinMatchingFraction(matching_fraction) + # & mt.matchcond.basics.BestMatchingParticles() + ), track_condition=mt.matchcond.MinLengthTrack(min_track_length), ) -- GitLab From 66a4a8eb5542257984d1140b37e53c4f9c6b7631 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:21:54 +0100 Subject: [PATCH 09/17] Define legend_inside parameter in exploration --- .../pipeline/utils/modelutils/exploration.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/etx4velo/pipeline/utils/modelutils/exploration.py b/etx4velo/pipeline/utils/modelutils/exploration.py index 70caaf84..880b4282 100644 --- a/etx4velo/pipeline/utils/modelutils/exploration.py +++ b/etx4velo/pipeline/utils/modelutils/exploration.py @@ -1,6 +1,7 @@ """A module that defines :py:class:`ParamExplorer`, a class that allows to vary a parameter and check the efficiency that is obtained for this choice. """ + from __future__ import annotations import typing import abc @@ -203,7 +204,7 @@ class ParamExplorer(abc.ABC): if track_metric_names is None: track_metric_names = [] - dict_performances: typing.Dict[typing.Tuple[str | None, str], typing.Dict[str, float]] = { # type: ignore + dict_performances: typing.Dict[typing.Tuple[str | None, str]] = { # type: ignore (category.name, metric_name): trackEvaluator.compute_metric( metric_name=metric_name, category=category, @@ -258,6 +259,7 @@ class ParamExplorer(abc.ABC): category_name_to_color: dict | None = None, step: str | None = None, with_err: bool = True, + legend_inside: bool | None = None, **kwargs, ) -> typing.Tuple[ Figure | npt.NDArray, @@ -296,6 +298,8 @@ class ParamExplorer(abc.ABC): 3-tuple of the Matplotlib Figures and Axes, and the dictionary of metric values for every tuple ``(value, category.name, metric_name)`` """ + if legend_inside is None: + legend_inside = same_fig if category_name_to_color is None: category_name_to_color = {} if step is None: @@ -339,7 +343,7 @@ class ParamExplorer(abc.ABC): 1, len(metric_names), same_fig=False, - figsize=(12, 6), + figsize=(8, 6) if legend_inside else (12, 6), ) else: figs_with_cat = None @@ -448,11 +452,19 @@ class ParamExplorer(abc.ABC): plotools.pad_on_top(ax=ax) plotools.add_text(ax=ax, ha="left", va="top") + if legend_inside: + legend_kwargs = dict() + else: + legend_kwargs = dict(loc="center left", bbox_to_anchor=(1, 0.5)) + if same_fig: - axes[0].legend() + if legend_inside: + axes[0].legend(**legend_kwargs) + else: + axes[-1].legend(**legend_kwargs) else: for metric_idx, _ in enumerate(metric_names): - axes[metric_idx].legend(loc="center left", bbox_to_anchor=(1, 0.5)) + axes[metric_idx].legend(**legend_kwargs) if same_fig: if output_path is None: -- GitLab From e6f44e9bba73c99cd90bea0f7fbf19104249cc33 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:22:42 +0100 Subject: [PATCH 10/17] Clarify what fonts are used for plots --- etx4velo/pipeline/utils/plotutils/plotconfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etx4velo/pipeline/utils/plotutils/plotconfig.py b/etx4velo/pipeline/utils/plotutils/plotconfig.py index c2f5b87f..4c637837 100644 --- a/etx4velo/pipeline/utils/plotutils/plotconfig.py +++ b/etx4velo/pipeline/utils/plotutils/plotconfig.py @@ -85,7 +85,8 @@ def configure_matplotlib(): **{ # Font "font.family": "serif", - # "font.serif": "Computer Modern Roman", + "font.serif": "Computer Modern Roman", + # "font.serif": "Times", # Latex "text.latex.preamble": r"\usepackage{amsmath}", "text.usetex": True, # Put to False to disable Latex -- GitLab From e9eeee839fa3d1fc3fff4cf26fc1e548019931e5 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Mon, 5 Feb 2024 19:24:16 +0100 Subject: [PATCH 11/17] Change metric_learning to embedding everywhere (except analysis notebooks) --- etx4velo/pipeline_configs/example.yaml | 6 +++--- .../focal-loss-nopid-shared-hits-triplets.yaml | 8 ++++---- .../pipeline_configs/focal-loss-nopid-shared-hits.yaml | 4 ++-- .../focal-loss-nopid-triplets-embedding-1.yaml | 8 ++++---- ...d-triplets-embedding-3-withspillover-d2max0.02.yaml | 6 +++--- ...d-triplets-embedding-3-withspillover-embedding.yaml | 8 ++++---- ...s-embedding-3-withspillover-new-same-d2max0.02.yaml | 6 +++--- ...s-nopid-triplets-embedding-3-withspillover-new.yaml | 10 +++++----- ...-loss-nopid-triplets-embedding-3-withspillover.yaml | 8 ++++---- .../focal-loss-nopid-triplets-embedding-3.yaml | 8 ++++---- ...opid-triplets-embedding-6-withspillover-nonorm.yaml | 6 +++--- ...d-triplets-embedding-6-withspillover-unsquared.yaml | 6 +++--- ...-loss-nopid-triplets-embedding-6-withspillover.yaml | 6 +++--- .../focal-loss-nopid-triplets-embedding.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-1.yaml | 6 +++--- etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml | 4 ++-- etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml | 8 ++++---- .../scifi-3-xuvx-long-checkpointing.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml | 8 ++++---- .../scifi-3-xuvx-reconstructible-deep-no-triplets.yaml | 8 ++++---- .../scifi-3-xuvx-reconstructible-deep.yaml | 8 ++++---- .../pipeline_configs/scifi-3-xuvx-reconstructible.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-3-xuvx.yaml | 8 ++++---- etx4velo/pipeline_configs/scifi-5.yaml | 6 +++--- .../pipeline_configs/velo-6-query-long-strange.yaml | 10 +++++----- etx4velo/pipeline_configs/velo-6-query-long.yaml | 10 +++++----- .../velo-6-triplets-improved-strange-fixed.yaml | 10 +++++----- .../velo-6-triplets-improved-strange-long-fixed.yaml | 10 +++++----- .../velo-6-triplets-improved-strange-long.yaml | 10 +++++----- .../velo-6-triplets-improved-strange.yaml | 10 +++++----- .../pipeline_configs/velo-6-triplets-improved.yaml | 10 +++++----- .../velo-scatter-sum-long-epoch-128.yaml | 6 +++--- .../pipeline_configs/velo-scatter-sum-long-epoch.yaml | 6 +++--- etx4velo/pipeline_configs/velo-scatter-sum-max.yaml | 6 +++--- etx4velo/pipeline_configs/velo-scatter-sum.yaml | 6 +++--- .../velo-scatter-sum_max-checkpointing.yaml | 6 +++--- etx4velo/pipeline_configs/velo_diff.yaml | 6 +++--- etx4velo/pipeline_configs/velo_edge_based.yaml | 6 +++--- .../pipeline_configs/velo_edge_based_long_epoch.yaml | 6 +++--- .../velo_edge_based_very_long_epoch.yaml | 6 +++--- .../velo_edge_based_very_long_epoch_128.yaml | 6 +++--- .../velo_edge_based_very_long_epoch_64.yaml | 6 +++--- etx4velo/pipeline_configs/velo_gcn.yaml | 6 +++--- etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml | 6 +++--- .../pipeline_configs/velo_nonrecursive_edge_based.yaml | 6 +++--- etx4velo/snakefiles/embedding.snake | 10 +++++----- etx4velo/snakefiles/gnn.snake | 4 ++-- etx4velo/snakefiles/plotfactory.snake | 2 +- etx4velo/snakefiles/track_building.snake | 4 ++-- etx4velo/snakefiles/trackfactory.snake | 4 ++-- 53 files changed, 186 insertions(+), 186 deletions(-) diff --git a/etx4velo/pipeline_configs/example.yaml b/etx4velo/pipeline_configs/example.yaml index 46ae38b8..29b52c27 100644 --- a/etx4velo/pipeline_configs/example.yaml +++ b/etx4velo/pipeline_configs/example.yaml @@ -62,7 +62,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true # Whether to load an event of the training set at a time # Whether to split the training sample by `trainset_split` partitions # and define an "epoch" as going through one partition. @@ -122,7 +122,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" lazy: true # Whether to load an event of the training set at a time # Whether to split the training sample by `trainset_split` partitions @@ -188,5 +188,5 @@ track_building: # Minimal triplet score required to filter out the fake triplets before building # the tracks triplet_score_cut: 0.38 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml index 9d973106..98229b87 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" # Model parameters feature_indices: 4 @@ -65,7 +65,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_cut: 0.5 bidir: False @@ -105,9 +105,9 @@ track_building: edge_score_cut: 0.6 triplet_score_cut: 0.2 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml index a0405d2c..68d3aebf 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml @@ -33,7 +33,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" # Model parameters feature_indices: 4 @@ -64,7 +64,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_cut: 0.5 bidir: False diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml index 84b98ddb..90f31bb2 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml @@ -35,7 +35,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 model: triplet_interaction @@ -75,7 +75,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_cut: 0.5 bidir: False @@ -115,9 +115,9 @@ track_building: edge_score_cut: 0.6 triplet_score_cut: 0.2 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml index f9960ef7..9e0c78ad 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml @@ -36,7 +36,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -77,7 +77,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.36 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml index cdf45a97..899357fc 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml @@ -41,7 +41,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -81,7 +81,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -119,9 +119,9 @@ track_building: edge_score_cut: 0.5 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml index a7b4f348..150de14b 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -83,7 +83,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -120,5 +120,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.38 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml index 138a6aef..4e37b173 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -113,14 +113,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml index 699c3fff..1a5c761b 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml @@ -39,7 +39,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -78,7 +78,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -116,9 +116,9 @@ track_building: edge_score_cut: 0.5 triplet_score_cut: 0.25 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml index 4307389e..dc47c905 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml @@ -39,7 +39,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 model: triplet_interaction @@ -80,7 +80,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -126,9 +126,9 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.40 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml index 66cdfc49..e4809f74 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml @@ -36,7 +36,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -78,7 +78,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml index 2ae790aa..0b6400b5 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml @@ -41,7 +41,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -84,7 +84,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -121,5 +121,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml index 7528a02e..9efec383 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml @@ -41,7 +41,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -83,7 +83,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -120,5 +120,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml index 4c715a19..cafa3b5a 100644 --- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml +++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 model: triplet_interaction @@ -74,7 +74,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -116,9 +116,9 @@ track_building: edge_score_cut: 0.6 triplet_score_cut: 0.4 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/scifi-1.yaml b/etx4velo/pipeline_configs/scifi-1.yaml index ead44301..354d88bb 100644 --- a/etx4velo/pipeline_configs/scifi-1.yaml +++ b/etx4velo/pipeline_configs/scifi-1.yaml @@ -38,7 +38,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -80,7 +80,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -117,5 +117,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml index e12baa9b..ffb4b90f 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -80,7 +80,7 @@ metric_learning: metric_learning_xu_vx: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "metric_learning_xu_vx_processed" lazy: true trainset_split: 80 diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml index 72ae667f..b75246e7 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -79,7 +79,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -121,11 +121,11 @@ track_building: edge_score_cut: 0.86 triplet_score_cut: 0.1 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.86 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml index 64a5aa70..f1023e89 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -80,7 +80,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -127,11 +127,11 @@ track_building: elbow_left: 0.75 elbow_right: 0.75 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.86 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml index fedc7cd6..57288733 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -79,7 +79,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -121,11 +121,11 @@ track_building: edge_score_cut: 0.84 triplet_score_cut: 0.1 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.87 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml index 4dbc4fb8..c3b0e1d3 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -79,7 +79,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -121,11 +121,11 @@ track_building: edge_score_cut: 0.85 triplet_score_cut: 0.3 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.85 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml index c53afac4..7bf72b98 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -79,7 +79,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -121,11 +121,11 @@ track_building: edge_score_cut: 0.86 triplet_score_cut: 0.3 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.87 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml index e0ee2cbc..a9ea5d47 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml @@ -39,7 +39,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -81,7 +81,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.8 triplet_score_cut: 0.2 @@ -118,11 +118,11 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.85 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml index 7b0f7e11..21819e5e 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml @@ -39,7 +39,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -81,7 +81,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.8 triplet_score_cut: 0.2 @@ -118,11 +118,11 @@ track_building: edge_score_cut: 0.85 triplet_score_cut: 0.3 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.86 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml index 7fa596de..7db39276 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml @@ -39,7 +39,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -81,7 +81,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.8 triplet_score_cut: 0.2 @@ -118,11 +118,11 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.85 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml index 3fbf8eb2..c735e6d6 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml @@ -37,7 +37,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -79,7 +79,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.65 triplet_score_cut: 0.4 @@ -121,11 +121,11 @@ track_building: edge_score_cut: 0.86 triplet_score_cut: 0.3 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.86 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx.yaml index c6ba3744..b1ee1c08 100644 --- a/etx4velo/pipeline_configs/scifi-3-xuvx.yaml +++ b/etx4velo/pipeline_configs/scifi-3-xuvx.yaml @@ -38,7 +38,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -80,7 +80,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -117,7 +117,7 @@ track_building: edge_score_cut: 0.9 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" @@ -125,5 +125,5 @@ track_building_from_edges: edge_score_cut: 0.9 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" diff --git a/etx4velo/pipeline_configs/scifi-5.yaml b/etx4velo/pipeline_configs/scifi-5.yaml index fcee066c..c820f71b 100644 --- a/etx4velo/pipeline_configs/scifi-5.yaml +++ b/etx4velo/pipeline_configs/scifi-5.yaml @@ -38,7 +38,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 80 on_step: false @@ -80,7 +80,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -117,5 +117,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml b/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml index 812d37c1..81396f79 100644 --- a/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml +++ b/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -87,7 +87,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -124,14 +124,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-query-long.yaml b/etx4velo/pipeline_configs/velo-6-query-long.yaml index 80770bc4..c6bb9bc1 100644 --- a/etx4velo/pipeline_configs/velo-6-query-long.yaml +++ b/etx4velo/pipeline_configs/velo-6-query-long.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -85,7 +85,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -122,14 +122,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml index 7dc65861..0dfcfd9c 100644 --- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml +++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -87,7 +87,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -124,14 +124,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml index 7a7dd5cd..66e4bb5f 100644 --- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml +++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -86,7 +86,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -123,14 +123,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml index 7a7dd5cd..66e4bb5f 100644 --- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml +++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -86,7 +86,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -123,14 +123,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml index 7dc65861..0dfcfd9c 100644 --- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml +++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 50 on_step: false @@ -87,7 +87,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -124,14 +124,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml index 3645574b..f856fa9d 100644 --- a/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml +++ b/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml @@ -42,7 +42,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -85,7 +85,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -122,14 +122,14 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" track_building_from_edges: edge_score_cut: 0.8 - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_from_edges_processed" track_building_perfect: - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "perfect_track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml index 0d3096ec..6ac01403 100644 --- a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml +++ b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -113,5 +113,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml index 9ae102f2..664768e7 100644 --- a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml +++ b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -113,5 +113,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.32 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml index 682aaefb..93751e2a 100644 --- a/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml +++ b/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -113,5 +113,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-scatter-sum.yaml b/etx4velo/pipeline_configs/velo-scatter-sum.yaml index ae403089..5435669d 100644 --- a/etx4velo/pipeline_configs/velo-scatter-sum.yaml +++ b/etx4velo/pipeline_configs/velo-scatter-sum.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -113,5 +113,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml b/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml index ea7f8368..ca6eac8d 100644 --- a/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml +++ b/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -114,5 +114,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_diff.yaml b/etx4velo/pipeline_configs/velo_diff.yaml index 4f84e5e7..f2539d55 100644 --- a/etx4velo/pipeline_configs/velo_diff.yaml +++ b/etx4velo/pipeline_configs/velo_diff.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_edge_based.yaml b/etx4velo/pipeline_configs/velo_edge_based.yaml index f41cb487..b23c615f 100644 --- a/etx4velo/pipeline_configs/velo_edge_based.yaml +++ b/etx4velo/pipeline_configs/velo_edge_based.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -116,5 +116,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.34 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml b/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml index 4b867abb..efb14bd9 100644 --- a/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml +++ b/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.34 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml index e009981a..764860d8 100644 --- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml +++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -116,5 +116,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml index 42d240ea..6fb58541 100644 --- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml +++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -75,7 +75,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -114,5 +114,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml index cf2b096c..27ac97cb 100644 --- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml +++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_gcn.yaml b/etx4velo/pipeline_configs/velo_gcn.yaml index 4aaca744..ffcd1efa 100644 --- a/etx4velo/pipeline_configs/velo_gcn.yaml +++ b/etx4velo/pipeline_configs/velo_gcn.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml b/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml index e9a6b90d..925896b1 100644 --- a/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml +++ b/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -115,5 +115,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml b/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml index ece9552a..decae5d1 100644 --- a/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml +++ b/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml @@ -34,7 +34,7 @@ processing: metric_learning: # Dataset parameters input_subdirectory: "processed" - output_subdirectory: "metric_learning_processed" + output_subdirectory: "embedding_processed" lazy: true trainset_split: 100 on_step: false @@ -76,7 +76,7 @@ metric_learning: gnn: # Dataset parameters - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "gnn_processed" edge_score_cut: 0.5 triplet_score_cut: 0.2 @@ -116,5 +116,5 @@ track_building: edge_score_cut: 0.4 triplet_score_cut: 0.26 # input_subdirectory: "gnn_processed" - input_subdirectory: "metric_learning_processed" + input_subdirectory: "embedding_processed" output_subdirectory: "track_building_processed" diff --git a/etx4velo/snakefiles/embedding.snake b/etx4velo/snakefiles/embedding.snake index 62c8c3ce..bf299bcb 100644 --- a/etx4velo/snakefiles/embedding.snake +++ b/etx4velo/snakefiles/embedding.snake @@ -52,7 +52,7 @@ rule build_embedding: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -91,7 +91,7 @@ rule plot_embedding_edge_plane_diff: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -116,7 +116,7 @@ rule plot_embedding_n_neigbhours: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -140,7 +140,7 @@ rule plot_embedding_edge_performance: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -172,7 +172,7 @@ rule plot_embedding_best_tracking_performance: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), diff --git a/etx4velo/snakefiles/gnn.snake b/etx4velo/snakefiles/gnn.snake index cecbf353..97c1d671 100644 --- a/etx4velo/snakefiles/gnn.snake +++ b/etx4velo/snakefiles/gnn.snake @@ -31,7 +31,7 @@ rule plot_edge_triplet_performance: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -63,7 +63,7 @@ rule plot_gnn_best_tracking_performance: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), diff --git a/etx4velo/snakefiles/plotfactory.snake b/etx4velo/snakefiles/plotfactory.snake index 15590b2e..e07f6d98 100644 --- a/etx4velo/snakefiles/plotfactory.snake +++ b/etx4velo/snakefiles/plotfactory.snake @@ -50,7 +50,7 @@ rule plot_embedding_explanation: batch_dir=op.join( get_test_batch_dir( experiment_name="{experiment_name}", - stage="metric_learning_processed", + stage="embedding_processed", test_dataset_name="{test_dataset_name}", ), "done", diff --git a/etx4velo/snakefiles/track_building.snake b/etx4velo/snakefiles/track_building.snake index 92defdbf..cf0a3399 100644 --- a/etx4velo/snakefiles/track_building.snake +++ b/etx4velo/snakefiles/track_building.snake @@ -5,7 +5,7 @@ rule build_tracks: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -34,7 +34,7 @@ rule build_tracks_from_edges: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), diff --git a/etx4velo/snakefiles/trackfactory.snake b/etx4velo/snakefiles/trackfactory.snake index 79ac94e6..ebd4a830 100644 --- a/etx4velo/snakefiles/trackfactory.snake +++ b/etx4velo/snakefiles/trackfactory.snake @@ -12,7 +12,7 @@ rule plot_crossing_tracks: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), @@ -45,7 +45,7 @@ rule plot_shared_electron_tracks: get_test_batch_dir( experiment_name="{experiment_name}", test_dataset_name="{test_dataset_name}", - stage="metric_learning_processed", + stage="embedding_processed", ), "done", ), -- GitLab From 133a3af27d463d64c34edae388ec413dde425a94 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:05:14 +0100 Subject: [PATCH 12/17] Update GPU environment --- setup/gpu_environment_cuda118.yaml | 55 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/setup/gpu_environment_cuda118.yaml b/setup/gpu_environment_cuda118.yaml index 87f047ad..35044999 100644 --- a/setup/gpu_environment_cuda118.yaml +++ b/setup/gpu_environment_cuda118.yaml @@ -1,4 +1,4 @@ -name: etx4velo_env_212_118 +name: etx4velo_env channels: - rapidsai @@ -9,30 +9,33 @@ channels: - bioconda dependencies: -- tqdm -- jupyterlab -- ipywidgets -- numpy -- scipy -- pandas -- pyarrow -- numba>=0.57.0 -- matplotlib -- uncertainties -- tabulate +- python>=3.10 +- tqdm # progress bar +- jupyterlab # jupyter notebooks +- ipywidgets # interactive widgets in jupyter notebooks +- ipympl # interactive matplotlib plots in jupyter notebooks +- numpy # matrix operations on CPU +- scipy # connected component algorithm on CPU +- pandas # dataframes +- pyarrow # parquet file format +- numba>=0.57.0 # Just-in-time compilation on CPU +- matplotlib # plotting +- uncertainties # error propagation +- tabulate # pretty printing of tables (for `montetracko` package) - pytorch=2.1.2 -- pytorch-cuda==11.8 -- torchvision +- pytorch-cuda==11.8 # CUDA for PyTorch +- torchvision # for sigmoid focal loss - pytorch-lightning -- pytorch-scatter -- pyg -- cudatoolkit==11.8 -- cupy -- cudf=23.12 -- cugraph -- faiss-gpu=1.7.4 -- black -- snakemake-minimal -- onnx -- torchinfo -- onnxruntime +- pytorch-scatter # for scatter operations within the GNN +- pyg # for GNN. Only used for its data container so far +- cudatoolkit==11.8 # CUDA for the RAPIDS packages and for faiss +- cupy # GPU matrix operations +- cudf=23.12 # GPU dataframes +- cugraph # Connected component algorithm on GPU +- faiss-gpu=1.7.4 # k-nearest neighbours search on GPU +- black # formatting +- snakemake-minimal # workflow management (used to produce many plots) +- onnx # Open Neural Network Exchange format +- torchinfo # model summary +- onnxruntime # ONNX runtime +- root # efficiency uncertainty computation -- GitLab From 6de8706580377099a907970f607b4903be1defed Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:05:22 +0100 Subject: [PATCH 13/17] Update the CPU environment --- setup/cpu_environment.yaml | 45 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/setup/cpu_environment.yaml b/setup/cpu_environment.yaml index f531ab71..625368f0 100644 --- a/setup/cpu_environment.yaml +++ b/setup/cpu_environment.yaml @@ -7,25 +7,28 @@ channels: - bioconda dependencies: -- python>=3.9 -- tqdm # progress bars -- jupyterlab # notebooks -- ipywidgets # beautiful process bar -- numpy # arrays -- scipy # connected component on CPU +- tqdm # progress bar +- jupyterlab # jupyter notebooks +- ipywidgets # interactive widgets in jupyter notebooks +- ipympl # interactive matplotlib plots in jupyter notebooks +- numpy # matrix operations on CPU +- scipy # connected component algorithm on CPU - pandas # dataframes -- pyarrow # parquet files -- numba>=0.57.0 # Just-in-time compilation for operations that cannot be vectorised in numpy -- matplotlib # create figures -- uncertainties # numbers with uncertainties -- tabulate # report efficiencies in beautiful table -- pytorch==2.0.1 # Neural networks -- cpuonly # pytorch on CPUs -- torchvision # focal loss -- pytorch-lightning # training -- pytorch-scatter # scatter operations used in GNNs -- pyg # graphs -- faiss-cpu # kNN -- black # python formatter -- snakemake-minimal # generate figures in a systematic way -- root # computation of errors on efficiencies +- pyarrow # parquet file format +- numba>=0.57.0 # Just-in-time compilation on CPU +- matplotlib # plotting +- uncertainties # error propagation +- tabulate # pretty printing of tables (for `montetracko` package) +- pytorch=2.1.2 +- cpuonly # PyTorch on CPU +- torchvision # for sigmoid focal loss +- pytorch-lightning +- pytorch-scatter # for scatter operations within the GNN +- pyg # for GNN. Only used for its data container so far +- faiss-cpu # k-nearest neighbours search +- black # formatting +- snakemake-minimal # workflow management (used to produce many plots) +- onnx # Open Neural Network Exchange format +- torchinfo # model summary +- onnxruntime # ONNX runtime +- root # efficiency uncertainty computation -- GitLab From 7361827cebf2acd72943a7bc07fa3a85fcc90f22 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:46:42 +0100 Subject: [PATCH 14/17] Define introduction page for the tutorial --- tutorial/00_introduction.md | 241 ++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 tutorial/00_introduction.md diff --git a/tutorial/00_introduction.md b/tutorial/00_introduction.md new file mode 100644 index 00000000..569d2495 --- /dev/null +++ b/tutorial/00_introduction.md @@ -0,0 +1,241 @@ +# Introduction + +Welcome to our tutorial series on utilizing the ETX4VELO repository +and its associated packages. + +As we begin, please keep in mind that my expertise is not in professional software +development! +Consequently, you might encounter areas within the repository where object definitions +and overall structure could be improved. + +## Setting up + +To follow this series of tutorial, it is more convenient to have access to a NVIDIA +GPU. + +Additionally, a UNIX-like operating system is recommended to follow along +with the setup and examples provided. + +### Check your CUDA drivers + +Start by ensuring that your system has the CUDA drivers installed. +You can verify this by running the `nvidia-smi` command in your terminal: +```bash +$ nvidia-smi +Mon Feb 5 21:37:10 2024 ++---------------------------------------------------------------------------------------+ +| NVIDIA-SMI 535.154.05 Driver Version: 535.154.05 CUDA Version: 12.2 | +|-----------------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|=========================================+======================+======================| +| 0 NVIDIA A100 80GB PCIe On | 00000000:E3:00.0 Off | On | +| N/A 38C P0 66W / 300W | 381MiB / 81920MiB | N/A Default | +| | | Enabled | ++-----------------------------------------+----------------------+----------------------+ + ++---------------------------------------------------------------------------------------+ +| MIG devices: | ++------------------+--------------------------------+-----------+-----------------------+ +| GPU GI CI MIG | Memory-Usage | Vol| Shared | +| ID ID Dev | BAR1-Usage | SM Unc| CE ENC DEC OFA JPG | +| | | ECC| | +|==================+================================+===========+=======================| +| 0 1 0 0 | 344MiB / 40192MiB | 42 0 | 3 0 2 0 0 | +| | 2MiB / 65535MiB | | | ++------------------+--------------------------------+-----------+-----------------------+ + ++---------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=======================================================================================| +| 0 1 0 46496 C ...aforge/envs/etx4velo_new/bin/python 298MiB | ++--------------------------------------------------------------------------------------- +``` +The output should indicate a CUDA version of 11.8 or higher, +which is necessary for compatibility with the latest PyTorch releases. + +Note that it's not required to manually install the CUDA toolkit as it will +be included in the packages installed later on. + +## Install `mamba` + +```{info} +If you already have `mamba`, you can skip this section. +``` + +We recommend using [mamba](https://mamba.readthedocs.io/en/latest/index.html) +for package management due to its significantly faster performance compared to conda. + +```{info} +If `conda` is already installed, you have the option to +[install mamba in your base environment](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html#existing-conda-install-not-recommended). +However, this approach is not generally recommended. + +If you prefer, you can continue using conda, though mamba offers a more efficient experience! +``` + +As of September 2023, mamba is automatically included with `miniforge`. +Follow the [steps below](https://github.com/conda-forge/miniforge) +to download and install the latest version of Miniforge, which bundles mamba: +```bash +curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" +bash Miniforge3-$(uname)-$(uname -m).sh +``` + +During installation, you'll be prompted to update your shell profile +to automatically initialize conda. +Accepting this option will add the necessary initialization commands to your `~/.bashrc` file: +```bash +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('/home/{yourusername}/miniforge3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +if [ $? -eq 0 ]; then + eval "$__conda_setup" +else + if [ -f "/home/{yourusername}/miniforge3/etc/profile.d/conda.sh" ]; then + . "/home/{yourusername}/miniforge3/etc/profile.d/conda.sh" + else + export PATH="/home/{yourusername}/miniforge3/bin:$PATH" + fi +fi +unset __conda_setup + +if [ -f "/home/{yourusername}/miniforge3/etc/profile.d/mamba.sh" ]; then + . "/home/{yourusername}/miniforge3/etc/profile.d/mamba.sh" +fi +# <<< conda initialize <<< +``` +(where `/home/{yourusername}/miniforge3` is the path to your `miniforge` installation) +You may reopen your terminal for the changes to take effect. + +You can check that conda is installed by executing +```bash +$ conda +usage: conda [-h] [-v] [--no-plugins] [-V] COMMAND ... + +conda is a tool for managing and deploying applications, environments and packages. + +options: + -h, --help Show this help message and exit. + -v, --verbose Can be used multiple times. Once for detailed output, twice for INFO logging, + thrice for DEBUG logging, four times for TRACE logging. + --no-plugins Disable all plugins that are not built into conda. + -V, --version Show the conda version number and exit. + +commands: + The following built-in and plugins subcommands are available. + + COMMAND + activate Activate a conda environment. + clean Remove unused packages and caches. + compare Compare packages between conda environments. + config Modify configuration values in .condarc. + create Create a new conda environment from a list of specified packages. + deactivate Deactivate the current active conda environment. + doctor Display a health report for your environment. + env See `conda env --help`. + info Display information about current conda install. + init Initialize conda for shell interaction. + install Install a list of packages into a specified conda environment. + list List installed packages in a conda environment. + notices Retrieve latest channel notifications. + package Create low-level conda packages. (EXPERIMENTAL) + remove (uninstall) + Remove a list of packages from a specified conda environment. + rename Rename an existing environment. + repoquery Advanced search for repodata. + run Run an executable in a conda environment. + search Search for packages and display associated information using the MatchSpec + format. + update (upgrade) Update conda packages to the latest compatible version. +``` + + +## Clone the ETX4VELO repository + +Clone the [`ETX4VELO` repository](https://gitlab.cern.ch/gdl4hep/etx4velo/-/tree/dev) +and switch to the `dev` branch. +```bash +git clone ssh://git@gitlab.cern.ch:7999/gdl4hep/etx4velo.git +cd etx4velo +git checkout dev +``` + +## Set up the conda environment + +The configuration for our project environment is stored within the `setup/` directory +of the ETX4VELO repository. + +To create and install the environment, +navigate to ETX4VELO's root directory and execute the following command: +```bash +conda create -f setup/gpu_environment_cuda118.yaml +``` +We encourage you to review the `setup/gpu_environment_cuda118.yaml` environment file +to understand the purpose behind each package included in this environment. + +To check the environment is working as expected, launch a Python shell +and execute the following commands to ensure that the critical libraries, +especially those requiring CUDA, are correctly loaded and functioning. +```bash +$ python3 +Python 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import torch +>>> import faiss # check that CUDA-enabled libraries are loading correctly +>>> import cudf # check that CUDA-enabled libraries are loading correctly +>>> torch.cuda.is_available() # Confirm PyTorch recognizes your CUDA device +True +``` +To start using the environment, activate it with the following command: +```bash +conda activate etx4velo_env +``` + +```{info} +For those without access to an NVIDIA GPU, a CPU-only version of the environment is available. +You can create this alternative environment using: +```bash +conda create -f setup/gpu_environment_cpu.yaml +``` + +## Download Data + +The necessary data for this tutorial series is stored on the LHCb EOS space, at +``` +/eos/lhcb/user/a/anthonyc/tracking/data +``` + +We will focus on downloading the following subsets: +- Training data: `/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset` +- Test data: `/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset` + +For those with access to LXPLUS, where the EOS system is mounted, +you can download the data directly to your desired directory using `rsync` (or `scp`): +```bash +# Navigate to your data directory +cd <datadir> + +# Download the training data subset +rsync -arv <username>@lxplus.cern.ch:/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset . + +# Download and extract the test samples +rsync -arv --progress <username>@lxplus.cern.ch:/eos/lhcb/user/a/anthonyc/tracking/data/data_validation/v2.4/reference_samples_tutorial.tar.lz4 . +lz4 -d reference_samples_tutorial.tar.lz4 -c | tar xvf - # untar archive +mv reference_samples_tutorial reference_samples # rename +rm reference_samples_tutorial.tar.lz4 # delete archive +``` +Replace `<datadir>` with your target directory path and `<username>` with your LXPLUS username. + +```{info} +Using the XRootD client provides an efficient alternative for downloading files +from EOS. +This method requires the installation of both the Kerberos client +and the XRootD client on your machine. + +For detailed instructions on setting up and using the XRootD client, +refer to [this guide](https://xdigi2csv.docs.cern.ch/master/Access/2.download_csv.html#using-a-xrootd-client-recommended). +``` -- GitLab From 37efe8eac101d167f9f32774690990b7d9833d04 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:48:08 +0100 Subject: [PATCH 15/17] Move tutorial to readme folder --- {tutorial => readme/tutorial}/00_introduction.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tutorial => readme/tutorial}/00_introduction.md (100%) diff --git a/tutorial/00_introduction.md b/readme/tutorial/00_introduction.md similarity index 100% rename from tutorial/00_introduction.md rename to readme/tutorial/00_introduction.md -- GitLab From 2049d7d6a3c078ceda552b7acc96db7ddaa35d8d Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:49:59 +0100 Subject: [PATCH 16/17] Add tutorial series to doc --- docs/source/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 65da6700..bd03d136 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -59,6 +59,14 @@ The trained weights of the various models can be found `here <https://cernbox.ce guide/* +.. toctree:: + :caption: Tutorial Series + :glob: + :hidden: + + tutorial/* + + .. toctree:: :caption: APIDOC :hidden: -- GitLab From eda2e9e775afb7d8e3fe4d761428c6b09c207b45 Mon Sep 17 00:00:00 2001 From: Anthony Correia <anthony.correia@cern.ch> Date: Tue, 6 Feb 2024 15:59:39 +0100 Subject: [PATCH 17/17] Properly use mamba instead of conda! --- readme/tutorial/00_introduction.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readme/tutorial/00_introduction.md b/readme/tutorial/00_introduction.md index 569d2495..fc2bdf78 100644 --- a/readme/tutorial/00_introduction.md +++ b/readme/tutorial/00_introduction.md @@ -172,8 +172,11 @@ of the ETX4VELO repository. To create and install the environment, navigate to ETX4VELO's root directory and execute the following command: ```bash -conda create -f setup/gpu_environment_cuda118.yaml +mamba create -f setup/gpu_environment_cuda118.yaml ``` +If you get an error such as `The following package could not be installed`, +try to replace `mamba create` by `mamba env create`. + We encourage you to review the `setup/gpu_environment_cuda118.yaml` environment file to understand the purpose behind each package included in this environment. -- GitLab