From b33675dc03933857604cc876b53db0700acd4acf Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:17:47 +0100
Subject: [PATCH 01/17] Fix output path of plot_embedding_edge_performance

---
 .../evaluation/embedding/plot_embedding_edge_performance.py     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py b/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py
index f8cc81b4..3ce244a5 100644
--- a/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py
+++ b/etx4velo/evaluation/embedding/plot_embedding_edge_performance.py
@@ -1,6 +1,7 @@
 """A script that plots the edge performance (edge purity and efficiency)
 as a function of the maximal squared distance.
 """
+
 from argparse import ArgumentParser
 import os.path as op
 
@@ -29,6 +30,7 @@ if __name__ == "__main__":
         else op.join(
             cdirs.performance_directory,
             load_config(pipeline_config_path)["common"]["experiment_name"],
+            "embedding",
         )
     )
 
-- 
GitLab


From 7cc0b13d76f47e34e3f1a87beccee5acfd0e2a98 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:18:35 +0100
Subject: [PATCH 02/17] Save figures in svg and pdf by default

---
 etx4velo/pipeline/utils/plotutils/plotools.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/etx4velo/pipeline/utils/plotutils/plotools.py b/etx4velo/pipeline/utils/plotutils/plotools.py
index 1c9e327e..ff5147bb 100644
--- a/etx4velo/pipeline/utils/plotutils/plotools.py
+++ b/etx4velo/pipeline/utils/plotutils/plotools.py
@@ -1,5 +1,6 @@
 """Define some global utilies for plots.
 """
+
 from __future__ import annotations
 import typing
 import os
@@ -15,22 +16,19 @@ from matplotlib.text import Text
 @typing.overload
 def get_figs_axes_on_grid(
     nrows: int, ncols: int, same_fig: typing.Literal[True] = True, **kwargs
-) -> typing.Tuple[Figure, npt.NDArray]:
-    ...
+) -> typing.Tuple[Figure, npt.NDArray]: ...
 
 
 @typing.overload
 def get_figs_axes_on_grid(
     nrows: int, ncols: int, same_fig: typing.Literal[False], **kwargs
-) -> typing.Tuple[npt.NDArray, npt.NDArray]:
-    ...
+) -> typing.Tuple[npt.NDArray, npt.NDArray]: ...
 
 
 @typing.overload
 def get_figs_axes_on_grid(
     nrows: int, ncols: int, same_fig: bool, **kwargs
-) -> typing.Tuple[Figure | npt.NDArray, npt.NDArray]:
-    ...
+) -> typing.Tuple[Figure | npt.NDArray, npt.NDArray]: ...
 
 
 def get_figs_axes_on_grid(
@@ -183,7 +181,7 @@ def hide_repetitive_labels_in_grid(axes: typing.Collection[typing.Collection[Axe
 def save_fig(
     fig: Figure,
     path: str,
-    exts: typing.List[str] = [".pdf", ".png"],
+    exts: typing.List[str] = [".pdf", ".svg"],
     bbox_inches: str | None = "tight",
     **kwargs,
 ):
@@ -198,10 +196,13 @@ def save_fig(
     fig.tight_layout()
 
     path_without_ext, ext = os.path.splitext(path)
-    if ext not in [".png", ".pdf"]:
+    if ext not in exts:
         path_without_ext = path
 
     for ext in exts:
+        assert ext.startswith(".")
         overall_path = path_without_ext + ext
-        fig.savefig(path_without_ext + ext, bbox_inches=bbox_inches, **kwargs)
+        fig.savefig(
+            path_without_ext + ext, format=ext[1:], bbox_inches=bbox_inches, **kwargs
+        )
         print("Figure was saved in", overall_path)
-- 
GitLab


From 44292cdd063319ee8321682d53bc64de9ac4f074 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:19:03 +0100
Subject: [PATCH 03/17] Add matching condition to select best track to particle
 match

---
 montetracko | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/montetracko b/montetracko
index 37f05c8c..742fa387 160000
--- a/montetracko
+++ b/montetracko
@@ -1 +1 @@
-Subproject commit 37f05c8c1162b204386b7fdf6fbc1ced8a802aa3
+Subproject commit 742fa3876253e3cae9f62432cd677fedc6c56340
-- 
GitLab


From 181f11423d59c5e50e1b8f3a7870ff0025eb8d78 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:19:36 +0100
Subject: [PATCH 04/17] Format plot_embedding_best_tracking_performance.py

---
 .../embedding/plot_embedding_best_tracking_performance.py       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
index 4851a418..53db8591 100644
--- a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
+++ b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
@@ -1,5 +1,6 @@
 """A script that plots the metric saved during the learning.
 """
+
 from argparse import ArgumentParser
 import montetracko.lhcb as mtb
 
@@ -42,7 +43,6 @@ if __name__ == "__main__":
         model=embedding_model,
         path_or_config=pipeline_config_path,
         partition=test_dataset_name,
-        # list_squared_distance_max=[0.015, 0.020],
         list_squared_distance_max=[
             0.005,
             0.0075,
-- 
GitLab


From c285419a494c836ea613f94c9a9267540386768a Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:19:47 +0100
Subject: [PATCH 05/17] Put legend inside by default

---
 .../embedding/plot_embedding_best_tracking_performance.py        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
index 53db8591..4c4889ab 100644
--- a/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
+++ b/etx4velo/evaluation/embedding/plot_embedding_best_tracking_performance.py
@@ -74,4 +74,5 @@ if __name__ == "__main__":
         output_path=output_wpath,
         same_fig=False,
         lhcb=True,
+        legend_inside=True,
     )
-- 
GitLab


From ddc65445fd77920ec760a2f564b88bc0e58cb821 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:20:44 +0100
Subject: [PATCH 06/17] Set color of test datasets to black in
 plot_embedding_performance_given_squared_distance_max_k_max

---
 etx4velo/pipeline/Embedding/embedding_plots.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etx4velo/pipeline/Embedding/embedding_plots.py b/etx4velo/pipeline/Embedding/embedding_plots.py
index c50128cc..76b865e6 100644
--- a/etx4velo/pipeline/Embedding/embedding_plots.py
+++ b/etx4velo/pipeline/Embedding/embedding_plots.py
@@ -130,7 +130,7 @@ def plot_embedding_performance_given_squared_distance_max_k_max(
                     if show_err
                     else None
                 ),
-                color=partition_to_color.get(partition_name),
+                color=partition_to_color.get(partition_name, "k"),
                 label=partition_to_label.get(partition_name, partition_label),
                 marker=".",
             )
-- 
GitLab


From e013b0fa37086fd30c86c7745b9a427043eb9ff0 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:21:01 +0100
Subject: [PATCH 07/17] Fix typing and formatting of embedding_plots.py

---
 etx4velo/pipeline/Embedding/embedding_plots.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/etx4velo/pipeline/Embedding/embedding_plots.py b/etx4velo/pipeline/Embedding/embedding_plots.py
index 76b865e6..690ef7f1 100644
--- a/etx4velo/pipeline/Embedding/embedding_plots.py
+++ b/etx4velo/pipeline/Embedding/embedding_plots.py
@@ -1,5 +1,6 @@
 """A module that handles the validation plots for the embedding phase specifically.
 """
+
 from __future__ import annotations
 import typing
 import os.path as op
@@ -180,7 +181,9 @@ def plot_best_performances_squared_distance_max(
 ) -> typing.Tuple[
     Figure | npt.NDArray,
     typing.List[Axes],
-    typing.Dict[float, typing.Dict[typing.Tuple[str | None, str], float]],
+    typing.Dict[
+        float, typing.Dict[typing.Tuple[str | None, str], typing.Dict[str, float]]
+    ],
 ]:
     """Plot best performance for perfect inference as a function of the squared
     maximal distance.
-- 
GitLab


From b1d6c52df209585deb7a0a2bad9668d9cb436703 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:21:28 +0100
Subject: [PATCH 08/17] Use but comment out BestMatchingParticles matching
 condition

---
 etx4velo/pipeline/Evaluation/matching.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/etx4velo/pipeline/Evaluation/matching.py b/etx4velo/pipeline/Evaluation/matching.py
index d3eff864..e94a3ba7 100644
--- a/etx4velo/pipeline/Evaluation/matching.py
+++ b/etx4velo/pipeline/Evaluation/matching.py
@@ -28,7 +28,10 @@ def perform_matching(
         df_events_hits_particles=df_hits_particles,
         df_events_particles=df_particles,
         df_events_tracks_hits=df_tracks,
-        matching_condition=mt.matchcond.MinMatchingFraction(matching_fraction),
+        matching_condition=(
+            mt.matchcond.MinMatchingFraction(matching_fraction)
+            # & mt.matchcond.basics.BestMatchingParticles()
+        ),
         track_condition=mt.matchcond.MinLengthTrack(min_track_length),
     )
 
-- 
GitLab


From 66a4a8eb5542257984d1140b37e53c4f9c6b7631 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:21:54 +0100
Subject: [PATCH 09/17] Define legend_inside parameter in exploration

---
 .../pipeline/utils/modelutils/exploration.py  | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/etx4velo/pipeline/utils/modelutils/exploration.py b/etx4velo/pipeline/utils/modelutils/exploration.py
index 70caaf84..880b4282 100644
--- a/etx4velo/pipeline/utils/modelutils/exploration.py
+++ b/etx4velo/pipeline/utils/modelutils/exploration.py
@@ -1,6 +1,7 @@
 """A module that defines :py:class:`ParamExplorer`, a class that allows to vary
 a parameter and check the efficiency that is obtained for this choice.
 """
+
 from __future__ import annotations
 import typing
 import abc
@@ -203,7 +204,7 @@ class ParamExplorer(abc.ABC):
         if track_metric_names is None:
             track_metric_names = []
 
-        dict_performances: typing.Dict[typing.Tuple[str | None, str], typing.Dict[str, float]] = {  # type: ignore
+        dict_performances: typing.Dict[typing.Tuple[str | None, str]] = {  # type: ignore
             (category.name, metric_name): trackEvaluator.compute_metric(
                 metric_name=metric_name,
                 category=category,
@@ -258,6 +259,7 @@ class ParamExplorer(abc.ABC):
         category_name_to_color: dict | None = None,
         step: str | None = None,
         with_err: bool = True,
+        legend_inside: bool | None = None,
         **kwargs,
     ) -> typing.Tuple[
         Figure | npt.NDArray,
@@ -296,6 +298,8 @@ class ParamExplorer(abc.ABC):
             3-tuple of the Matplotlib Figures and Axes, and the dictionary of
             metric values for every tuple ``(value, category.name, metric_name)``
         """
+        if legend_inside is None:
+            legend_inside = same_fig
         if category_name_to_color is None:
             category_name_to_color = {}
         if step is None:
@@ -339,7 +343,7 @@ class ParamExplorer(abc.ABC):
                     1,
                     len(metric_names),
                     same_fig=False,
-                    figsize=(12, 6),
+                    figsize=(8, 6) if legend_inside else (12, 6),
                 )
             else:
                 figs_with_cat = None
@@ -448,11 +452,19 @@ class ParamExplorer(abc.ABC):
                 plotools.pad_on_top(ax=ax)
                 plotools.add_text(ax=ax, ha="left", va="top")
 
+        if legend_inside:
+            legend_kwargs = dict()
+        else:
+            legend_kwargs = dict(loc="center left", bbox_to_anchor=(1, 0.5))
+
         if same_fig:
-            axes[0].legend()
+            if legend_inside:
+                axes[0].legend(**legend_kwargs)
+            else:
+                axes[-1].legend(**legend_kwargs)
         else:
             for metric_idx, _ in enumerate(metric_names):
-                axes[metric_idx].legend(loc="center left", bbox_to_anchor=(1, 0.5))
+                axes[metric_idx].legend(**legend_kwargs)
 
         if same_fig:
             if output_path is None:
-- 
GitLab


From e6f44e9bba73c99cd90bea0f7fbf19104249cc33 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:22:42 +0100
Subject: [PATCH 10/17] Clarify what fonts are used for plots

---
 etx4velo/pipeline/utils/plotutils/plotconfig.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/etx4velo/pipeline/utils/plotutils/plotconfig.py b/etx4velo/pipeline/utils/plotutils/plotconfig.py
index c2f5b87f..4c637837 100644
--- a/etx4velo/pipeline/utils/plotutils/plotconfig.py
+++ b/etx4velo/pipeline/utils/plotutils/plotconfig.py
@@ -85,7 +85,8 @@ def configure_matplotlib():
         **{
             # Font
             "font.family": "serif",
-            # "font.serif": "Computer Modern Roman",
+            "font.serif": "Computer Modern Roman",
+            # "font.serif": "Times",
             # Latex
             "text.latex.preamble": r"\usepackage{amsmath}",
             "text.usetex": True,  # Put to False to disable Latex
-- 
GitLab


From e9eeee839fa3d1fc3fff4cf26fc1e548019931e5 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Mon, 5 Feb 2024 19:24:16 +0100
Subject: [PATCH 11/17] Change metric_learning to embedding everywhere (except
 analysis notebooks)

---
 etx4velo/pipeline_configs/example.yaml                 |  6 +++---
 .../focal-loss-nopid-shared-hits-triplets.yaml         |  8 ++++----
 .../pipeline_configs/focal-loss-nopid-shared-hits.yaml |  4 ++--
 .../focal-loss-nopid-triplets-embedding-1.yaml         |  8 ++++----
 ...d-triplets-embedding-3-withspillover-d2max0.02.yaml |  6 +++---
 ...d-triplets-embedding-3-withspillover-embedding.yaml |  8 ++++----
 ...s-embedding-3-withspillover-new-same-d2max0.02.yaml |  6 +++---
 ...s-nopid-triplets-embedding-3-withspillover-new.yaml | 10 +++++-----
 ...-loss-nopid-triplets-embedding-3-withspillover.yaml |  8 ++++----
 .../focal-loss-nopid-triplets-embedding-3.yaml         |  8 ++++----
 ...opid-triplets-embedding-6-withspillover-nonorm.yaml |  6 +++---
 ...d-triplets-embedding-6-withspillover-unsquared.yaml |  6 +++---
 ...-loss-nopid-triplets-embedding-6-withspillover.yaml |  6 +++---
 .../focal-loss-nopid-triplets-embedding.yaml           |  8 ++++----
 etx4velo/pipeline_configs/scifi-1.yaml                 |  6 +++---
 etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml      |  4 ++--
 etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml     |  8 ++++----
 .../scifi-3-xuvx-long-checkpointing.yaml               |  8 ++++----
 etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml  |  8 ++++----
 etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml       |  8 ++++----
 etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml      |  8 ++++----
 .../scifi-3-xuvx-reconstructible-deep-no-triplets.yaml |  8 ++++----
 .../scifi-3-xuvx-reconstructible-deep.yaml             |  8 ++++----
 .../pipeline_configs/scifi-3-xuvx-reconstructible.yaml |  8 ++++----
 etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml      |  8 ++++----
 etx4velo/pipeline_configs/scifi-3-xuvx.yaml            |  8 ++++----
 etx4velo/pipeline_configs/scifi-5.yaml                 |  6 +++---
 .../pipeline_configs/velo-6-query-long-strange.yaml    | 10 +++++-----
 etx4velo/pipeline_configs/velo-6-query-long.yaml       | 10 +++++-----
 .../velo-6-triplets-improved-strange-fixed.yaml        | 10 +++++-----
 .../velo-6-triplets-improved-strange-long-fixed.yaml   | 10 +++++-----
 .../velo-6-triplets-improved-strange-long.yaml         | 10 +++++-----
 .../velo-6-triplets-improved-strange.yaml              | 10 +++++-----
 .../pipeline_configs/velo-6-triplets-improved.yaml     | 10 +++++-----
 .../velo-scatter-sum-long-epoch-128.yaml               |  6 +++---
 .../pipeline_configs/velo-scatter-sum-long-epoch.yaml  |  6 +++---
 etx4velo/pipeline_configs/velo-scatter-sum-max.yaml    |  6 +++---
 etx4velo/pipeline_configs/velo-scatter-sum.yaml        |  6 +++---
 .../velo-scatter-sum_max-checkpointing.yaml            |  6 +++---
 etx4velo/pipeline_configs/velo_diff.yaml               |  6 +++---
 etx4velo/pipeline_configs/velo_edge_based.yaml         |  6 +++---
 .../pipeline_configs/velo_edge_based_long_epoch.yaml   |  6 +++---
 .../velo_edge_based_very_long_epoch.yaml               |  6 +++---
 .../velo_edge_based_very_long_epoch_128.yaml           |  6 +++---
 .../velo_edge_based_very_long_epoch_64.yaml            |  6 +++---
 etx4velo/pipeline_configs/velo_gcn.yaml                |  6 +++---
 etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml  |  6 +++---
 .../pipeline_configs/velo_nonrecursive_edge_based.yaml |  6 +++---
 etx4velo/snakefiles/embedding.snake                    | 10 +++++-----
 etx4velo/snakefiles/gnn.snake                          |  4 ++--
 etx4velo/snakefiles/plotfactory.snake                  |  2 +-
 etx4velo/snakefiles/track_building.snake               |  4 ++--
 etx4velo/snakefiles/trackfactory.snake                 |  4 ++--
 53 files changed, 186 insertions(+), 186 deletions(-)

diff --git a/etx4velo/pipeline_configs/example.yaml b/etx4velo/pipeline_configs/example.yaml
index 46ae38b8..29b52c27 100644
--- a/etx4velo/pipeline_configs/example.yaml
+++ b/etx4velo/pipeline_configs/example.yaml
@@ -62,7 +62,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true # Whether to load an event of the training set at a time
   # Whether to split the training sample by `trainset_split` partitions
   # and  define an "epoch" as going through one partition.
@@ -122,7 +122,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   lazy: true # Whether to load an event of the training set at a time
   # Whether to split the training sample by `trainset_split` partitions
@@ -188,5 +188,5 @@ track_building:
   # Minimal triplet score required to filter out the fake triplets before building
   # the tracks
   triplet_score_cut: 0.38
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml
index 9d973106..98229b87 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits-triplets.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
 
   # Model parameters
   feature_indices: 4
@@ -65,7 +65,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_cut: 0.5
   bidir: False
@@ -105,9 +105,9 @@ track_building:
   edge_score_cut: 0.6
   triplet_score_cut: 0.2
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml
index a0405d2c..68d3aebf 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-shared-hits.yaml
@@ -33,7 +33,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
 
   # Model parameters
   feature_indices: 4
@@ -64,7 +64,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_cut: 0.5
   bidir: False
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml
index 84b98ddb..90f31bb2 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-1.yaml
@@ -35,7 +35,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   model: triplet_interaction
@@ -75,7 +75,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_cut: 0.5
   bidir: False
@@ -115,9 +115,9 @@ track_building:
   edge_score_cut: 0.6
   triplet_score_cut: 0.2
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml
index f9960ef7..9e0c78ad 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-d2max0.02.yaml
@@ -36,7 +36,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -77,7 +77,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.36
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml
index cdf45a97..899357fc 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-embedding.yaml
@@ -41,7 +41,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -81,7 +81,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -119,9 +119,9 @@ track_building:
   edge_score_cut: 0.5
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml
index a7b4f348..150de14b 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new-same-d2max0.02.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -83,7 +83,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -120,5 +120,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.38
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml
index 138a6aef..4e37b173 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover-new.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -113,14 +113,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml
index 699c3fff..1a5c761b 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3-withspillover.yaml
@@ -39,7 +39,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -78,7 +78,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -116,9 +116,9 @@ track_building:
   edge_score_cut: 0.5
   triplet_score_cut: 0.25
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml
index 4307389e..dc47c905 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-3.yaml
@@ -39,7 +39,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   model: triplet_interaction
@@ -80,7 +80,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -126,9 +126,9 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.40
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml
index 66cdfc49..e4809f74 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-nonorm.yaml
@@ -36,7 +36,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -78,7 +78,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml
index 2ae790aa..0b6400b5 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover-unsquared.yaml
@@ -41,7 +41,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -84,7 +84,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -121,5 +121,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml
index 7528a02e..9efec383 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding-6-withspillover.yaml
@@ -41,7 +41,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -83,7 +83,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -120,5 +120,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml
index 4c715a19..cafa3b5a 100644
--- a/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml
+++ b/etx4velo/pipeline_configs/focal-loss-nopid-triplets-embedding.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   model: triplet_interaction
@@ -74,7 +74,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -116,9 +116,9 @@ track_building:
   edge_score_cut: 0.6
   triplet_score_cut: 0.4
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/scifi-1.yaml b/etx4velo/pipeline_configs/scifi-1.yaml
index ead44301..354d88bb 100644
--- a/etx4velo/pipeline_configs/scifi-1.yaml
+++ b/etx4velo/pipeline_configs/scifi-1.yaml
@@ -38,7 +38,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -80,7 +80,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -117,5 +117,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml
index e12baa9b..ffb4b90f 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-links.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -80,7 +80,7 @@ metric_learning:
 
 metric_learning_xu_vx:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "metric_learning_xu_vx_processed"
   lazy: true
   trainset_split: 80
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml
index 72ae667f..b75246e7 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-2.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -79,7 +79,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -121,11 +121,11 @@ track_building:
   edge_score_cut: 0.86
   triplet_score_cut: 0.1
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.86
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml
index 64a5aa70..f1023e89 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-checkpointing.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -80,7 +80,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -127,11 +127,11 @@ track_building:
     elbow_left: 0.75
     elbow_right: 0.75
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.86
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml
index fedc7cd6..57288733 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long-deep.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -79,7 +79,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -121,11 +121,11 @@ track_building:
   edge_score_cut: 0.84
   triplet_score_cut: 0.1
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.87
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml
index 4dbc4fb8..c3b0e1d3 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-long.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -79,7 +79,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -121,11 +121,11 @@ track_building:
   edge_score_cut: 0.85
   triplet_score_cut: 0.3
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.85
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml
index c53afac4..7bf72b98 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-naive.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -79,7 +79,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -121,11 +121,11 @@ track_building:
   edge_score_cut: 0.86
   triplet_score_cut: 0.3
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.87
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml
index e0ee2cbc..a9ea5d47 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep-no-triplets.yaml
@@ -39,7 +39,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -81,7 +81,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.8
   triplet_score_cut: 0.2
@@ -118,11 +118,11 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.85
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml
index 7b0f7e11..21819e5e 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible-deep.yaml
@@ -39,7 +39,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -81,7 +81,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.8
   triplet_score_cut: 0.2
@@ -118,11 +118,11 @@ track_building:
   edge_score_cut: 0.85
   triplet_score_cut: 0.3
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.86
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml
index 7fa596de..7db39276 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-reconstructible.yaml
@@ -39,7 +39,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -81,7 +81,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.8
   triplet_score_cut: 0.2
@@ -118,11 +118,11 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.85
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml
index 3fbf8eb2..c735e6d6 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx-scifi.yaml
@@ -37,7 +37,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -79,7 +79,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.65
   triplet_score_cut: 0.4
@@ -121,11 +121,11 @@ track_building:
   edge_score_cut: 0.86
   triplet_score_cut: 0.3
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.86
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-3-xuvx.yaml b/etx4velo/pipeline_configs/scifi-3-xuvx.yaml
index c6ba3744..b1ee1c08 100644
--- a/etx4velo/pipeline_configs/scifi-3-xuvx.yaml
+++ b/etx4velo/pipeline_configs/scifi-3-xuvx.yaml
@@ -38,7 +38,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -80,7 +80,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -117,7 +117,7 @@ track_building:
   edge_score_cut: 0.9
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 
@@ -125,5 +125,5 @@ track_building_from_edges:
   edge_score_cut: 0.9
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
diff --git a/etx4velo/pipeline_configs/scifi-5.yaml b/etx4velo/pipeline_configs/scifi-5.yaml
index fcee066c..c820f71b 100644
--- a/etx4velo/pipeline_configs/scifi-5.yaml
+++ b/etx4velo/pipeline_configs/scifi-5.yaml
@@ -38,7 +38,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 80
   on_step: false
@@ -80,7 +80,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -117,5 +117,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml b/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml
index 812d37c1..81396f79 100644
--- a/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml
+++ b/etx4velo/pipeline_configs/velo-6-query-long-strange.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -87,7 +87,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -124,14 +124,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-query-long.yaml b/etx4velo/pipeline_configs/velo-6-query-long.yaml
index 80770bc4..c6bb9bc1 100644
--- a/etx4velo/pipeline_configs/velo-6-query-long.yaml
+++ b/etx4velo/pipeline_configs/velo-6-query-long.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -85,7 +85,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -122,14 +122,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml
index 7dc65861..0dfcfd9c 100644
--- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml
+++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-fixed.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -87,7 +87,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -124,14 +124,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml
index 7a7dd5cd..66e4bb5f 100644
--- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml
+++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long-fixed.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -86,7 +86,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -123,14 +123,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml
index 7a7dd5cd..66e4bb5f 100644
--- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml
+++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange-long.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -86,7 +86,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -123,14 +123,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml
index 7dc65861..0dfcfd9c 100644
--- a/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml
+++ b/etx4velo/pipeline_configs/velo-6-triplets-improved-strange.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 50
   on_step: false
@@ -87,7 +87,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -124,14 +124,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml b/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml
index 3645574b..f856fa9d 100644
--- a/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml
+++ b/etx4velo/pipeline_configs/velo-6-triplets-improved.yaml
@@ -42,7 +42,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -85,7 +85,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -122,14 +122,14 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
 
 track_building_from_edges:
   edge_score_cut: 0.8
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_from_edges_processed"
 
 track_building_perfect:
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "perfect_track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml
index 0d3096ec..6ac01403 100644
--- a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml
+++ b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch-128.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -113,5 +113,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml
index 9ae102f2..664768e7 100644
--- a/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml
+++ b/etx4velo/pipeline_configs/velo-scatter-sum-long-epoch.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -113,5 +113,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.32
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml b/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml
index 682aaefb..93751e2a 100644
--- a/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml
+++ b/etx4velo/pipeline_configs/velo-scatter-sum-max.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -113,5 +113,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-scatter-sum.yaml b/etx4velo/pipeline_configs/velo-scatter-sum.yaml
index ae403089..5435669d 100644
--- a/etx4velo/pipeline_configs/velo-scatter-sum.yaml
+++ b/etx4velo/pipeline_configs/velo-scatter-sum.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -113,5 +113,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml b/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml
index ea7f8368..ca6eac8d 100644
--- a/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml
+++ b/etx4velo/pipeline_configs/velo-scatter-sum_max-checkpointing.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -114,5 +114,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_diff.yaml b/etx4velo/pipeline_configs/velo_diff.yaml
index 4f84e5e7..f2539d55 100644
--- a/etx4velo/pipeline_configs/velo_diff.yaml
+++ b/etx4velo/pipeline_configs/velo_diff.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_edge_based.yaml b/etx4velo/pipeline_configs/velo_edge_based.yaml
index f41cb487..b23c615f 100644
--- a/etx4velo/pipeline_configs/velo_edge_based.yaml
+++ b/etx4velo/pipeline_configs/velo_edge_based.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -116,5 +116,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.34
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml b/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml
index 4b867abb..efb14bd9 100644
--- a/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml
+++ b/etx4velo/pipeline_configs/velo_edge_based_long_epoch.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.34
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml
index e009981a..764860d8 100644
--- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml
+++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -116,5 +116,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml
index 42d240ea..6fb58541 100644
--- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml
+++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_128.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -75,7 +75,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -114,5 +114,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml
index cf2b096c..27ac97cb 100644
--- a/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml
+++ b/etx4velo/pipeline_configs/velo_edge_based_very_long_epoch_64.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_gcn.yaml b/etx4velo/pipeline_configs/velo_gcn.yaml
index 4aaca744..ffcd1efa 100644
--- a/etx4velo/pipeline_configs/velo_gcn.yaml
+++ b/etx4velo/pipeline_configs/velo_gcn.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml b/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml
index e9a6b90d..925896b1 100644
--- a/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml
+++ b/etx4velo/pipeline_configs/velo_nonrecursive_diff.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -115,5 +115,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml b/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml
index ece9552a..decae5d1 100644
--- a/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml
+++ b/etx4velo/pipeline_configs/velo_nonrecursive_edge_based.yaml
@@ -34,7 +34,7 @@ processing:
 metric_learning:
   # Dataset parameters
   input_subdirectory: "processed"
-  output_subdirectory: "metric_learning_processed"
+  output_subdirectory: "embedding_processed"
   lazy: true
   trainset_split: 100
   on_step: false
@@ -76,7 +76,7 @@ metric_learning:
 
 gnn:
   # Dataset parameters
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "gnn_processed"
   edge_score_cut: 0.5
   triplet_score_cut: 0.2
@@ -116,5 +116,5 @@ track_building:
   edge_score_cut: 0.4
   triplet_score_cut: 0.26
   # input_subdirectory: "gnn_processed"
-  input_subdirectory: "metric_learning_processed"
+  input_subdirectory: "embedding_processed"
   output_subdirectory: "track_building_processed"
diff --git a/etx4velo/snakefiles/embedding.snake b/etx4velo/snakefiles/embedding.snake
index 62c8c3ce..bf299bcb 100644
--- a/etx4velo/snakefiles/embedding.snake
+++ b/etx4velo/snakefiles/embedding.snake
@@ -52,7 +52,7 @@ rule build_embedding:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -91,7 +91,7 @@ rule plot_embedding_edge_plane_diff:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -116,7 +116,7 @@ rule plot_embedding_n_neigbhours:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -140,7 +140,7 @@ rule plot_embedding_edge_performance:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -172,7 +172,7 @@ rule plot_embedding_best_tracking_performance:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
diff --git a/etx4velo/snakefiles/gnn.snake b/etx4velo/snakefiles/gnn.snake
index cecbf353..97c1d671 100644
--- a/etx4velo/snakefiles/gnn.snake
+++ b/etx4velo/snakefiles/gnn.snake
@@ -31,7 +31,7 @@ rule plot_edge_triplet_performance:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -63,7 +63,7 @@ rule plot_gnn_best_tracking_performance:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
diff --git a/etx4velo/snakefiles/plotfactory.snake b/etx4velo/snakefiles/plotfactory.snake
index 15590b2e..e07f6d98 100644
--- a/etx4velo/snakefiles/plotfactory.snake
+++ b/etx4velo/snakefiles/plotfactory.snake
@@ -50,7 +50,7 @@ rule plot_embedding_explanation:
         batch_dir=op.join(
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
                 test_dataset_name="{test_dataset_name}",
             ),
             "done",
diff --git a/etx4velo/snakefiles/track_building.snake b/etx4velo/snakefiles/track_building.snake
index 92defdbf..cf0a3399 100644
--- a/etx4velo/snakefiles/track_building.snake
+++ b/etx4velo/snakefiles/track_building.snake
@@ -5,7 +5,7 @@ rule build_tracks:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -34,7 +34,7 @@ rule build_tracks_from_edges:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
diff --git a/etx4velo/snakefiles/trackfactory.snake b/etx4velo/snakefiles/trackfactory.snake
index 79ac94e6..ebd4a830 100644
--- a/etx4velo/snakefiles/trackfactory.snake
+++ b/etx4velo/snakefiles/trackfactory.snake
@@ -12,7 +12,7 @@ rule plot_crossing_tracks:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
@@ -45,7 +45,7 @@ rule plot_shared_electron_tracks:
             get_test_batch_dir(
                 experiment_name="{experiment_name}",
                 test_dataset_name="{test_dataset_name}",
-                stage="metric_learning_processed",
+                stage="embedding_processed",
             ),
             "done",
         ),
-- 
GitLab


From 133a3af27d463d64c34edae388ec413dde425a94 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:05:14 +0100
Subject: [PATCH 12/17] Update GPU environment

---
 setup/gpu_environment_cuda118.yaml | 55 ++++++++++++++++--------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/setup/gpu_environment_cuda118.yaml b/setup/gpu_environment_cuda118.yaml
index 87f047ad..35044999 100644
--- a/setup/gpu_environment_cuda118.yaml
+++ b/setup/gpu_environment_cuda118.yaml
@@ -1,4 +1,4 @@
-name: etx4velo_env_212_118
+name: etx4velo_env
 
 channels:
 - rapidsai
@@ -9,30 +9,33 @@ channels:
 - bioconda
 
 dependencies:
-- tqdm
-- jupyterlab
-- ipywidgets
-- numpy
-- scipy
-- pandas
-- pyarrow
-- numba>=0.57.0
-- matplotlib
-- uncertainties
-- tabulate
+- python>=3.10
+- tqdm # progress bar
+- jupyterlab # jupyter notebooks
+- ipywidgets # interactive widgets in jupyter notebooks
+- ipympl # interactive matplotlib plots in jupyter notebooks
+- numpy # matrix operations on CPU
+- scipy # connected component algorithm on CPU
+- pandas # dataframes
+- pyarrow # parquet file format
+- numba>=0.57.0 # Just-in-time compilation on CPU
+- matplotlib # plotting
+- uncertainties # error propagation
+- tabulate # pretty printing of tables (for `montetracko` package)
 - pytorch=2.1.2
-- pytorch-cuda==11.8
-- torchvision
+- pytorch-cuda==11.8 # CUDA for PyTorch
+- torchvision # for sigmoid focal loss
 - pytorch-lightning
-- pytorch-scatter
-- pyg
-- cudatoolkit==11.8
-- cupy
-- cudf=23.12
-- cugraph
-- faiss-gpu=1.7.4
-- black
-- snakemake-minimal
-- onnx
-- torchinfo
-- onnxruntime
+- pytorch-scatter # for scatter operations within the GNN
+- pyg # for GNN. Only used for its data container so far
+- cudatoolkit==11.8 # CUDA for the RAPIDS packages and for faiss
+- cupy # GPU matrix operations
+- cudf=23.12 # GPU dataframes
+- cugraph # Connected component algorithm on GPU
+- faiss-gpu=1.7.4 # k-nearest neighbours search on GPU
+- black # formatting
+- snakemake-minimal # workflow management (used to produce many plots)
+- onnx # Open Neural Network Exchange format
+- torchinfo # model summary
+- onnxruntime # ONNX runtime
+- root # efficiency uncertainty computation
-- 
GitLab


From 6de8706580377099a907970f607b4903be1defed Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:05:22 +0100
Subject: [PATCH 13/17] Update the CPU environment

---
 setup/cpu_environment.yaml | 45 ++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/setup/cpu_environment.yaml b/setup/cpu_environment.yaml
index f531ab71..625368f0 100644
--- a/setup/cpu_environment.yaml
+++ b/setup/cpu_environment.yaml
@@ -7,25 +7,28 @@ channels:
 - bioconda
 
 dependencies:
-- python>=3.9
-- tqdm # progress bars
-- jupyterlab # notebooks
-- ipywidgets # beautiful process bar
-- numpy # arrays
-- scipy # connected component on CPU
+- tqdm # progress bar
+- jupyterlab # jupyter notebooks
+- ipywidgets # interactive widgets in jupyter notebooks
+- ipympl # interactive matplotlib plots in jupyter notebooks
+- numpy # matrix operations on CPU
+- scipy # connected component algorithm on CPU
 - pandas # dataframes
-- pyarrow # parquet files
-- numba>=0.57.0 # Just-in-time compilation for operations that cannot be vectorised in numpy
-- matplotlib # create figures
-- uncertainties # numbers with uncertainties
-- tabulate # report efficiencies in beautiful table
-- pytorch==2.0.1 # Neural networks
-- cpuonly # pytorch on CPUs
-- torchvision # focal loss
-- pytorch-lightning # training
-- pytorch-scatter # scatter operations used in GNNs
-- pyg # graphs
-- faiss-cpu # kNN
-- black # python formatter
-- snakemake-minimal # generate figures in a systematic way
-- root # computation of errors on efficiencies
+- pyarrow # parquet file format
+- numba>=0.57.0 # Just-in-time compilation on CPU
+- matplotlib # plotting
+- uncertainties # error propagation
+- tabulate # pretty printing of tables (for `montetracko` package)
+- pytorch=2.1.2
+- cpuonly # PyTorch on CPU
+- torchvision # for sigmoid focal loss
+- pytorch-lightning
+- pytorch-scatter # for scatter operations within the GNN
+- pyg # for GNN. Only used for its data container so far
+- faiss-cpu # k-nearest neighbours search
+- black # formatting
+- snakemake-minimal # workflow management (used to produce many plots)
+- onnx # Open Neural Network Exchange format
+- torchinfo # model summary
+- onnxruntime # ONNX runtime
+- root # efficiency uncertainty computation
-- 
GitLab


From 7361827cebf2acd72943a7bc07fa3a85fcc90f22 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:46:42 +0100
Subject: [PATCH 14/17] Define introduction page for the tutorial

---
 tutorial/00_introduction.md | 241 ++++++++++++++++++++++++++++++++++++
 1 file changed, 241 insertions(+)
 create mode 100644 tutorial/00_introduction.md

diff --git a/tutorial/00_introduction.md b/tutorial/00_introduction.md
new file mode 100644
index 00000000..569d2495
--- /dev/null
+++ b/tutorial/00_introduction.md
@@ -0,0 +1,241 @@
+# Introduction
+
+Welcome to our tutorial series on utilizing the ETX4VELO repository
+and its associated packages.
+
+As we begin, please keep in mind that my expertise is not in professional software
+development!
+Consequently, you might encounter areas within the repository where object definitions
+and overall structure could be improved.
+
+## Setting up
+
+To follow this series of tutorial, it is more convenient to have access to a NVIDIA
+GPU.
+
+Additionally, a UNIX-like operating system is recommended to follow along
+with the setup and examples provided.
+
+### Check your CUDA drivers
+
+Start by ensuring that your system has the CUDA drivers installed.
+You can verify this by running the `nvidia-smi` command in your terminal:
+```bash
+$ nvidia-smi
+Mon Feb  5 21:37:10 2024       
++---------------------------------------------------------------------------------------+
+| NVIDIA-SMI 535.154.05             Driver Version: 535.154.05   CUDA Version: 12.2     |
+|-----------------------------------------+----------------------+----------------------+
+| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
+|                                         |                      |               MIG M. |
+|=========================================+======================+======================|
+|   0  NVIDIA A100 80GB PCIe          On  | 00000000:E3:00.0 Off |                   On |
+| N/A   38C    P0              66W / 300W |    381MiB / 81920MiB |     N/A      Default |
+|                                         |                      |              Enabled |
++-----------------------------------------+----------------------+----------------------+
+
++---------------------------------------------------------------------------------------+
+| MIG devices:                                                                          |
++------------------+--------------------------------+-----------+-----------------------+
+| GPU  GI  CI  MIG |                   Memory-Usage |        Vol|      Shared           |
+|      ID  ID  Dev |                     BAR1-Usage | SM     Unc| CE ENC DEC OFA JPG    |
+|                  |                                |        ECC|                       |
+|==================+================================+===========+=======================|
+|  0    1   0   0  |             344MiB / 40192MiB  | 42      0 |  3   0    2    0    0 |
+|                  |               2MiB / 65535MiB  |           |                       |
++------------------+--------------------------------+-----------+-----------------------+
+                                                                                         
++---------------------------------------------------------------------------------------+
+| Processes:                                                                            |
+|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
+|        ID   ID                                                             Usage      |
+|=======================================================================================|
+|    0    1    0      46496      C   ...aforge/envs/etx4velo_new/bin/python      298MiB |
++---------------------------------------------------------------------------------------
+```
+The output should indicate a CUDA version of 11.8 or higher,
+which is necessary for compatibility with the latest PyTorch releases.
+
+Note that it's not required to manually install the CUDA toolkit as it will
+be included in the packages installed later on.
+
+## Install `mamba`
+
+```{info}
+If you already have `mamba`, you can skip this section.
+```
+
+We recommend using [mamba](https://mamba.readthedocs.io/en/latest/index.html)
+for package management due to its significantly faster performance compared to conda.
+
+```{info}
+If `conda` is already installed, you have the option to
+[install mamba in your base environment](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html#existing-conda-install-not-recommended).
+However, this approach is not generally recommended.
+
+If you prefer, you can continue using conda, though mamba offers a more efficient experience!
+```
+
+As of September 2023, mamba is automatically included with `miniforge`.
+Follow the [steps below](https://github.com/conda-forge/miniforge)
+to download and install the latest version of Miniforge, which bundles mamba:
+```bash
+curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+bash Miniforge3-$(uname)-$(uname -m).sh
+```
+
+During installation, you'll be prompted to update your shell profile
+to automatically initialize conda.
+Accepting this option will add the necessary initialization commands to your `~/.bashrc` file:
+```bash
+# >>> conda initialize >>>
+# !! Contents within this block are managed by 'conda init' !!
+__conda_setup="$('/home/{yourusername}/miniforge3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
+if [ $? -eq 0 ]; then
+    eval "$__conda_setup"
+else
+    if [ -f "/home/{yourusername}/miniforge3/etc/profile.d/conda.sh" ]; then
+        . "/home/{yourusername}/miniforge3/etc/profile.d/conda.sh"
+    else
+        export PATH="/home/{yourusername}/miniforge3/bin:$PATH"
+    fi
+fi
+unset __conda_setup
+
+if [ -f "/home/{yourusername}/miniforge3/etc/profile.d/mamba.sh" ]; then
+    . "/home/{yourusername}/miniforge3/etc/profile.d/mamba.sh"
+fi
+# <<< conda initialize <<<
+```
+(where `/home/{yourusername}/miniforge3` is the path to your `miniforge` installation)
+You may reopen your terminal for the changes to take effect.
+
+You can check that conda is installed by executing
+```bash
+$ conda
+usage: conda [-h] [-v] [--no-plugins] [-V] COMMAND ...
+
+conda is a tool for managing and deploying applications, environments and packages.
+
+options:
+  -h, --help          Show this help message and exit.
+  -v, --verbose       Can be used multiple times. Once for detailed output, twice for INFO logging,
+                      thrice for DEBUG logging, four times for TRACE logging.
+  --no-plugins        Disable all plugins that are not built into conda.
+  -V, --version       Show the conda version number and exit.
+
+commands:
+  The following built-in and plugins subcommands are available.
+
+  COMMAND
+    activate          Activate a conda environment.
+    clean             Remove unused packages and caches.
+    compare           Compare packages between conda environments.
+    config            Modify configuration values in .condarc.
+    create            Create a new conda environment from a list of specified packages.
+    deactivate        Deactivate the current active conda environment.
+    doctor            Display a health report for your environment.
+    env               See `conda env --help`.
+    info              Display information about current conda install.
+    init              Initialize conda for shell interaction.
+    install           Install a list of packages into a specified conda environment.
+    list              List installed packages in a conda environment.
+    notices           Retrieve latest channel notifications.
+    package           Create low-level conda packages. (EXPERIMENTAL)
+    remove (uninstall)
+                      Remove a list of packages from a specified conda environment.
+    rename            Rename an existing environment.
+    repoquery         Advanced search for repodata.
+    run               Run an executable in a conda environment.
+    search            Search for packages and display associated information using the MatchSpec
+                      format.
+    update (upgrade)  Update conda packages to the latest compatible version.
+```
+
+
+## Clone the ETX4VELO repository
+
+Clone the [`ETX4VELO` repository](https://gitlab.cern.ch/gdl4hep/etx4velo/-/tree/dev)
+and switch to the `dev` branch.
+```bash
+git clone ssh://git@gitlab.cern.ch:7999/gdl4hep/etx4velo.git
+cd etx4velo
+git checkout dev
+```
+
+## Set up the conda environment
+
+The configuration for our project environment is stored within the `setup/` directory
+of the ETX4VELO repository.
+
+To create and install the environment,
+navigate to ETX4VELO's root directory and execute the following command:
+```bash
+conda create -f setup/gpu_environment_cuda118.yaml
+```
+We encourage you to review the `setup/gpu_environment_cuda118.yaml` environment file
+to understand the purpose behind each package included in this environment.
+
+To check the environment is working as expected, launch a Python shell
+and execute the following commands to ensure that the critical libraries,
+especially those requiring CUDA, are correctly loaded and functioning.
+```bash
+$ python3
+Python 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import torch
+>>> import faiss # check that CUDA-enabled libraries are loading correctly
+>>> import cudf  # check that CUDA-enabled libraries are loading correctly
+>>> torch.cuda.is_available()  # Confirm PyTorch recognizes your CUDA device
+True
+```
+To start using the environment, activate it with the following command:
+```bash
+conda activate etx4velo_env
+```
+
+```{info}
+For those without access to an NVIDIA GPU, a CPU-only version of the environment is available.
+You can create this alternative environment using:
+```bash
+conda create -f setup/gpu_environment_cpu.yaml
+```
+
+## Download Data
+
+The necessary data for this tutorial series is stored on the LHCb EOS space, at
+```
+/eos/lhcb/user/a/anthonyc/tracking/data
+```
+
+We will focus on downloading the following subsets:
+- Training data: `/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset`
+- Test data: `/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset`
+
+For those with access to LXPLUS, where the EOS system is mounted,
+you can download the data directly to your desired directory using `rsync` (or `scp`):
+```bash
+# Navigate to your data directory
+cd <datadir>
+
+# Download the training data subset
+rsync -arv <username>@lxplus.cern.ch:/eos/lhcb/user/a/anthonyc/tracking/data/csv/v2.4/minbias-sim10b-xdigi_subset .
+
+# Download and extract the test samples
+rsync -arv --progress <username>@lxplus.cern.ch:/eos/lhcb/user/a/anthonyc/tracking/data/data_validation/v2.4/reference_samples_tutorial.tar.lz4 .
+lz4 -d reference_samples_tutorial.tar.lz4 -c | tar xvf - # untar archive
+mv reference_samples_tutorial reference_samples # rename
+rm reference_samples_tutorial.tar.lz4 # delete archive
+```
+Replace `<datadir>` with your target directory path and `<username>` with your LXPLUS username.
+
+```{info}
+Using the XRootD client provides an efficient alternative for downloading files
+from EOS.
+This method requires the installation of both the Kerberos client
+and the XRootD client on your machine.
+
+For detailed instructions on setting up and using the XRootD client,
+refer to [this guide](https://xdigi2csv.docs.cern.ch/master/Access/2.download_csv.html#using-a-xrootd-client-recommended).
+```
-- 
GitLab


From 37efe8eac101d167f9f32774690990b7d9833d04 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:48:08 +0100
Subject: [PATCH 15/17] Move tutorial to readme folder

---
 {tutorial => readme/tutorial}/00_introduction.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {tutorial => readme/tutorial}/00_introduction.md (100%)

diff --git a/tutorial/00_introduction.md b/readme/tutorial/00_introduction.md
similarity index 100%
rename from tutorial/00_introduction.md
rename to readme/tutorial/00_introduction.md
-- 
GitLab


From 2049d7d6a3c078ceda552b7acc96db7ddaa35d8d Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:49:59 +0100
Subject: [PATCH 16/17] Add tutorial series to doc

---
 docs/source/index.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/source/index.rst b/docs/source/index.rst
index 65da6700..bd03d136 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -59,6 +59,14 @@ The trained weights of the various models can be found `here <https://cernbox.ce
    guide/*
 
 
+.. toctree::
+   :caption: Tutorial Series
+   :glob:
+   :hidden:
+
+   tutorial/*
+
+
 .. toctree::
    :caption: APIDOC
    :hidden:
-- 
GitLab


From eda2e9e775afb7d8e3fe4d761428c6b09c207b45 Mon Sep 17 00:00:00 2001
From: Anthony Correia <anthony.correia@cern.ch>
Date: Tue, 6 Feb 2024 15:59:39 +0100
Subject: [PATCH 17/17] Properly use mamba instead of conda!

---
 readme/tutorial/00_introduction.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/readme/tutorial/00_introduction.md b/readme/tutorial/00_introduction.md
index 569d2495..fc2bdf78 100644
--- a/readme/tutorial/00_introduction.md
+++ b/readme/tutorial/00_introduction.md
@@ -172,8 +172,11 @@ of the ETX4VELO repository.
 To create and install the environment,
 navigate to ETX4VELO's root directory and execute the following command:
 ```bash
-conda create -f setup/gpu_environment_cuda118.yaml
+mamba create -f setup/gpu_environment_cuda118.yaml
 ```
+If you get an error such as `The following package could not be installed`,
+try to replace `mamba create` by `mamba env create`. 
+
 We encourage you to review the `setup/gpu_environment_cuda118.yaml` environment file
 to understand the purpose behind each package included in this environment.
 
-- 
GitLab