diff --git a/Core/include/Acts/Vertexing/AMVFInfo.hpp b/Core/include/Acts/Vertexing/AMVFInfo.hpp
index 74297bdc8443853a9519843ae66b0547236d1b8c..8174a09dba79814929c0aacf15178cb2a7ee7817 100644
--- a/Core/include/Acts/Vertexing/AMVFInfo.hpp
+++ b/Core/include/Acts/Vertexing/AMVFInfo.hpp
@@ -44,8 +44,7 @@ struct VertexInfo {
   // Vector of all track currently held by vertex
   std::vector<const input_track_t*> trackLinks;
 
-  std::map<const input_track_t*, std::unique_ptr<const BoundParameters>>
-      ip3dParams;
+  std::map<const input_track_t*, const BoundParameters> ip3dParams;
 };
 
 }  // namespace Acts
\ No newline at end of file
diff --git a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
index 2eada8b97b8f4f8c4384bfe0b61f2b5fa827c202..0656a2b91921ea573b51d1d8e63b319068906448 100644
--- a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
+++ b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.hpp
@@ -141,6 +141,13 @@ class AdaptiveMultiVertexFinder {
     // constraint is provided
     std::pair<double, double> defaultConstrFitQuality{0., -3.};
 
+    // Do an adaptive multi vertex fit after
+    // a bad vertex was removed.
+    // If false, the old fitter state is just copied,
+    // this should give the same results with better
+    // performance. To be further investigated.
+    bool refitAfterBadVertex = true;
+
   };  // Config struct
 
   struct State {};
diff --git a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
index 642d8b5a3a4e338fcc1c1c37ffe46f9e01fe1d45..b9822c1d47687fcfdd8f139cca0a780b42061561 100644
--- a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
+++ b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFinder.ipp
@@ -38,6 +38,8 @@ auto Acts::AdaptiveMultiVertexFinder<vfitter_t, sfinder_t>::find(
   while (((m_cfg.addSingleTrackVertices && seedTracks.size() > 0) ||
           ((!m_cfg.addSingleTrackVertices) && seedTracks.size() > 1)) &&
          iteration < m_cfg.maxIterations) {
+    auto oldFitterState = fitterState;
+
     // Tracks that are used for searching compatible tracks
     // near a vertex candidate
     std::vector<const InputTrack_t*> myTracks;
@@ -121,17 +123,32 @@ auto Acts::AdaptiveMultiVertexFinder<vfitter_t, sfinder_t>::find(
     if (not keepVertex) {
       allVertices.pop_back();
       allVerticesPtr.pop_back();
-      // Update fitter state with removed vertex candidate
-      fitterState.updateTrkToVerticesMultiMap(allVerticesPtr);
 
-      // TODO: clean tracksAtVerticesMap maybe here? i.e. remove all entries
-      // with old vertex?
+      if (!m_cfg.refitAfterBadVertex) {
+        fitterState.vertexCollection = oldFitterState.vertexCollection;
+        fitterState.annealingState = oldFitterState.annealingState;
+        fitterState.vtxInfoMap.clear();
+        for (const auto& vtx : allVerticesPtr) {
+          fitterState.vtxInfoMap.insert(
+              std::make_pair(vtx, oldFitterState.vtxInfoMap[vtx]));
+        }
+        fitterState.trackToVerticesMultiMap =
+            oldFitterState.trackToVerticesMultiMap;
+        fitterState.tracksAtVerticesMap = oldFitterState.tracksAtVerticesMap;
+
+      } else {
+        // Update fitter state with removed vertex candidate
+        fitterState.updateTrkToVerticesMultiMap(allVerticesPtr);
+
+        // TODO: clean tracksAtVerticesMap maybe here? i.e. remove all entries
+        // with old vertex?
 
-      // Do the fit with removed vertex
-      auto fitResult = m_cfg.vertexFitter.fit(fitterState, allVerticesPtr,
-                                              m_cfg.linearizer, vFitterOptions);
-      if (!fitResult.ok()) {
-        return fitResult.error();
+        // Do the fit with removed vertex
+        auto fitResult = m_cfg.vertexFitter.fit(
+            fitterState, allVerticesPtr, m_cfg.linearizer, vFitterOptions);
+        if (!fitResult.ok()) {
+          return fitResult.error();
+        }
       }
     }
     iteration++;
diff --git a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFitter.ipp b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFitter.ipp
index 819c1a9b40d45b65c10d884dc52d9f9af23433ff..38828343570f84577b4b99d463ef50718562d0ec 100644
--- a/Core/include/Acts/Vertexing/AdaptiveMultiVertexFitter.ipp
+++ b/Core/include/Acts/Vertexing/AdaptiveMultiVertexFitter.ipp
@@ -223,8 +223,7 @@ Acts::Result<void> Acts::
       return res.error();
     }
     // Set ip3dParams for current trackAtVertex
-    currentVtxInfo.ip3dParams.insert(
-        std::make_pair(trk, std::move(res.value())));
+    currentVtxInfo.ip3dParams.insert(std::make_pair(trk, *(res.value())));
   }
   return {};
 }
@@ -254,11 +253,11 @@ Acts::AdaptiveMultiVertexFitter<input_track_t, linearizer_t>::
       // Set ip3dParams for current trackAtVertex
       auto value = std::move(res.value());
 
-      currentVtxInfo.ip3dParams.insert(std::make_pair(trk, std::move(value)));
+      currentVtxInfo.ip3dParams.insert(std::make_pair(trk, *value));
     }
     // Set compatibility with current vertex
     auto compRes = m_cfg.ipEst.getVertexCompatibility(
-        geoContext, currentVtxInfo.ip3dParams.at(trk).get(),
+        geoContext, &(currentVtxInfo.ip3dParams.at(trk)),
         VectorHelpers::position(currentVtxInfo.oldPosition));
     if (!compRes.ok()) {
       return compRes.error();
@@ -304,8 +303,8 @@ Acts::Result<void> Acts::AdaptiveMultiVertexFitter<
         }
         // Update the vertex with the new track
         auto updateRes =
-            KalmanVertexUpdater::updateVertexWithTrack<input_track_t>(
-                vtx, trkAtVtx);
+            KalmanVertexUpdater::updateVertexWithTrack<input_track_t>(vtx,
+                                                                      trkAtVtx);
         if (!updateRes.ok()) {
           return updateRes.error();
         }