Skip to content

Fix handling of `UserSpecifiedANonExistentTape` inside `QueueCleanupRunner`

Problem

As demonstrated in #1034 (comment 8989166), the function RdbmsTapeCatalogue::modifyTapeStat() may throw an exception UserSpecifiedANonExistentTape inside the QueueCleanupRunner which causes the entire process to crash.

This exception may simply caused by another maintenance process doing a state change on the same tape, at the same time.

{"epoch_time":1737972756.175063849,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"ERROR","pid":373,"tid":373,"message":"In MaintenanceHandler::exceptionThrowingRunChild(): received an exception. Backtrace follows.","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected"}
{"epoch_time":1737972756.175145222,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":0,"traceFrame":"/lib64/libctacommon.so.0(cta::exception::Backtrace::Backtrace(bool)+0x6b) [0x7ff94ae383d1]"}
{"epoch_time":1737972756.175192213,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":1,"traceFrame":"/lib64/libctacommon.so.0(cta::exception::Exception::Exception(std::basic_string_view<char, std::char_traits<char> >, bool)+0x91) [0x7ff94ae393e9]"}
{"epoch_time":1737972756.175226788,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":2,"traceFrame":"/lib64/libctacommon.so.0(cta::exception::UserError::UserError(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x53) [0x7ff94ae3a8dd]"}
{"epoch_time":1737972756.175260295,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":3,"traceFrame":"/lib64/libctacatalogue.so.0(cta::catalogue::UserSpecifiedANonExistentTape::UserError(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x3e) [0x7ff95163a346]"}
{"epoch_time":1737972756.175296382,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":4,"traceFrame":"/lib64/libctacatalogue.so.0(cta::catalogue::RdbmsTapeCatalogue::modifyTapeState(cta::common::dataStructures::SecurityIdentity const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, cta::common::dataStructures::Tape::State const&, std::optional<cta::common::dataStructures::Tape::State> const&, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&)+0x764) [0x7ff95162cefa]"}
{"epoch_time":1737972756.175332600,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":5,"traceFrame":"/lib64/libctacatalogue.so.0(+0x3e72b2) [0x7ff9516a42b2]"}
{"epoch_time":1737972756.175365718,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":6,"traceFrame":"/lib64/libctacatalogue.so.0(+0x3ec8e1) [0x7ff9516a98e1]"}
{"epoch_time":1737972756.175437793,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":7,"traceFrame":"/lib64/libctacatalogue.so.0(cta::catalogue::TapeCatalogueRetryWrapper::modifyTapeState(cta::common::dataStructures::SecurityIdentity const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, cta::common::dataStructures::Tape::State const&, std::optional<cta::common::dataStructures::Tape::State> const&, std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&)+0x7d) [0x7ff9516a4347]"}
{"epoch_time":1737972756.175475387,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":8,"traceFrame":"/lib64/libctaobjectstore.so.0(cta::objectstore::QueueCleanupRunner::runOnePass(cta::log::LogContext&)+0x1154) [0x7ff95267c8e6]"}
{"epoch_time":1737972756.175506106,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":9,"traceFrame":"/usr/bin/cta-taped() [0x4df177]"}
{"epoch_time":1737972756.175559578,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":10,"traceFrame":"/usr/bin/cta-taped() [0x4deb1c]"}
{"epoch_time":1737972756.175588778,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":11,"traceFrame":"/usr/bin/cta-taped() [0x4e36b9]"}
{"epoch_time":1737972756.175617462,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":12,"traceFrame":"/usr/bin/cta-taped() [0x4e2663]"}
{"epoch_time":1737972756.175648731,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":13,"traceFrame":"/usr/bin/cta-taped() [0x49c9a0]"}
{"epoch_time":1737972756.175677623,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":14,"traceFrame":"/usr/bin/cta-taped() [0x49c6e0]"}
{"epoch_time":1737972756.175706132,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":15,"traceFrame":"/usr/bin/cta-taped() [0x49c1a1]"}
{"epoch_time":1737972756.175754054,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":16,"traceFrame":"/usr/bin/cta-taped() [0x486f05]"}
{"epoch_time":1737972756.175784682,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":17,"traceFrame":"/usr/bin/cta-taped() [0x487ab8]"}
{"epoch_time":1737972756.175813470,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":18,"traceFrame":"/lib64/libc.so.6(+0x295d0) [0x7ff94a0b25d0]"}
{"epoch_time":1737972756.175842450,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":19,"traceFrame":"/lib64/libc.so.6(__libc_start_main+0x80) [0x7ff94a0b2680]"}
{"epoch_time":1737972756.175872275,"local_time":"2025-01-27T11:12:36+0100","hostname":"cta-tpsrv02-0","program":"cta-taped","log_level":"INFO","pid":373,"tid":373,"message":"Stack trace","drive_name":"VDSTK02","instance":"CI","sched_backend":"ceph","SubprocessName":"maintenanceHandler","exceptionMessage":"Cannot modify the state of the tape V00101 because it does not exist or because a recent state change has been detected","traceFrameNumber":20,"traceFrame":"/usr/bin/cta-taped() [0x486975]"}