Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
scdaq
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
scouting-demonstrator
scdaq
Commits
7858c68b
Commit
7858c68b
authored
6 months ago
by
Giovanna Lazzari Miotto
Browse files
Options
Downloads
Patches
Plain Diff
fix: Access, dereference unique pointers correctly
parent
f1ada1a9
No related branches found
Branches containing commit
No related tags found
Tags containing commit
1 merge request
!109
Introduce pipeline synchronization, output template and parallel sink
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
src/OutputFile.h
+11
-13
11 additions, 13 deletions
src/OutputFile.h
src/OutputFileHandler.cc
+13
-12
13 additions, 12 deletions
src/OutputFileHandler.cc
src/sink.cc
+2
-10
2 additions, 10 deletions
src/sink.cc
with
26 additions
and
35 deletions
src/OutputFile.h
+
11
−
13
View file @
7858c68b
...
@@ -51,7 +51,9 @@ struct LumisectionMetadata {
...
@@ -51,7 +51,9 @@ struct LumisectionMetadata {
size_t
content_size
{};
size_t
content_size
{};
uint32_t
num_orbits
{};
uint32_t
num_orbits
{};
uint32_t
num_files
{};
uint32_t
num_files
{};
uint32_t
max_index
;
uint32_t
max_index
{};
LumisectionMetadata
()
=
default
;
// Max index per lumisection should only be enforced if CMSSW headers are enabled
// Max index per lumisection should only be enforced if CMSSW headers are enabled
explicit
LumisectionMetadata
(
uint32_t
max_index_per_ls
)
:
max_index
(
max_index_per_ls
)
{}
explicit
LumisectionMetadata
(
uint32_t
max_index_per_ls
)
:
max_index
(
max_index_per_ls
)
{}
...
@@ -62,7 +64,7 @@ struct LumisectionMetadata {
...
@@ -62,7 +64,7 @@ struct LumisectionMetadata {
<<
", LS="
<<
lumisection
<<
" ==> "
<<
new_ls
;
<<
", LS="
<<
lumisection
<<
" ==> "
<<
new_ls
;
// Lumisection should only ever change after structure has been reset
// Lumisection should only ever change after structure has been reset
if
(
lumisection
!=
0
&&
lumisection
!=
new_ls
)
{
if
(
lumisection
!=
0
&&
lumisection
!=
new_ls
)
{
LOG
(
INFO
)
<<
"
L
umisection s
kipped
s
o
me
steps
"
;
LOG
(
INFO
)
<<
"
NOTE: l
umisection s
hould remain the
s
a
me
unless reset
"
;
}
}
lumisection
=
new_ls
;
lumisection
=
new_ls
;
auto
new_file_idx
=
global_index
%
(
max_index
+
1
);
auto
new_file_idx
=
global_index
%
(
max_index
+
1
);
...
@@ -108,16 +110,14 @@ class Output {
...
@@ -108,16 +110,14 @@ class Output {
:
md_
(
std
::
move
(
md
)),
run_metrics_
(
run_ptr
)
{};
:
md_
(
std
::
move
(
md
)),
run_metrics_
(
run_ptr
)
{};
Output
()
=
default
;
Output
()
=
default
;
~
Output
()
{
~
Output
()
{}
if
(
bool
(
ls_footer_
))
free
(
ls_footer_
);
}
auto
GetHandle
()
const
{
return
handle_
;
}
auto
GetHandle
()
const
{
return
handle_
;
}
auto
GetMetadata
()
const
{
return
md_
;
}
auto
GetMetadata
()
const
{
return
md_
;
}
auto
GetLumisection
()
const
{
return
md_
.
lumisection
;
}
auto
GetLumisection
()
const
{
return
md_
.
lumisection
;
}
auto
GetLumisectionFooter
()
const
{
return
*
ls_footer_
;
}
auto
GetLumisectionFooter
()
const
{
return
ls_footer_
;
}
auto
HasPayload
()
const
{
return
md_
.
size
>
sizeof
(
HeaderType
);
}
auto
HasPayload
()
const
{
return
md_
.
size
>
sizeof
(
HeaderType
);
}
auto
HasLumisectionFooter
()
const
{
return
bool
(
ls_footer_
)
;
}
auto
HasLumisectionFooter
()
const
{
return
ls_footer_
.
lumisection
!=
0
;
}
auto
HasHeader
()
const
{
return
header_
.
has_value
();
};
auto
HasHeader
()
const
{
return
header_
.
has_value
();
};
auto
GetHeader
()
const
->
std
::
optional
<
HeaderType
>
{
return
header_
;
}
auto
GetHeader
()
const
->
std
::
optional
<
HeaderType
>
{
return
header_
;
}
...
@@ -125,9 +125,9 @@ class Output {
...
@@ -125,9 +125,9 @@ class Output {
header_
=
HeaderType
(
md_
.
source_id
,
md_
.
num_orbits
,
md_
.
run_number
,
md_
.
lumisection
,
md_
.
size
);
header_
=
HeaderType
(
md_
.
source_id
,
md_
.
num_orbits
,
md_
.
run_number
,
md_
.
lumisection
,
md_
.
size
);
}
}
void
SetLumisectionFooter
(
std
::
unique_ptr
<
Detail
::
LumisectionMetadata
>&&
ls
)
{
void
SetLumisectionFooter
(
Detail
::
LumisectionMetadata
ls
)
{
// The last file in a lumisection carries an LS metadata footer
// The last file in a lumisection carries an LS metadata footer
ls_footer_
=
ls
.
release
()
;
ls_footer_
=
ls
;
}
}
virtual
void
ReserveHeader
()
=
0
;
virtual
void
ReserveHeader
()
=
0
;
...
@@ -149,7 +149,7 @@ class Output {
...
@@ -149,7 +149,7 @@ class Output {
Detail
::
RunMetadata
*
run_metrics_
{
nullptr
};
Detail
::
RunMetadata
*
run_metrics_
{
nullptr
};
protected
:
protected
:
Detail
::
LumisectionMetadata
*
ls_footer_
{
nullptr
};
Detail
::
LumisectionMetadata
ls_footer_
{};
};
};
class
FileOutput
:
public
Output
{
class
FileOutput
:
public
Output
{
...
@@ -172,9 +172,7 @@ class FileOutput : public Output {
...
@@ -172,9 +172,7 @@ class FileOutput : public Output {
FileOutput
()
=
default
;
FileOutput
()
=
default
;
~
FileOutput
()
{
~
FileOutput
()
{}
if
(
bool
(
ls_footer_
))
free
(
ls_footer_
);
}
std
::
string
GetIdentifier
()
const
{
return
filename_
;
};
std
::
string
GetIdentifier
()
const
{
return
filename_
;
};
...
...
This diff is collapsed.
Click to expand it.
src/OutputFileHandler.cc
+
13
−
12
View file @
7858c68b
...
@@ -50,13 +50,12 @@ void OutputFileHandler::UpdateRunInfo(uint32_t next_run, uint32_t next_index) {
...
@@ -50,13 +50,12 @@ void OutputFileHandler::UpdateRunInfo(uint32_t next_run, uint32_t next_index) {
}
}
void
OutputFileHandler
::
CommitFile
()
{
void
OutputFileHandler
::
CommitFile
()
{
ls_
->
AddFileMetadata
(
outputFile_
.
GetMetadata
());
Detail
::
FileMetadata
file_meta
=
outputFile_
.
GetMetadata
();
Detail
::
FileMetadata
file_meta
=
outputFile_
.
GetMetadata
();
ls_
->
AddFileMetadata
(
file_meta
);
if
(
ls_
->
IsLastIndex
(
file_meta
.
index_in_ls
)
&&
IsMainPipeline
()
&&
HasCmsswHeaders
())
{
if
(
ls_
->
IsLastIndex
(
file_meta
.
index_in_ls
)
&&
IsMainPipeline
()
&&
HasCmsswHeaders
())
{
// If last in lumisection and using CMSSW header and is the main pipeline
// If last in lumisection and using CMSSW header and is the main pipeline
LOG
(
INFO
)
<<
"Last file in lumisection; handing over LS metadata footer"
;
LOG
(
INFO
)
<<
"Last file in lumisection "
<<
ls_
->
lumisection
<<
", handing over metadata footer and resetting."
;
outputFile_
.
SetLumisectionFooter
(
std
::
move
(
ls_
));
outputFile_
.
SetLumisectionFooter
(
*
ls_
);
LOG
(
INFO
)
<<
"Re-instantiating LumisectionMetadata"
;
ls_
=
std
::
make_unique
<
Detail
::
LumisectionMetadata
>
(
GetMaxFileIndexPerLumisection
());
ls_
=
std
::
make_unique
<
Detail
::
LumisectionMetadata
>
(
GetMaxFileIndexPerLumisection
());
}
}
...
@@ -69,10 +68,10 @@ int OutputFileHandler::StageSlice(const char *buffer, size_t size_bytes, uint32_
...
@@ -69,10 +68,10 @@ int OutputFileHandler::StageSlice(const char *buffer, size_t size_bytes, uint32_
bool
is_new_index
=
(
current_global_index_
!=
static_cast
<
int
>
(
file_index
));
bool
is_new_index
=
(
current_global_index_
!=
static_cast
<
int
>
(
file_index
));
if
(
is_new_run
||
is_new_index
)
{
if
(
is_new_run
||
is_new_index
)
{
LOG
(
INFO
)
<<
"Committing file because ... new_run? "
<<
is_new_run
<<
", new_index? "
//
LOG(INFO) << "Committing file because ... new_run? " << is_new_run << ", new_index? "
<<
is_new_index
<<
"(context: current_global_index_="
<<
current_global_index_
//
<< is_new_index << "(context: current_global_index_=" << current_global_index_
<<
", incoming_file_index="
<<
file_index
<<
", current_run="
<<
run_
.
number
//
<< ", incoming_file_index=" << file_index << ", current_run=" << run_.number
<<
", incoming_run="
<<
run_number
<<
")"
;
//
<< ", incoming_run=" << run_number << ")";
// Commit current file and start a new one
// Commit current file and start a new one
CommitFile
();
CommitFile
();
UpdateRunInfo
(
run_number
,
file_index
);
UpdateRunInfo
(
run_number
,
file_index
);
...
@@ -124,10 +123,12 @@ void OutputFileHandler::CommitRun() {
...
@@ -124,10 +123,12 @@ void OutputFileHandler::CommitRun() {
LOG
(
INFO
)
<<
"Lumi "
<<
std
::
to_string
(
ls_index
)
LOG
(
INFO
)
<<
"Lumi "
<<
std
::
to_string
(
ls_index
)
<<
". Assert that lumisection pointer exists: "
<<
bool
(
ls_
);
<<
". Assert that lumisection pointer exists: "
<<
bool
(
ls_
);
if
(
ls_
)
{
if
(
ls_
)
{
auto
lumi_number
=
ls_
->
lumisection
;
auto
ls_index_footer
=
ls_
->
lumisection
;
LOG
(
INFO
)
<<
"Lumi in LS struct: "
<<
lumi_number
;
if
(
ls_index_footer
!=
ls_index
)
{
LOG
(
INFO
)
<<
"Lumi as function of global file index="
<<
current_global_index_
<<
" ==> "
<<
ls_index
;
LOG
(
WARNING
)
<<
"Expected lumisection number "
<<
ls_index
<<
", but lumisection footer has "
<<
ls_index_footer
;
assert
(
lumi_number
==
ls_index
);
}
assert
(
ls_index_footer
==
ls_index
);
if
(
IsMainPipeline
())
{
if
(
IsMainPipeline
())
{
LOG
(
INFO
)
<<
"Committing lumi EOLS before EoR can be written"
;
LOG
(
INFO
)
<<
"Committing lumi EOLS before EoR can be written"
;
...
...
This diff is collapsed.
Click to expand it.
src/sink.cc
+
2
−
10
View file @
7858c68b
...
@@ -42,18 +42,10 @@ void Sink<TOutput>::CommitOutput(TOutput &f) {
...
@@ -42,18 +42,10 @@ void Sink<TOutput>::CommitOutput(TOutput &f) {
Detail
::
LumisectionMetadata
ls_meta
=
file
.
GetLumisectionFooter
();
Detail
::
LumisectionMetadata
ls_meta
=
file
.
GetLumisectionFooter
();
Detail
::
FileMetadata
file_meta
=
file
.
GetMetadata
();
Detail
::
FileMetadata
file_meta
=
file
.
GetMetadata
();
auto
run_number
=
file_meta
.
run_number
;
auto
run_number
=
file_meta
.
run_number
;
LOG
(
INFO
)
<<
"Writing scheduled Lumifooter now, ls_meta says="
<<
ls_meta
.
lumisection
LOG
(
INFO
)
<<
"Writing scheduled lumifooter for lumisection="
<<
ls_meta
.
lumisection
;
<<
" and file says="
<<
file_meta
.
lumisection
;
assert
(
ls_meta
.
lumisection
==
file_meta
.
lumisection
);
if
(
ls_meta
.
lumisection
==
file_meta
.
lumisection
)
{
LOG
(
INFO
)
<<
"Lumisection values match btw"
;
}
else
{
LOG
(
INFO
)
<<
"Oh no, values are different"
;
}
ls_meta
.
lumisection
=
file_meta
.
lumisection
;
WriteLumisectionFooter
(
run_number
,
ls_meta
);
WriteLumisectionFooter
(
run_number
,
ls_meta
);
LOG
(
INFO
)
<<
"Lumifooter written! "
<<
std
::
to_string
(
ls_meta
.
lumisection
);
file
.
run_metrics_
->
AddLumisectionMetadata
(
ls_meta
);
file
.
run_metrics_
->
AddLumisectionMetadata
(
ls_meta
);
LOG
(
INFO
)
<<
"Lumi metadata has been accumulated"
;
}
}
}
else
if
constexpr
(
std
::
is_same
<
TOutput
,
DaxOutput
>
())
{
}
else
if
constexpr
(
std
::
is_same
<
TOutput
,
DaxOutput
>
())
{
LOG
(
TRACE
)
<<
"Committing current DAX buffer"
;
LOG
(
TRACE
)
<<
"Committing current DAX buffer"
;
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment