Commit 48f199db authored by Eric Cano's avatar Eric Cano
Browse files

bug #72215: Remove the TAPEGATEWAYREQUEST table from the schema, and drop the...

bug #72215: Remove the TAPEGATEWAYREQUEST table from the schema, and drop the triggers that populate it.

Added tests for checksum, recalls, size. Not complete yet.
parent e3f1d87a
......@@ -185,6 +185,15 @@ sub check_migrated ( $ )
}
}
# Returns true if the file is recalled
sub check_recalled ( $ )
{
my $file_name = shift;
my $stager_qry=`stager_qry -M $file_name`;
return $stager_qry=~ /STAGED/;
}
# Check fileservers ready with timeout
sub poll_fileserver_readyness ( $$ )
{
......@@ -280,7 +289,7 @@ sub make_localfile ( $$ )
}
# remote files statuses:
# rfcped => migrated
# rfcped => migrated, "on tape", "being recalled" => staged
# rfcp file to either single or dual copy directory and push all the
# know properties of the local file to the remote file created for the occasion
......@@ -315,10 +324,50 @@ sub rfcp_localfile ( $$ )
}
}
# Remove staged files from the stager so they can be recalled.
sub cleanup_migrated ()
{
for ( my $i =0; $i < scalar (@remote_files); $i++ ) {
my %f = %{$remote_files[$i]};
if ( $f{type} eq "file" && $f{status} eq "migrated" ) {
`su $environment{username} -c \"stager_rm -M $f{name}\"`;
$remote_files[$i]->{status} = "on tape";
print "t=".elapsed_time()."s. Removed $f{name} (was migrated) from stager.\n";
}
}
}
# Stager_get file
sub stager_reget_from_tape ()
{
for ( my $i =0; $i < scalar (@remote_files); $i++ ) {
my %f = %{$remote_files[$i]};
if ( $f{type} eq "file" && $f{status} eq "migrated" ) {
`su $environment{username} -c \"stager_rm -M $f{name}\"`;
`su $environment{username} -c \"stager_get -M $f{name}\"`;
$remote_files[$i]->{status} = "being recalled";
print "t=".elapsed_time()."s. Initiated recall for $f{name}.\n";
}
}
}
#
# rfcp file, but give it a special traetement to asses the behaviour of the migration with bad/broken files
sub rfcp_localfile_break ( $$ )
{
my $breaking_type = shift;
die "TODO";
if ($breaking_type eq "missing castorfile") {
}
}
# Check remote entries: check presence (should be always true) and then status of all files listed in remote files list
# returns true is any file has changed status, allowig a caller function to tiem out on "nothing moves anymore"
sub check_remote_entries ()
{
my $dbh = shift;
my $changed_entries = 0;
for my $i ( 0 .. scalar (@remote_files) - 1 ) {
my %entry = %{$remote_files[$i]};
......@@ -327,26 +376,101 @@ sub check_remote_entries ()
if ( $nslsresult =~ /No such file or directory$/) {
die "Entry not found in name server: $entry{name}";
}
# if it's a file, check the migration status
undef $nslsresult;
# if it's a file we can do something.
if ( $entry{type} eq "file" ) {
# check the migration status and compare checksums.
if ( $entry{status} eq "rfcped" ) {
if ( check_migrated ( $entry{name} ) ) {
if ( check_migrated ( $entry{name} ) ) {
# Report the newly detected migration.
print "t=".elapsed_time()."s. File ".$entry{name}." now migrated to tape.\n";
$remote_files[$i]->{status} = "migrated";
# Validate the checksum.
my $remote_checksum_string = `su $environment{username} -c \"nsls --checksum $entry{name}\"`;
chomp $remote_checksum_string;
my ( $remote_checksum, $local_checksum );
if ( $remote_checksum_string =~ /AD\s+([[:xdigit:]]+)\s/ ) {
$remote_checksum = $1;
} else {
print "ERROR: Failed to interpret remote checksum: $remote_checksum_string.\n";
}
if ( $entry{adler32} =~ /adler32\(.*\) = \d+\, 0x([[:xdigit:]]+)/ ) {
$local_checksum = $1;
} else {
print "ERROR: Failed to interpret locally stored checksum: $entry{adler32}.\n";
}
if ( $local_checksum ne $remote_checksum ) {
print "ERROR: checksum mismatch beween remote and locally stored for $entry{name}: $local_checksum != $remote_checksum\n";
}
# Check that dual tape copies got mirgated as expected (and on different tapes)
$nslsresult = `su $environment{username} -c \"nsls --class $entry{name}\"`;
if ( $nslsresult =~ /^\s+(\d+)\s+\/castor/ ) {
my $class = $1;
my $nslistclass_result = `su $environment{username} -c \"nslistclass --id=$class\"`;
if ( $nslistclass_result =~ /NBCOPIES\s+(\d+)/ ) {
my $expected_copynb = $1;
my $found_copynb = `su $environment{username} -c \"nsls -T $entry{name} 2>/dev/null | wc -l\"`;
if ( $expected_copynb != $found_copynb ) {
print "ERROR: Unexpected number of copies for $entry{name}. Expected: $expected_copynb, found:$found_copynb\n";
print "stager record: ".`su $environment{username} -c \"stager_qry -M $entry{name}\"`;
print "TODO: handle this state as partially migrated\n";
}
} else {
print "ERROR: Failed to extract copynb for file class $class while processing file $entry{name}.\n";
}
} else {
print "ERROR: Failed to extract class for file ".$entry{name}."\n";
}
# Invalidate and recall the file
$changed_entries ++;
}
}
}
# check the recall status, rfcp in and compare checksums.
} elsif ($entry{status} eq "being recalled" ) {
if ( check_recalled ( $entry{name} ) ) {
print "t=".elapsed_time()."s. File ".$entry{name}." now recalled to disk.\n";
$remote_files[$i]->{status} eq "being recalled";
# rfcp in the file, check size and checksum.
my $local_copy=`mktemp`;
chomp $local_copy;
# hand over the file to the user
`chown $environment{username} $local_copy`;
# rfcp the recalled copy
`rfcp $entry{name} $local_copy`;
# computer checksum, get size and get rid of file
my $local_size = ( -s $local_copy );
my $local_checksum_string = `adler32 $local_copy`;
my $local_checksum;
if ( $local_checksum_string =~ /adler32\(.*\) = \d+\, 0x([[:xdigit:]]+)/ ) {
$local_checksum = $1;
}
my $stored_checksum;
if ( $entry{adler32} =~ /adler32\(.*\) = \d+\, 0x([[:xdigit:]]+)/ ) {
$stored_checksum = $1;
}
if ( $stored_checksum ne $local_checksum ) {
print "ERROR: checksum mismatch beween locally recalled file and stored value for $entry{name}: ".
"$local_checksum != $stored_checksum\n";
}
if ( $local_size != $entry{size} ) {
print "ERROR: size mismatch beween locally recalled file and stored value for $entry{name}: ".
"$local_size != ".$entry{size}."\n";
}
unlink $local_copy;
$changed_entries ++;
}
}
}
}
return $changed_entries;
}
# Get the number of non-migrated files
sub count_to_be_migrated ()
sub count_to_be_moved ()
{
my $ret = 0;
for ( @remote_files ) {
if ( ($_->{type} eq "file" ) && ($_ ->{status} eq "rfcped" ) ) {
if ( ($_->{type} eq "file" ) && ($_ ->{status} =~ /rfcped|being recalled/ ) ) {
$ret++;
}
}
......@@ -358,14 +482,15 @@ sub poll_moving_entries ( $$ )
{
my ( $poll_interval, $timeout ) = ( shift, shift );
my $starttime = `date +%s`;
while ( count_to_be_migrated() > 0 && ((`date +%s` - $starttime) < $timeout) ) {
while ( count_to_be_moved() > 0 && ((`date +%s` - $starttime) < $timeout) ) {
if ( check_remote_entries () ) {
print "t=".elapsed_time()."s. Saw at least one new migration...\n";
$starttime = `date +%s`;
}
sleep ( $poll_interval );
}
if (count_to_be_migrated() == 0 ) {
if (count_to_be_moved() == 0 ) {
print
return;
}
die "Timeout with ".count_to_be_migrated()." files to be migrated after $timeout s.";
......@@ -945,9 +1070,9 @@ sub check_leftovers ( $ )
}
$sth = $dbh -> prepare("SELECT count (*) from (
SELECT dc.id from diskcopy dc where
dc.status NOT IN ( 0 )
dc.status NOT IN ( 0, 7 )
UNION ALL
SELECT tc.id from tapecopy tc)");
SELECT tc.id from tapecopy tc)"); # Discopy_staged diskcopy_invalid
$sth -> execute ();
@row = $sth->fetchrow_array();
return $row[0];
......@@ -986,7 +1111,7 @@ sub print_leftovers ( $ )
FROM castorfile cf
LEFT OUTER JOIN diskcopy dc ON dc.castorfile = cf.id
LEFT OUTER JOIN tapecopy tc ON tc.castorfile = cf.id
WHERE dc.status NOT IN ( 0 )");
WHERE dc.status NOT IN ( 0, 7)"); # Discopy_staged diskcopy_invalid
$sth -> execute();
while ( my @row = $sth->fetchrow_array() ) {
nullize_arrays_undefs ( \@row );
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment