Commit 9a563ef6 authored by Eric Cano's avatar Eric Cano
Browse files

bug #72215: Remove the TAPEGATEWAYREQUEST table from the schema, and drop the...

bug #72215: Remove the TAPEGATEWAYREQUEST table from the schema, and drop the triggers that populate it.

Made the scripts more generic so that they can be used on different boxes.
parent 6a4459d7
......@@ -42,6 +42,7 @@ our @ISA = qw(Exporter);
our @export = qw(
read_config
check_environment
get_environment
make_seed
make_localfile
open_db
......@@ -71,6 +72,7 @@ our @export = qw(
nullize_arrays_undefs
print_leftovers
getOrastagerconfigParam
cleanup
); # Symbols to autoexport (:DEFAULT tag)
# keep track of the locally created files and of the recalled and locally rfcped files.
......@@ -91,9 +93,10 @@ sub read_config ( $ )
my $local_host = `hostname -s`;
chomp $local_host;
$environment{hostname} = $local_host;
my @per_host_vars = ( 'username', 'checkout_location');
my @per_host_vars = ( 'username', 'checkout_location', 'file_size', 'file_number', 'castor_directory');
my @global_vars = ( 'dbDir' , 'tstDir', 'stageUid', 'stageGid', 'adminList',
'originalDbSchema', 'originalDropSchema');
'originalDbSchema', 'originalDropSchema', 'castor_single_subdirectory',
'castor_dual_subdirectory' );
for my $i ( @per_host_vars ) {
$environment{$i}=getConfParam('TAPETEST', $i.'_'.$local_host, $config_file );
}
......@@ -119,7 +122,17 @@ sub check_environment ( )
die "Necessary file or directory not found";
}
}
# Extract enviroment variable (mainly for client application)
sub get_environement ( $ )
{
my $vname = shift;
if ( defined $environment{$vname} ) {
return $environment{$vname}
} else {
die "Variable $vname not found in environment";
}
}
# create a local seed file, returning the index to the file.
# Take 1 parameter: the size in
......@@ -139,12 +152,12 @@ sub make_seed ( $ )
my $starttime = `date +%s`;
if ($megs != 0) {
`dd if=/dev/urandom of=$file_name bs=1M count=$megs`;
$kflags=" oflags=append";
$bflags=" oflags=append";
$kflags=" oflag=append conv=notrunc";
$bflags=" oflag=append conv=notrunc";
}
if ($kilos != 0) {
`dd if=/dev/urandom of=$file_name bs=1K count=$kilos $kflags`;
$bflags=" oflags=append";
$bflags=" oflag=append conv=notrunc";
}
if ($bytes != 0) {
`dd if=/dev/urandom of=$file_name bs=1 count=$bytes $bflags`;
......@@ -165,7 +178,7 @@ sub make_seed ( $ )
print `chown $environment{'username'} $file_name`;
# We call it good enough
my $checksum = `adler32 $file_name`;
my $checksum = `adler32 $file_name 2>&1`;
chomp ($checksum);
my %file_entry = ( 'name' => $file_name,
'size' => $size,
......@@ -186,13 +199,17 @@ sub make_localfile ( $$ )
open NEW_FILE, "> $file_name" or die "In $package_name::make_localfile: failed to open $file_name for writing: $!";
print NEW_FILE "$file_name";
close NEW_FILE;
`dd if=$local_files[$seed_index]->{'name'} of=$file_name bs=1M`;
my $checksum = `adler32 $file_name`;
`dd if=$local_files[$seed_index]->{'name'} of=$file_name bs=1M oflag=append conv=notrunc`;
my $checksum = `adler32 $file_name 2>&1`;
chomp ($checksum);
my $size = ( -s $file_name );
my %file_entry = ( 'name' => $file_name,
'size' => $size,( -s $file_name ),
'size' => $size,
'adler32' => $checksum);
my $file_index = push @local_files, \%file_entry;
$file_index --;
return $file_index;
}
# Returns the value of the specified ORASTAGERCONFIG parameter.
......@@ -212,6 +229,7 @@ sub getOrastagerconfigParam ( $ )
while(<CONFIG>) {
chomp;
if(m/^DbCnvSvc\s+$paramName\s+(\w+)/) {
close CONFIG;
return $1;
}
}
......@@ -949,6 +967,128 @@ sub wipeAndRecreateDefaultLxcastodev03 ()
undef $dbh;
}
# Wipes and recreates the DB for lxcastordev04 (with a check to prevent wiping of unmigrated files
sub wipeAndRecreateDefaultLxcastodev04 ()
{
die "Please review and fix wipeAndRecreateDefaultLxcastodev04";
# Print error message and abort if the user is not root
my $uid = POSIX::getuid;
my $gid = POSIX::getgid;
if($uid != 0 || $gid !=0) {
print("ABORT: This script must be ran as root\n");
exit(-1);
}
# Make sure we're running on the proper machine.
my $host = `uname -n`;
if ( ! $host =~ /^lxcastordev04($|\.)/i ) {
print('ABORT: This script is only made to be run on host lxcastordev04\n');
exit -1;
}
# Ensure all of the daemons accessing the stager-database are dead
killDaemonWithTimeout('jobmanagerd' , 2);
killDaemonWithTimeout('mighunterd' , 2);
killDaemonWithTimeout('rechandlerd' , 2);
killDaemonWithTimeout('rhd' , 2);
killDaemonWithTimeout('rmmasterd' , 2);
killDaemonWithTimeout('rtcpclientd' , 2);
killDaemonWithTimeout('stagerd' , 2);
killDaemonWithTimeout('tapegatewayd', 2);
# Ensure there is no leftover in the DB
my $dbh = open_db ();
if ( check_leftovers ( $dbh ) ) {
print_leftovers ( $dbh );
$dbh->disconnect();
die ("Found leftovers in the stager's DB. Stopping here.");
}
my $checkout_location = $environment{checkout_location};
my $dbDir = $environment{dbDir};
my $originalDropSchema = $environment{originalDropSchema};
my $originalDbSchema = $environment{originalDbSchema};
my $originalDropSchemaFullpath=$checkout_location.'/'.$dbDir.'/'.$originalDropSchema;
my $originalDbSchemaFullpath=$checkout_location.'/'.$dbDir.'/'.$originalDbSchema;
die "ABORT: $originalDropSchema does not exist\n"
if ! -e $originalDropSchemaFullpath;
die "ABORT: $originalDbSchema does not exist\n"
if ! -e $originalDbSchemaFullpath;
my $dbUser = &getOrastagerconfigParam("user");
my $dbPasswd = &getOrastagerconfigParam("passwd");
my $dbName = &getOrastagerconfigParam("dbName");
executeSQLPlusScript ( $dbUser, $dbPasswd, $dbName,
$originalDropSchemaFullpath,
"Dropping schema");
my $stageGid = $environment{stageGid};
my $stageUid = $environment{stageUid};
my $adminList = $environment{adminList};
my $hacked_creation= `mktemp`;
chomp $hacked_creation;
`cat $originalDbSchemaFullpath > $hacked_creation`;
`sed -i s/^ACCEPT/--ACCEPT/ $hacked_creation`;
`sed -i s/^PROMPT/--PROMPT/ $hacked_creation`;
`sed -i s/^UNDEF/--UNDEF/ $hacked_creation`;
`sed -i s/\\&stageGid/$stageGid/g $hacked_creation`;
`sed -i s/\\&stageUid/$stageUid/g $hacked_creation`;
`sed -i s/\\&adminList/$adminList/g $hacked_creation`;
executeSQLPlusScript ( $dbUser, $dbPasswd, $dbName,
$hacked_creation, "Re-creating schema");
unlink $hacked_creation;
# Restart the demons
`/etc/init.d/jobmanagerd start`;
#`/etc/init.d/mighunterd start`;
`/etc/init.d/rechandlerd start`;
`/etc/init.d/rhd start`;
`/etc/init.d/rmmasterd start`;
#`/etc/init.d/rtcpclientd start`;
`/etc/init.d/stagerd start`;
#`/etc/init.d/tapegatewayd start`;
my $sleepPeriod = 2;
print("Sleeping $sleepPeriod seconds\n");
sleep($sleepPeriod);
my $rmGetNodesResult = `rmGetNodes | egrep 'name:'`;
print("\n");
print("rmGetNodes RESULTS\n");
print("==================\n");
print($rmGetNodesResult);
# Fill database with the standard set-up for a dev-box
`nslistclass | grep NAME | awk '{print \$2}' | xargs -i enterFileClass --Name {} --GetFromCns`;
`enterSvcClass --Name default --DiskPools default --DefaultFileSize 10485760 --FailJobsWhenNoSpace yes --NbDrives 1 --TapePool stager_dev04 --MigratorPolicy defaultMigrationPolicy --StreamPolicy streamPolicyAlwaysReturning1`;
`enterSvcClass --Name dev --DiskPools extra --DefaultFileSize 10485760 --FailJobsWhenNoSpace yes`;
`enterSvcClass --Name diskonly --DiskPools extra --ForcedFileClass temp --DefaultFileSize 10485760 --Disk1Behavior yes --FailJobsWhenNoSpace yes`;
`moveDiskServer default lxc2disk05.cern.ch`;
`moveDiskServer extra lxc2disk06.cern.ch`;
`rmAdminNode -r -R -n lxc2disk05.cern.ch`;
`rmAdminNode -r -R -n lxc2disk06.cern.ch`;
# Add a tape-pool to dev service-class ready for shared tape-pool tests
###`modifySvcClass --Name dev --AddTapePool stager_dev03_2 --MigratorPolicy defaultMigrationPolicy --StreamPolicy streamPolicyAlwaysReturning1`;
# Set the number of drives on the default and dev service-classes to desired number for each
###`modifySvcClass --Name default --NbDrives 1`;
###`modifySvcClass --Name dev --NbDrives 2`;
###`modifySvcClass --Name dev --AddTapePool stager_dev03_2`;
$dbh->disconnect();
undef $dbh;
}
# Migrate the stager's db to the new tapegateway schema. This includes a switch to the tapegatewayd (in DB)
# stops but does not start the demons
......@@ -1028,12 +1168,19 @@ sub getLdLibraryPathFromSrcPath ( $ ) {
return $ldLibraryPath;
}
sub cleanup () {
print "In CastorTapeTests::cleanup: cleaning up files\n";
for (@local_files) {
print "Cleanup: removing ".$_->{name}."\n";
unlink $_->{name};
}
}
# Final cleanup of the library.
# So far, get rid of leftover local files
END {
if (@local_files) {
unlink $_{'name'};
}
print "In CastorTapeTests::END: calling cleanup\n";
cleanup();
}
# # create a local
1; # this should be your last line
......@@ -3,6 +3,20 @@
# Host-dependant parameters
TAPETEST username_lxcastordev03 canoc3
TAPETEST checkout_location_lxcastordev03 /var/scratch/canoc3/test_gateway_r21263
TAPETEST file_size_lxcastordev03 10485760
TAPETEST file_number_lxcastordev03 10
TAPETEST castor_directory_lxcastordev03 /castor/cern.ch/dev/c/canoc3/
TAPETEST migration_timeout_lxcastordev03 60
TAPETEST poll_interval_lxcastordev03 5
TAPETEST username_lxcastordev04 murrayc3
TAPETEST checkout_location_lxcastordev04 /var/some_checkout
TAPETEST file_size_lxcastordev04 10485760
TAPETEST file_number_lxcastordev04 10
TAPETEST castor_directory_lxcastordev04 /castor/cern.ch/dev/c/murrayc3/
TAPETEST migration_timeout_lxcastordev04 60
TAPETEST poll_interval_lxcastordev04 5
# General parameters
TAPETEST dbDir castor/db
......@@ -11,4 +25,6 @@ TAPETEST stageUid 14029
TAPETEST stageGid 1474
TAPETEST adminList :1028
TAPETEST originalDbSchema stager_oracle_create.sql
TAPETEST originalDropSchema drop_oracle_schema.sql
\ No newline at end of file
TAPETEST originalDropSchema drop_oracle_schema.sql
TAPETEST castor_single_subdirectory migrate_and_recall_10_single_tape_copy_files
TAPETEST castor_dual_subdirectory migrate_and_recall_10_dual_tape_copy_files
......@@ -34,24 +34,19 @@ use DBI;
use DBD::Oracle qw(:ora_types);
use CastorTapeTests;
# Constants
# castor/cern.ch/dev/m/murrayc3/simpleTest_Sat_Aug_7.txt
my $DEV_FILE_NS_FILEID=5000043410;
my $adminList = ':1028'; # gid=1028(c3)
my $originalDbSchema = "stager_oracle_create.sql";
my $originalDropSchema = "drop_oracle_schema.sql";
my $stageGid = 1474; # gid=1474(st)
my $stageUid = 14029; # uid=14029(stage)
my $checkout_location = '/var/scratch/canoc3/test_gateway_r21263';
my $dbDir = 'castor/db';
my $tstDir = 'test/tapegateway_release_tests';
sub goodDaySingleAndDualCopyTest ( $$ );
sub main ();
# Make sure at least Ctrl-C triggers a cleanup
sub sigint()
{
warn "Interrupt caught. Exiting.";
exit (1);
}
$SIG{INT} = "sigint";
$SIG{TERM} = "sigint";
# Hook for main.
main();
......@@ -59,38 +54,56 @@ main();
sub main ()
{
# Initialize: setup the environment
CastorTapeTests::read_config ("./tapetests-lxcastordev.conf" );
my $conffile = './tapetests-lxcastordev.conf';
CastorTapeTests::read_config($conffile);
CastorTapeTests::check_environment ();
my $file_size = CastorTapeTests::get_environement('file_size');
my $file_number = CastorTapeTests::get_environement('file_number');
my $castor_directory = CastorTapeTests::get_environement('castor_directory');
my $single_subdir = CastorTapeTests::get_environement('castor_single_subdirectory');
my $dual_subdir = CastorTapeTests::get_environement('castor_dual_subdirectory');
my $username = CastorTapeTests::get_environement('username');
my $dbh = CastorTapeTests::open_db();
my $ret = 0;
if (CastorTapeTests::check_leftovers ( $dbh )) {
my $userless; # workaround to prevent emacs from fscking the indentation.
CastorTapeTests::print_leftovers ( $dbh );
$dbh->disconnect();
exit 1;
$dbh->disconnect();
exit 1;
}
$dbh->disconnect();
# Nuke and start clean
CastorTapeTests::wipeAndRecreateDefaultLxcastodev03();
my $seed_index = CastorTapeTests::make_seed (100*1024*1024);
my $host = `hostname -s`; chomp $host;
if ( $host eq 'lxcastordev03' ) {
my $u;
CastorTapeTests::wipeAndRecreateDefaultLxcastodev03();
} elsif ( $host eq 'lxcastordev04' ) {
my $u;
CastorTapeTests::wipeAndRecreateDefaultLxcastodev04();
} else {
die "Unexpected host.";
}
my $seed_index = CastorTapeTests::make_seed ($file_size);
# Test with the old system
CastorTapeTests::startDaemons();
# On first run, clean house
print "Cleaning up test directories /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_{single,dual}_tape_copy_files\n";
print `su canoc3 -c "for p in /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_{single,dual}_tape_copy_files; do nsrm -r -f \\\$p; done"`;
print "Cleaning up test directories $castor_directory\{$single_subdir,$dual_subdir\}\n";
print `su $username -c "for p in $castor_directory\{$single_subdir,$dual_subdir\}; do nsrm -r -f \\\$p; done"`;
# Re-create the directories:
print `su canoc3 -c "nsmkdir /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_single_tape_copy_files"`;
print `su canoc3 -c "nschclass largeuser /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_single_tape_copy_files"`;
print `su canoc3 -c "nsmkdir /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_dual_tape_copy_files"`;
print `su canoc3 -c "nschclass test2 /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_dual_tape_copy_files"`;
print `su $username -c "nsmkdir $castor_directory$single_subdir"`;
print `su $username -c "nschclass largeuser $castor_directory$single_subdir"`;
print `su $username -c "nsmkdir $castor_directory$dual_subdir"`;
print `su $username -c "nschclass test2 $castor_directory$dual_subdir"`;
# First iteration of the test
goodDaySingleAndDualCopyTest ($seed_index, 10);
goodDaySingleAndDualCopyTest ($seed_index, $file_number);
# Migrate to the new system, with tape gateway, still running rtcpclientd
$dbh=CastorTapeTests::open_db();
......@@ -99,17 +112,17 @@ sub main ()
CastorTapeTests::startDaemons();
# Second iteration of the test
goodDaySingleAndDualCopyTest ($seed_index, 10);
goodDaySingleAndDualCopyTest ($seed_index, $file_number);
# Switch to tape gateway
CastorTapeTests::stopAndSwitchToTapeGatewayd ( $dbh );
CastorTapeTests::startDaemons();
# Third iteration of the test
goodDaySingleAndDualCopyTest ($seed_index, 10);
goodDaySingleAndDualCopyTest ($seed_index, $file_number);
print "Cleaning up test directories /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_{single,dual}_tape_copy_files\n";
print `su canoc3 -c "for p in /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_{single,dual}_tape_copy_files; do nsrm -r -f \\\$p; done"`;
print "Cleaning up test directories $castor_directory\{$single_subdir,$dual_subdir\}\n";
print `su $username -c "for p in $castor_directory\{$single_subdir,$dual_subdir\}; do nsrm -r -f \\\$p; done"`;
exit 0;
}
......@@ -117,19 +130,27 @@ sub main ()
# stager_rm them and then recall them. Eventually, drop them.
sub goodDaySingleAndDualCopyTest ( $$ )
{
my ( $seed_index, $file_number ) = ( shift, shift);
for my $sd ('single', 'dual') {
for my $i (0 .. ($file_number - 1) ) {
my $file_name="/tmp/".`uuidgen`;
chomp $file_name;
CastorTapeTests::make_localfile( $seed_index, $file_name );
`su canoc3 -c \"rfcp $file_name /castor/cern.ch/dev/c/canoc3/migrate_and_recall_10_${sd}_tape_copy_files\"`;
}
}
my $dbh=CastorTapeTests::open_db();
if ( CastorTapeTests::check_leftovers_poll_timeout ( $dbh, 2, 60 )) {
print "Timeout expired for initial cleanup of files.";
print_leftovers ($dbh);
die;
my ( $seed_index, $file_number ) = ( shift, shift);
my $castor_directory = CastorTapeTests::get_environement('castor_directory');
my $single_subdir = CastorTapeTests::get_environement('castor_single_subdirectory');
my $dual_subdir = CastorTapeTests::get_environement('castor_dual_subdirectory');
my $username = CastorTapeTests::get_environement('username');
my $poll = CastorTapeTests::get_environement('poll_interval');
my $timeout = CastorTapeTests::get_environement('migration_timeout');
for my $sd ($single_subdir, $dual_subdir) {
for my $i (0 .. ($file_number - 1) ) {
my $file_name="/tmp/".`uuidgen`;
chomp $file_name;
CastorTapeTests::make_localfile( $seed_index, $file_name );
`su $username -c \"rfcp $file_name ${castor_directory}${sd}\"`;
}
}
my $dbh=CastorTapeTests::open_db();
if ( CastorTapeTests::check_leftovers_poll_timeout ( $dbh, $poll, $timeout )) {
print "Timeout expired for initial cleanup of files.";
CastorTapeTests::print_leftovers ($dbh);
die;
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment