Commit 8e132484 authored by Scott Snyder's avatar Scott Snyder Committed by Graeme Stewart
Browse files

Adjust for checkreq in AtlasPolicy-01-08-15 and fix other checkreq warnings....

Adjust for checkreq in AtlasPolicy-01-08-15 and fix other checkreq warnings. (RelationalCollection-00-02-04)
parent 0f45ca50
package RelationalCollection
# imported from LCG/POOL to ATLAS by:
author Marcin Nowak
use AtlasPolicy AtlasPolicy-*
private
use GaudiInterface GaudiInterface-* External
use AtlasCORAL AtlasCORAL-* External
use POOLCore POOLCore-* Database/APR
use CollectionBase CollectionBase-* Database/APR
use PersistentDataModel PersistentDataModel-* Database
apply_pattern pool_plugin_library
#========= TESTS
use TestPolicy TestPolicy-*
pattern RelationalCollection_test_run \
use TestTools TestTools-* AtlasTest ; \
application <name>_test -suffix=_<name> "../tests/<name>/*.cpp" application_suffix="" ; \
document athenarun_launcher <name>_utest -group=$(whichGroup) \
athenarun_exe="'../${CMTCONFIG}/<name>_test'" \
athenarun_pre="'. ../cmt/setup.sh ; rm -f pool*.root *xml'" \
athenarun_opt="" \
athenarun_out="' > <name>_test.log 2>&1'" \
athenarun_post="'post.sh <name>_test $(q)<extrapatterns>$(q)'"
# The unit tests
apply_pattern RelationalCollection_test_run name=WriteRead
apply_pattern RelationalCollection_test_run name=WriteUpdate
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#include "DBLock.h"
#include "POOLCore/Exception.h"
#include "RelationalCollectionNames.h"
#include "RelationalCollectionBindVariables.h"
#include "RelationalAccess/TableDescription.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "RelationalAccess/ITable.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ITablePrivilegeManager.h"
#include "RelationalAccess/ITransaction.h"
#include "RelationalAccess/ITableDataEditor.h"
#include "CoralBase/MessageStream.h"
#include "CoralBase/AttributeSpecification.h"
#include "CoralBase/Attribute.h"
#include "CoralBase/AttributeList.h"
#include "CoralBase/TimeStamp.h"
#ifdef WIN32
# include <windows.h>
# define sleep(x) Sleep((x)*1000)
std::string userinfo() {
return "Windows";
}
#elif defined __APPLE__
std::string userinfo() {
return "Apple";
}
#else
// hopefully UNIX
# include <unistd.h>
# include <limits.h>
# include <pwd.h>
std::string userinfo() {
static char hostname[HOST_NAME_MAX+1];
if( gethostname( hostname, HOST_NAME_MAX ) ) {
*hostname = 0; //error
}
hostname[HOST_NAME_MAX] = 0;
struct passwd *pwdent = getpwuid(geteuid());
const char *username_p = (pwdent? pwdent->pw_name : 0);
std::string username;
if( username_p )
username = username_p;
return std::string(hostname) + "/" + username;
}
#endif
using namespace pool;
using namespace std;
using namespace pool::RelationalCollection;
DBLock::DBLock( coral::ISessionProxy& session, std::string collName )
: m_session( session ),
m_name( collName ),
m_locked( false ),
m_autorelease( true ),
m_lockIngoreAge( 4000 ),
m_maxWait( 300 )
{
const char* env = getenv("POOL_RCOLL_LOCKAGELIMIT");
if( env ) {
int age = atoi(env);
if( age < 60 && age != 0 ) {
// 0 means infinity
m_lockIngoreAge = 60;
} else {
m_lockIngoreAge = age;
}
}
env = getenv("POOL_RCOLL_LOCKTIMEOUT");
if( env ) {
int to = atoi(env);
if( to < 10 && to != 0 ) {
// 0 means infinity
m_maxWait = 10;
} else {
m_maxWait = to;
}
}
}
DBLock::~DBLock()
{
if( m_locked && m_autorelease ) {
unlock();
m_session.transaction().commit();
}
}
void
DBLock::setAutorelease( bool ar )
{
m_autorelease = ar;
}
void
DBLock::lock( bool XTrans )
{
using namespace pool::RelationalCollection;
coral::MessageStream log( "pool::RelationalCollection::DBLock" );
coral::ISchema& nominalSchema = m_session.nominalSchema();
if( !nominalSchema.existsTable( RelationalCollectionNames::nameOfCollectionLockTable() ) ) {
// create the table where the locking is done
coral::TableDescription description( RelationalCollectionNames::nameOfCollectionLockTable() );
description.setName( RelationalCollectionNames::nameOfCollectionLockTable() );
description.insertColumn(
RelationalCollectionNames::entryIDVarInCollectionLockTable(),
coral::AttributeSpecification::typeNameForType<std::string>(), 50, false );
description.insertColumn(
RelationalCollectionNames::collectionNameVarInCollectionLockTable(),
coral::AttributeSpecification::typeNameForType<std::string>(), 500, false );
description.insertColumn(
RelationalCollectionNames::clientInfoVarInCollectionLockTable(),
coral::AttributeSpecification::typeNameForType<std::string>(), 500, false );
description.insertColumn(
RelationalCollectionNames::lockTypeVarInCollectionLockTable(),
coral::AttributeSpecification::typeNameForType<std::string>(), 20, false );
description.insertColumn(
RelationalCollectionNames::timestampVarInCollectionLockTable(),
coral::AttributeSpecification::typeNameForType<coral::TimeStamp>(), 2, false );
description.setPrimaryKey( RelationalCollectionNames::entryIDVarInCollectionLockTable() );
log << coral::Debug << " Creating lock TABLE:" << RelationalCollectionNames::nameOfCollectionLockTable() << coral::MessageStream::endmsg;
nominalSchema.createTable( description ); //.privilegeManager().grantToPublic( coral::ITablePrivilegeManager::Select );
}
int total_wait = 0;
do {
coral::ITable& lockTable = m_session.nominalSchema().tableHandle(
RelationalCollectionNames::nameOfCollectionLockTable() );
coral::IQuery* query = lockTable.newQuery();
query->addToOutputList( RelationalCollectionNames::entryIDVarInCollectionLockTable() );
query->addToOutputList( RelationalCollectionNames::collectionNameVarInCollectionLockTable() );
query->addToOutputList( RelationalCollectionNames::lockTypeVarInCollectionLockTable() );
query->addToOutputList( RelationalCollectionNames::timestampVarInCollectionLockTable() );
query->addToOutputList( RelationalCollectionNames::clientInfoVarInCollectionLockTable() );
coral::AttributeList whereDataForLock;
whereDataForLock.extend<std::string>(
RelationalCollectionNames::entryIDVarInCollectionLockTable() );
whereDataForLock.begin()->data<std::string>() = "LOCK";
string whereClauseForLock = RelationalCollectionNames::entryIDVarInCollectionLockTable() + " = :" + RelationalCollectionNames::entryIDVarInCollectionLockTable();
query->setCondition( whereClauseForLock, whereDataForLock );
query->limitReturnedRows( 1, 0 );
query->setForUpdate();
log << coral::Debug << " Checking the LOCK table:" << RelationalCollectionNames::nameOfCollectionLockTable() << coral::MessageStream::endmsg;
coral::ICursor& cursor = query->execute();
if( !cursor.next() ) {
log << coral::Debug << " Creating lock ROW in " << RelationalCollectionNames::nameOfCollectionLockTable() << coral::MessageStream::endmsg;
// no entry - new table or row removed by hand. Add the row
coral::AttributeList rowBuffer;
rowBuffer.extend< std::string >( RelationalCollectionNames::entryIDVarInCollectionLockTable() );
rowBuffer.extend< std::string >( RelationalCollectionNames::collectionNameVarInCollectionLockTable() );
rowBuffer.extend< std::string >( RelationalCollectionNames::clientInfoVarInCollectionLockTable() );
rowBuffer.extend< std::string >( RelationalCollectionNames::lockTypeVarInCollectionLockTable() );
rowBuffer.extend< coral::TimeStamp >( RelationalCollectionNames::timestampVarInCollectionLockTable() );
rowBuffer[0].data< std::string >() = "LOCK";
rowBuffer[1].data< std::string >() = m_name;
rowBuffer[2].data< std::string >() = userinfo();
rowBuffer[3].data< std::string >() = XTrans? "PERM" : "TEMP";
rowBuffer[4].data< coral::TimeStamp >() = coral::TimeStamp::now();
lockTable.dataEditor().insertRow( rowBuffer );
} else {
string lockType = cursor.currentRow()[RelationalCollectionNames::lockTypeVarInCollectionLockTable()].data< std::string >();
long long lockTime = cursor.currentRow()[RelationalCollectionNames::timestampVarInCollectionLockTable()].data< coral::TimeStamp >().total_nanoseconds() / 1000000000;
long long nowTime = coral::TimeStamp::now().total_nanoseconds() / 1000000000;
long long lock_age = nowTime - lockTime;
log << coral::Debug << " Current lock type is " << lockType << coral::MessageStream::endmsg;
if( lockType == "PERM" ) {
string lock_creator = cursor.currentRow()[RelationalCollectionNames::clientInfoVarInCollectionLockTable()].data< std::string >();
string locked_collection = cursor.currentRow()[RelationalCollectionNames::collectionNameVarInCollectionLockTable()].data< std::string >();
log << coral::Warning << " Permanent collection database lock detected (" << lock_age << " sec old) made by " << lock_creator << " on collection " << locked_collection << coral::MessageStream::endmsg;
if( m_lockIngoreAge==0 || lock_age < m_lockIngoreAge ) {
delete query; query = 0;
// release locks and wait some
m_session.transaction().commit();
if( total_wait >= m_maxWait ) {
log << coral::Warning << " Wait time exceeded: maximum specified wait time: " << m_maxWait << " sec - giving up" << coral::MessageStream::endmsg;
throw pool::Exception("Lock wait time exceeded",
"RelationalCollection::open::DBLock::lock",
"RelationalCollection" );
}
log << coral::Warning << " Sleeping for 10s" << coral::MessageStream::endmsg;
sleep(10);
total_wait += 10;
bool forUpdate(false);
m_session.transaction().start( forUpdate );
continue;
} else {
log << coral::Warning << " Ignoring database lock: greater than " << m_lockIngoreAge << coral::MessageStream::endmsg;
}
}
log << coral::Debug << " Setting database lock to " << ( XTrans? "'PERM'" : "'TEMP'" ) << coral::MessageStream::endmsg;
// update the row in the LOCK table to set databse lock and store lock type
string setClause =
RelationalCollectionNames::collectionNameVarInCollectionLockTable()
+ " = '" + m_name + "'"
+ ", " + RelationalCollectionNames::clientInfoVarInCollectionLockTable()
+ " = '" + userinfo() + "'"
+ ", " + RelationalCollectionNames::timestampVarInCollectionLockTable()
+ " = :newtime "
+ ", " + RelationalCollectionNames::lockTypeVarInCollectionLockTable()
+ " = " + ( XTrans? "'PERM'" : "'TEMP'" );
whereDataForLock.extend< coral::TimeStamp >( "newtime" );
whereDataForLock[1].data< coral::TimeStamp >() = coral::TimeStamp::now();
lockTable.dataEditor().updateRows( setClause, whereClauseForLock, whereDataForLock );
}
delete query;
m_locked = true;
return;
}while( true );
}
void
DBLock::unlock()
{
using namespace pool::RelationalCollection;
coral::ISchema& nominalSchema = m_session.nominalSchema();
coral::ITable& lockTable = nominalSchema.tableHandle(
RelationalCollectionNames::nameOfCollectionLockTable() );
string setClause =
RelationalCollectionNames::lockTypeVarInCollectionLockTable() + " = 'NONE'";
coral::AttributeList whereDataForLock;
whereDataForLock.extend<std::string>(
RelationalCollectionNames::entryIDVarInCollectionLockTable() );
whereDataForLock.begin()->data<std::string>() = "LOCK";
string whereClauseForLock = RelationalCollectionNames::entryIDVarInCollectionLockTable() + " = :" + RelationalCollectionNames::entryIDVarInCollectionLockTable();
lockTable.dataEditor().updateRows( setClause, whereClauseForLock, whereDataForLock );
m_locked = false;
}
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#ifndef RELATIONALCOLLECTION_DBLOCK_H
#define RELATIONALCOLLECTION_DBLOCK_H
#include "RelationalAccess/ISessionProxy.h"
namespace pool {
namespace RelationalCollection {
class DBLock
{
public:
DBLock( coral::ISessionProxy& session, std::string collName );
~DBLock();
/// set a global lock to serialize collection operations (creation etc)
/// if XTrans is flase, the lock will be gone when transaction is committed
/// if XTrans is true, the lock will be permanent
void lock( bool Xtrans );
/// remove global lock
void unlock();
/// set lock release behavior in the destructor
void setAutorelease( bool ar );
private:
/// Reference to current database access session.
coral::ISessionProxy& m_session;
std::string m_name;
/// true if this object holds a database lock
bool m_locked;
/// if true (default) try to remove the lock in the destructor
bool m_autorelease;
/// ignore database lock if older than this limit (in sec)
long long m_lockIngoreAge;
/// give up waiting for database access after this time (in sec)
int m_maxWait;
};
}
}
#endif
/*
Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
*/
#include "GUIDQuery.h"
#include "RelationalCollectionNames.h"
#include "PersistentDataModel/Token.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "RelationalAccess/ISessionProxy.h"
#include "RelationalAccess/ISessionProperties.h"
#include "CoralBase/Attribute.h"
#include "CoralBase/AttributeList.h"
#include "CoralBase/MessageStream.h"
#include "POOLCore/Exception.h"
#include "CollectionBase/ICollectionDescription.h"
#include <iostream>
using namespace std;
namespace pool {
namespace RelationalCollection {
// convert empty Tokens into zeroes
string guidFromToken( const Token& token ) {
return token.dbID().toString();
}
void GUIDQuery::readGUIDs()
{
m_guids.clear();
coral::MessageStream log( "pool::RelationalCollection::readGUIDs()" );
if( m_session.properties().flavorName() != "Oracle" || m_selectedTokenColumnNames.size() <= 1 ) {
// simple version for a single token, or non-oracle database
for( std::set<std::string>::const_iterator
tokenNameI = m_selectedTokenColumnNames.begin(),
end = m_selectedTokenColumnNames.end();
tokenNameI != end; ++tokenNameI ) {
log << coral::Debug
<< "Querying GUIDs for token " << *tokenNameI << coral::MessageStream::endmsg;
readGUIDs( *tokenNameI );
}
return;
}
// Oracle version - single query for all tokens, using CASE syntax
prepareQueryCondition();
prepareQuery();
// Create a subquery that produces a sequence (1,2,.., <number of tokens>)
// used in the main query to iterate over the tokens in a single row
string tokenSubQueryName = "token_seq";
string tokenNumName = "token_num";
coral::IQueryDefinition& subquery = m_query->defineSubQuery(tokenSubQueryName);
subquery.addToTableList( RelationalCollectionNames::nameOfCollectionDescriptionsTable() );
coral::AttributeList bindings;
bindings.extend(tokenNumName, typeid(unsigned));
bindings[tokenNumName].data<unsigned>() = m_selectedTokenColumnNames.size();
log << coral::Debug << "Selecting " << bindings[tokenNumName].data<unsigned>() << " tokens" << coral::MessageStream::endmsg;
subquery.setCondition("rownum<= :"+tokenNumName, bindings);
subquery.addToOutputList("rownum", tokenNumName);
m_query->addToTableList( tokenSubQueryName );
// create the case statements for the main query - one switch per token
// it will extract the token link reference and token name
std::ostringstream sqlTokID;
std::ostringstream sqlTokName;
sqlTokID << "case";
sqlTokName << "case";
int tokenN = 1;
for( std::set<std::string>::const_iterator
tokenNameI = m_selectedTokenColumnNames.begin(),
end = m_selectedTokenColumnNames.end();
tokenNameI != end; ++tokenNameI )
{
string oid1ColumnName = m_tableTokenColumnPrefixForCollectionTokenColumnName.find( *tokenNameI )->second + RelationalCollectionNames::oid_1_variableInCollectionDataTable();
sqlTokID << " when " << tokenSubQueryName << "." << tokenNumName << "=" <<tokenN
<< " then " << oid1ColumnName;
sqlTokName << " when " << tokenSubQueryName << "." << tokenNumName << "=" <<tokenN
<< " then " << "'" << *tokenNameI << "'";
tokenN++;
}
sqlTokID << " end ";
sqlTokName << " end ";
m_query->addToOutputList( sqlTokID.str(), "TokenLink" );
m_outputDataBuffer->extend( "TokenLink", typeid(unsigned) );
m_query->addToOutputList( sqlTokName.str(), "TokenName" );
m_outputDataBuffer->extend( "TokenName", typeid(string) );
// Execute query and retrieve cursor for navigation over result.
m_query->defineOutput( *m_outputDataBuffer );
m_query->setDistinct();
coral::ICursor& cursor = m_query->execute();
// map token link IDs into GUIDs using the link tables retrieved when opening the collection
Token token;
while( cursor.next() ) {
unsigned linkId = cursor.currentRow()[0].data< unsigned >();
const string& tokenName = cursor.currentRow()[1].data< string >();
const string& fragmentName = m_description.collectionFragmentName( tokenName );
std::map< unsigned, std::string >* tokenKeyForLinkId = m_mapOfTokenKeyForLinkIdMaps.find( fragmentName )->second;
token.fromString( tokenKeyForLinkId->find( linkId )->second );
m_guids.insert( make_pair(guidFromToken(token), tokenName) );
//cout << token.dbID() << " " << tokenName << endl;
}
}
void GUIDQuery::readGUIDs( const std::string& tokenName )
{
prepareQueryCondition();
prepareQuery();
// Form the OID_1 column name of the Token.
std::string oid1ColumnName = m_tableTokenColumnPrefixForCollectionTokenColumnName.find( tokenName ) ->second
+ RelationalCollectionNames::oid_1_variableInCollectionDataTable();
m_query->addToOutputList( oid1ColumnName );
m_outputDataBuffer->extend( oid1ColumnName, typeid(unsigned) );
// Execute query and retrieve cursor for navigation over result.
m_query->defineOutput( *m_outputDataBuffer );
m_query->setDistinct();
coral::ICursor& cursor = m_query->execute();
// map token link IDs into GUIDs using the link tables retrieved when opening the collection
string fragmentName = m_description.collectionFragmentName( tokenName );
std::map< unsigned, std::string >* tokenKeyForLinkId = m_mapOfTokenKeyForLinkIdMaps.find( fragmentName )->second;
Token token;
while( cursor.next() ) {
unsigned linkId = cursor.currentRow()[0].data< unsigned >();
token.fromString( tokenKeyForLinkId->find( linkId )->second );
m_guids.insert( make_pair(guidFromToken(token),tokenName) );
}
}
void GUIDQuery::addToAttributeOutputList( const std::string& columnName )
{
std::string errorMsg( "Attribute `" + columnName + "' is not a Token" );
throw pool::Exception( errorMsg,
"RelationalCollectionGUIDQuery::addToOutputList",
"RelationalCollection" );
}
const GUIDQuery::CountedGroupedGUIDs&
GUIDQuery::getGroupedGUIDs()
{
m_groupedGUIDs.clear();
coral::MessageStream log( "pool::RelationalCollection::getGroupedGUIDs()" );
prepareQueryCondition();
prepareQuery();
/* create query in the form:
SELECT COUNT(*), tokens GROUP BY tokens
it will extract the token link references */
m_query->addToOutputList( "count(*)", "count" );
m_outputDataBuffer->extend( "count", typeid(unsigned) );
string sep, groupby;
for( std::set<std::string>::const_iterator
tokenNameI = m_selectedTokenColumnNames.begin(),
end = m_selectedTokenColumnNames.end();
tokenNameI != end; ++tokenNameI )
{
string oid1ColumnName = m_tableTokenColumnPrefixForCollectionTokenColumnName.find( *tokenNameI )->second + RelationalCollectionNames::oid_1_variableInCollectionDataTable();
m_query->addToOutputList( oid1ColumnName, *tokenNameI );
m_outputDataBuffer->extend( *tokenNameI, typeid(unsigned) );
groupby += sep + oid1ColumnName;
sep = ",";
m_groupedGUIDs.tokenNames.push_back( *tokenNameI );
}
m_query->groupBy( groupby );
// Execute query and retrieve cursor for navigation over result.
m_query->defineOutput( *m_outputDataBuffer );
coral::ICursor& cursor = m_query->execute();
// map token link IDs into GUIDs using the link tables retrieved when opening the collection
Token token;
while( cursor.next() ) {
countedGUIDGroup_t row;
row.first = cursor.currentRow()[0].data< unsigned >();
for( std::set<std::string>::const_iterator
tokenNameI = m_selectedTokenColumnNames.begin(),
end = m_selectedTokenColumnNames.end();
tokenNameI != end; ++tokenNameI )
{
unsigned linkId = cursor.currentRow()[*tokenNameI].data< unsigned >();
const string& fragmentName = m_description.collectionFragmentName( *tokenNameI );
std::map< unsigned, std::string >* tokenKeyForLinkId = m_mapOfTokenKeyForLinkIdMaps.find( fragmentName )->second;
token.fromString( tokenKeyForLinkId->find( linkId )->second );
row.second.push_back( guidFromToken(token) );
}
// check for duplicated rows - result of an old linktable insert bug
// if found, just sum up the count
bool found(false);
for( vector< countedGUIDGroup_t >::iterator rowi = m_groupedGUIDs.groupedGUIDRows.begin(),
end = m_groupedGUIDs.groupedGUIDRows.end(); rowi != end; ++rowi ) {
bool match(true);
for( unsigned t = 0; t < rowi->second.size(); t++ ) {
if( row.second[t] != rowi->second[t] ) {
match = false; break;
}
}
if( match ) {
rowi->first += row.first;
found = true;
break;
}
}
if( !found )