Commit ecca8e01 authored by Volodymyr Yurchenko's avatar Volodymyr Yurchenko
Browse files

Rewrite the clean up section, parse parent dir from XML, add details to validation output

parent 4a468599
...@@ -8,7 +8,6 @@ import java.io.FileOutputStream; ...@@ -8,7 +8,6 @@ import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
...@@ -84,6 +83,18 @@ public class ArchiveMemberDelete { ...@@ -84,6 +83,18 @@ public class ArchiveMemberDelete {
// final ByteArrayOutputStream out = new ByteArrayOutputStream(); // final ByteArrayOutputStream out = new ByteArrayOutputStream();
// commander.setLine(new JSONPrintWriter(out), null); // commander.setLine(new JSONPrintWriter(out), null);
System.out.println(); System.out.println();
System.out.println("[" + new Date() + "] Processing " + xmlEntry);
// Parse the parent directory from the entry, e.g.:
// /alice/sim/2018/LHC18e1a/246053/075/BKG/TrackRefs.root ->
// /alice/sim/2018/LHC18e1a/246053/075
String parentdir = xmlEntry.substring(0, xmlEntry.lastIndexOf("/"));
final String lastStringToken = parentdir.substring(parentdir.lastIndexOf("/") + 1);
if (!lastStringToken.matches("^\\d+$")) {
parentdir = parentdir.substring(0, parentdir.lastIndexOf("/"));
}
final String registerPath = parentdir + "/registertemp";
LFN remoteLFN = null; LFN remoteLFN = null;
try { try {
...@@ -95,35 +106,53 @@ public class ArchiveMemberDelete { ...@@ -95,35 +106,53 @@ public class ArchiveMemberDelete {
return; return;
} }
// Clean up previous iterations
//
if (remoteLFN == null || !remoteLFN.exists) { if (remoteLFN == null || !remoteLFN.exists) {
if (!commander.c_api.find(xmlEntry.substring(0, xmlEntry.lastIndexOf("/")), ".deleted", 0).isEmpty()) // TrackRefs was deleted, registertemp contains valid files
System.out.println("[" + new Date() + "] " + xmlEntry + ": already deleted, continue."); System.err.println("[" + new Date() + "] " + xmlEntry + ": LFN doesn't exist, checking registertemp");
else
System.err.println("[" + new Date() + "] " + xmlEntry + ": LFN doesn't exist. Abort."); if (commander.c_api.getLFN(registerPath) != null) {
// Move everything from registertemp to parent
// Check if there are any leftovers and clean up System.out.println("[" + new Date() + "] " + "registertemp found, moving out its content");
// (get the archive from last modified root files in remote directory, get archive's parent dir, check if registertemp exists there, move files) for (final LFN file : commander.c_api.getLFNs(registerPath)) {
final Collection<LFN> membersSet = commander.c_api.find(xmlEntry.substring(0, xmlEntry.lastIndexOf("/")), "*root", 0); if (file == null || !file.exists) {
LFN remoteArchiveLFN = null; System.err.println("[" + new Date() + "] " + "Failed to get directory listing for " + registerPath + ". Abort.");
long lastModified = 0; return;
for (final LFN l : membersSet) }
if (l.ctime.getTime() > lastModified) System.out.println("[" + new Date() + "] " + "Moving " + registerPath + "/" + file.getFileName());
remoteArchiveLFN = commander.c_api.getRealLFN(l.getCanonicalName()); if (commander.c_api.moveLFN(registerPath + "/" + file.getFileName(), parentdir + "/" + file.getFileName()) == null) {
System.err.println("[" + new Date() + "] " + "Failed to move " + file.getFileName() + ". Abort.");
if (remoteArchiveLFN != null) { return;
final String registerPath = remoteArchiveLFN.getParentName() + System.getProperty("file.separator") + "registertemp"; }
for (final LFN file : commander.c_api.find(registerPath, "*root", 0)) {
System.out.println("Moving " + registerPath + System.getProperty("file.separator") + file.getFileName());
commander.c_api.moveLFN(registerPath + System.getProperty("file.separator") + file.getFileName(),
remoteArchiveLFN.getParentName() + System.getProperty("file.separator") + file.getFileName());
} }
// Delete registertemp dir since all files are moved
if (registerPath.length() > 20) // Safety check if (registerPath.length() > 20) // Safety check
commander.c_api.removeLFN(registerPath, true); commander.c_api.removeLFN(registerPath, true);
} }
else {
if (!commander.c_api.find(parentdir, ".deleted", 0).isEmpty()) {
System.out.println("[" + new Date() + "] " + "registertemp is not there, all DONE");
}
else {
System.out.println("[" + new Date() + "] " + "registertemp is not there, but .deleted NOT FOUND. Abort.");
}
}
return; return;
} }
// Else
// TrackRefs was not deleted, remove invalid files from registertemp
if (commander.c_api.getLFN(registerPath) != null) {
// Delete registertemp dir since it can be corrupted
if (registerPath.length() > 20) // Safety check
commander.c_api.removeLFN(registerPath, true);
}
else {
System.out.println("[" + new Date() + "] " + "registertemp is not there, continue with the main procedure");
}
// Continue basic checks
// Check if we are able to get PFN list // Check if we are able to get PFN list
List<PFN> remotePFN = null; List<PFN> remotePFN = null;
try { try {
...@@ -149,8 +178,6 @@ public class ArchiveMemberDelete { ...@@ -149,8 +178,6 @@ public class ArchiveMemberDelete {
return; return;
} }
System.out.println("[" + new Date() + "] Processing " + remoteFile);
// If the file is not a member of any archives, just delete it // If the file is not a member of any archives, just delete it
if (remoteLFN.equals(remoteArchiveLFN)) { if (remoteLFN.equals(remoteArchiveLFN)) {
System.out.println("[" + new Date() + "] " + remoteFile + " is a real file, we'll simply delete it"); System.out.println("[" + new Date() + "] " + remoteFile + " is a real file, we'll simply delete it");
...@@ -170,7 +197,7 @@ public class ArchiveMemberDelete { ...@@ -170,7 +197,7 @@ public class ArchiveMemberDelete {
} }
commander.c_api.removeLFN(remoteFile); commander.c_api.removeLFN(remoteFile);
commander.c_api.touchLFN(remoteLFN.getParentName() + System.getProperty("file.separator") + ".deleted" + remoteFileSize); commander.c_api.touchLFN(parentdir + "/" + ".deleted" + remoteFileSize);
System.out.println("[" + new Date() + "] " + remoteFile); System.out.println("[" + new Date() + "] " + remoteFile);
System.out.println("[" + new Date() + "] Reclaimed " + remoteFileSize + " bytes of disk space"); System.out.println("[" + new Date() + "] Reclaimed " + remoteFileSize + " bytes of disk space");
...@@ -184,7 +211,7 @@ public class ArchiveMemberDelete { ...@@ -184,7 +211,7 @@ public class ArchiveMemberDelete {
final List<PFN> remoteArchivePFN = Dispatcher.execute(new PFNforReadOrDel(commander.getUser(), commander.getSite(), AccessType.DELETE, remoteArchiveLFN, null, null)).getPFNs(); final List<PFN> remoteArchivePFN = Dispatcher.execute(new PFNforReadOrDel(commander.getUser(), commander.getSite(), AccessType.DELETE, remoteArchiveLFN, null, null)).getPFNs();
if (remoteArchivePFN.size() == 0) { if (remoteArchivePFN.size() == 0) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Archive is orphaned"); System.err.println("[" + new Date() + "] " + remoteFile + ": Archive is orphaned");
validation.println("Orphaned archive"); validation.println("Orphaned archive " + remoteFile);
return; return;
} }
...@@ -194,6 +221,11 @@ public class ArchiveMemberDelete { ...@@ -194,6 +221,11 @@ public class ArchiveMemberDelete {
final long jobID = remoteArchiveLFN.jobid; final long jobID = remoteArchiveLFN.jobid;
final List<LFN> remoteArchiveMembers = commander.c_api.getArchiveMembers(remoteArchive); final List<LFN> remoteArchiveMembers = commander.c_api.getArchiveMembers(remoteArchive);
if (remoteArchiveMembers == null || remoteArchiveMembers.isEmpty()) {
System.err.println("[" + new Date() + "] Failed to get members of the remote archive: " + remoteArchive);
validation.println("Failed to get members of " + remoteArchive);
return;
}
if (remoteArchiveMembers.size() == 1) { if (remoteArchiveMembers.size() == 1) {
// RemoteLFN is the only file in remoteArchive // RemoteLFN is the only file in remoteArchive
// No point in downloading it, just remove file and archive // No point in downloading it, just remove file and archive
...@@ -221,7 +253,7 @@ public class ArchiveMemberDelete { ...@@ -221,7 +253,7 @@ public class ArchiveMemberDelete {
commander.c_api.removeLFN(remoteArchive); commander.c_api.removeLFN(remoteArchive);
// Create file marker to leave trace // Create file marker to leave trace
commander.c_api.touchLFN(remoteLFN.getParentName() + System.getProperty("file.separator") + ".deleted" + remoteArchiveLFN.getSize()); commander.c_api.touchLFN(parentdir + "/" + ".deleted" + remoteArchiveLFN.getSize());
System.out.println("[" + new Date() + "] " + memberName + " was " + remoteLFN.getSize() + " bytes"); System.out.println("[" + new Date() + "] " + memberName + " was " + remoteLFN.getSize() + " bytes");
System.out.println("[" + new Date() + "] " + "Old archive was " + remoteArchiveLFN.getSize() + " bytes"); System.out.println("[" + new Date() + "] " + "Old archive was " + remoteArchiveLFN.getSize() + " bytes");
...@@ -241,7 +273,7 @@ public class ArchiveMemberDelete { ...@@ -241,7 +273,7 @@ public class ArchiveMemberDelete {
commander.c_api.downloadFile(remoteArchive, localArchive, "-silent"); commander.c_api.downloadFile(remoteArchive, localArchive, "-silent");
if (!localArchive.exists()) { if (!localArchive.exists()) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to download remote archive " + remoteArchive); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to download remote archive " + remoteArchive);
validation.println("Download failed"); validation.println("Download failed " + remoteArchive);
return; return;
} }
...@@ -251,7 +283,7 @@ public class ArchiveMemberDelete { ...@@ -251,7 +283,7 @@ public class ArchiveMemberDelete {
if (!unzip(archiveName, memberName)) { if (!unzip(archiveName, memberName)) {
System.err.println( System.err.println(
"[" + new Date() + "] " + remoteFile + ": Failed to extract files from archive: " + System.getProperty("user.dir") + System.getProperty("file.separator") + archiveName); "[" + new Date() + "] " + remoteFile + ": Failed to extract files from archive: " + System.getProperty("user.dir") + System.getProperty("file.separator") + archiveName);
validation.println("Extraction failed"); validation.println("Extraction failed " + remoteArchive);
return; return;
} }
localArchive.delete(); localArchive.delete();
...@@ -262,8 +294,7 @@ public class ArchiveMemberDelete { ...@@ -262,8 +294,7 @@ public class ArchiveMemberDelete {
final OutputEntry entry = new OutputEntry(archiveName, listOfFiles, "", Long.valueOf(jobID)); final OutputEntry entry = new OutputEntry(archiveName, listOfFiles, "", Long.valueOf(jobID));
entry.createZip(System.getProperty("user.dir") + System.getProperty("file.separator") + "extracted"); entry.createZip(System.getProperty("user.dir") + System.getProperty("file.separator") + "extracted");
final String registerPath = remoteArchiveLFN.getParentName() + System.getProperty("file.separator") + "registertemp"; final String newArchiveFullPath = registerPath + "/" + archiveName;
final String newArchiveFullPath = registerPath + System.getProperty("file.separator") + archiveName;
// Upload the new archive to the Grid // Upload the new archive to the Grid
// //
...@@ -271,31 +302,30 @@ public class ArchiveMemberDelete { ...@@ -271,31 +302,30 @@ public class ArchiveMemberDelete {
final File newArchive = new File(System.getProperty("user.dir") + System.getProperty("file.separator") + "extracted" + System.getProperty("file.separator") + archiveName); final File newArchive = new File(System.getProperty("user.dir") + System.getProperty("file.separator") + "extracted" + System.getProperty("file.separator") + archiveName);
LFN newArchiveLFN = commander.c_api.getLFN(newArchiveFullPath);
if (newArchiveLFN != null) {
// Delete registertemp/root_archive.zip if there is any
commander.c_api.removeLFN(newArchiveFullPath);
}
commander.c_api.uploadFile(newArchive, newArchiveFullPath, "-w", "-S", "disk:1"); // Create only one replica commander.c_api.uploadFile(newArchive, newArchiveFullPath, "-w", "-S", "disk:1"); // Create only one replica
final LFN newArchiveLFN = commander.c_api.getLFN(newArchiveFullPath); newArchiveLFN = commander.c_api.getLFN(newArchiveFullPath);
if (newArchiveLFN == null || !newArchiveLFN.exists) { if (newArchiveLFN == null || !newArchiveLFN.exists) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to upload archive " + newArchiveFullPath); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to upload archive " + newArchiveFullPath);
validation.println("Upload failed"); validation.println("Upload failed " + newArchiveFullPath);
return; return;
} }
// Register files in the catalogue // Register files in the catalogue
// //
// Create registertemp dir
if (commander.c_api.createCatalogueDirectory(registerPath) == null) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to create new directory " + registerPath);
validation.println("Mkdir failed");
return;
}
// Create subdirs (like BKG/) // Create subdirs (like BKG/)
for (final String file : listOfFiles) { for (final String file : listOfFiles) {
if (file.contains("/")) { if (file.contains("/")) {
if (commander.c_api.createCatalogueDirectory(registerPath + System.getProperty("file.separator") + file.substring(0, file.lastIndexOf("/")), true) == null) { if (commander.c_api.createCatalogueDirectory(registerPath + "/" + file.substring(0, file.lastIndexOf("/")), true) == null) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to create new directory " + registerPath + System.getProperty("file.separator") System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to create new directory " + registerPath + "/" + file.substring(0, file.lastIndexOf("/")));
+ file.substring(0, file.lastIndexOf("/"))); validation.println("Mkdir failed " + registerPath + "/" + file.substring(0, file.lastIndexOf("/")));
validation.println("Mkdir failed");
// Delete all newly created entries and directories // Delete all newly created entries and directories
if (registerPath.length() > 20) // Safety check if (registerPath.length() > 20) // Safety check
...@@ -306,13 +336,13 @@ public class ArchiveMemberDelete { ...@@ -306,13 +336,13 @@ public class ArchiveMemberDelete {
} }
System.out.println("[" + new Date() + "] Registering files in the catalogue"); System.out.println("[" + new Date() + "] Registering files in the catalogue");
CatalogueApiUtils.registerEntry(entry, registerPath + System.getProperty("file.separator"), commander.getUser()); CatalogueApiUtils.registerEntry(entry, registerPath + "/", commander.getUser());
for (final String file : listOfFiles) { for (final String file : listOfFiles) {
final LFN entryLFN = commander.c_api.getLFN(registerPath + System.getProperty("file.separator") + file); final LFN entryLFN = commander.c_api.getLFN(registerPath + "/" + file);
if (entryLFN == null || !entryLFN.exists) { if (entryLFN == null || !entryLFN.exists) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to register entry " + registerPath + System.getProperty("file.separator") + file); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to register entry " + registerPath + "/" + file);
validation.println("Register failed"); validation.println("Register failed " + registerPath + "/" + file);
// Delete all newly created entries and directories // Delete all newly created entries and directories
if (registerPath.length() > 20) // Safety check if (registerPath.length() > 20) // Safety check
...@@ -328,7 +358,7 @@ public class ArchiveMemberDelete { ...@@ -328,7 +358,7 @@ public class ArchiveMemberDelete {
System.out.println("[" + new Date() + "] Deleting " + member.getCanonicalName()); System.out.println("[" + new Date() + "] Deleting " + member.getCanonicalName());
if (!commander.c_api.removeLFN(member.getCanonicalName())) { if (!commander.c_api.removeLFN(member.getCanonicalName())) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to delete old archive member " + member.getCanonicalName()); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to delete old archive member " + member.getCanonicalName());
validation.println("Archive member deletion failed"); validation.println("Archive member deletion failed " + remoteArchive);
// Delete all newly created entries and directories // Delete all newly created entries and directories
if (registerPath.length() > 20) // Safety check if (registerPath.length() > 20) // Safety check
...@@ -344,7 +374,7 @@ public class ArchiveMemberDelete { ...@@ -344,7 +374,7 @@ public class ArchiveMemberDelete {
// Remove lfn of the old archive // Remove lfn of the old archive
if (!commander.c_api.removeLFN(remoteArchive)) { if (!commander.c_api.removeLFN(remoteArchive)) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to delete old archive " + remoteArchive); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to delete old archive " + remoteArchive);
validation.println("Archive deletion failed"); validation.println("Archive deletion failed " + remoteArchive);
// Delete all newly created entries and directories // Delete all newly created entries and directories
if (registerPath.length() > 20) // Safety check if (registerPath.length() > 20) // Safety check
...@@ -368,22 +398,24 @@ public class ArchiveMemberDelete { ...@@ -368,22 +398,24 @@ public class ArchiveMemberDelete {
} }
// Create file marker to leave trace // Create file marker to leave trace
commander.c_api.touchLFN(remoteLFN.getParentName() + System.getProperty("file.separator") + ".deleted" + (remoteArchiveLFN.getSize() - newArchive.length())); if (commander.c_api.touchLFN(parentdir + "/" + ".deleted" + (remoteArchiveLFN.getSize() - newArchive.length())) == null) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Could not create .deleted marker");
}
// Rename uploaded archive // Rename uploaded archive
// //
System.out.println("[" + new Date() + "] Renaming uploaded archive"); System.out.println("[" + new Date() + "] Renaming uploaded archive");
if (commander.c_api.moveLFN(registerPath + System.getProperty("file.separator") + archiveName, remoteArchive) == null) { if (commander.c_api.moveLFN(registerPath + "/" + archiveName, remoteArchive) == null) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to rename the archive " + registerPath + System.getProperty("file.separator") + archiveName); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to rename the archive " + registerPath + "/" + archiveName);
validation.println("Renaming failed"); validation.println("Renaming failed " + registerPath + "/" + archiveName);
return; return;
} }
// Rename new archive members // Rename new archive members
for (final String file : listOfFiles) { for (final String file : listOfFiles) {
if (commander.c_api.moveLFN(registerPath + System.getProperty("file.separator") + file, remoteArchiveLFN.getParentName() + System.getProperty("file.separator") + file) == null) { if (commander.c_api.moveLFN(registerPath + "/" + file, parentdir + "/" + file) == null) {
System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to rename archive member " + remoteArchiveLFN.getParentName() + System.getProperty("file.separator") + file); System.err.println("[" + new Date() + "] " + remoteFile + ": Failed to rename archive member " + parentdir + "/" + file);
validation.println("Renaming failed"); validation.println("Renaming failed " + registerPath + "/" + file);
return; return;
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment