Commit e7fb42f9 authored by Tim Hallyburton's avatar Tim Hallyburton
Browse files

Merge branch 'master' into logging_rework

parents 861ba305 c7844e62
......@@ -20,7 +20,7 @@ cd ../build_eclipse
cp ../trusted_authorities.jks .
mkdir config
mkdir -p config
cp ../config/config.properties ../config/monitoring.properties config/
......
......@@ -31,19 +31,13 @@ public class JCentral {
*/
public static void main(final String[] args) throws KeyStoreException {
try {
JAKeyStore.loadServerKeyStorage();
} catch (final Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
logger.setLevel(Level.WARNING);
Context.addToLoggingContext("JCentral");
try {
// SimpleCatalogueApiService catalogueAPIService = new SimpleCatalogueApiService();
// catalogueAPIService.start();
JAKeyStore.loadKeyStore();
TomcatServer.startTomcatServer(0);
DispatchSSLServer.runService();
......
......@@ -139,7 +139,6 @@ public class TomcatServer {
Connector connector = new Connector("org.apache.coyote.http11.Http11NioProtocol");
connector.setProtocol("org.apache.coyote.http11.Http11NioProtocol");
connector.setPort(tomcatPort);
connector.setSecure(true);
connector.setScheme("https");
......
......@@ -64,7 +64,6 @@ public class GetTokenCertificate extends Request {
final TokenCertificateType certificateType;
final String extension;
final int validity;
final X509Certificate userCertificate;
final String requestedUser;
// incoming fields
......@@ -81,17 +80,15 @@ public class GetTokenCertificate extends Request {
* @param certificateType
* @param extension
* @param validity
* @param userCertificate
* the certificate the user presented to identify itself. This
* will restrict the validity of the issued token
*/
public GetTokenCertificate(final AliEnPrincipal user, final String requestedUser, final TokenCertificateType certificateType, final String extension, final int validity, final X509Certificate userCertificate) {
setRequestUser(user);
public GetTokenCertificate(final AliEnPrincipal user, final String requestedUser, final TokenCertificateType certificateType, final String extension, final int validity) {
setRequestUser(user);
this.certificateType = certificateType;
this.extension = extension;
this.validity = validity;
this.userCertificate = userCertificate;
this.requestedUser = requestedUser;
}
......@@ -101,7 +98,7 @@ public class GetTokenCertificate extends Request {
throw new IllegalArgumentException("Certificate type cannot be null");
DnBuilder builder = CA.dn().setC("ch").setO("AliEn");
final String requester = getEffectiveRequester().getDefaultUser();
final String requested = getEffectiveRequester().canBecome(requestedUser) ? requestedUser : requester;
......@@ -110,9 +107,6 @@ public class GetTokenCertificate extends Request {
if (getEffectiveRequester().isJob() || getEffectiveRequester().isJobAgent())
throw new IllegalArgumentException("You can't request a User token as JobAgent or Job");
if (userCertificate == null)
throw new IllegalArgumentException("When issuing a user certificate you need to pass the current one, that will limit the validity of the issued token");
builder = builder.setCn("Users").setCn(requester).setOu(requested);
break;
case JOB_TOKEN:
......@@ -145,12 +139,15 @@ public class GetTokenCertificate extends Request {
ZonedDateTime notAfter = ZonedDateTime.now().plus(amount);
if (userCertificate != null) {
final ZonedDateTime userNotAfter = userCertificate.getNotAfter().toInstant().atZone(ZoneId.systemDefault());
if (getEffectiveRequester().getUserCert() != null) {
final ZonedDateTime userNotAfter = getEffectiveRequester().getUserCert()[0].getNotAfter().toInstant().atZone(ZoneId.systemDefault());
if (notAfter.isAfter(userNotAfter))
notAfter = userNotAfter;
}
else {
throw new IllegalArgumentException("When issuing a user certificate you need to pass the current one, that will limit the validity of the issued token");
}
final javax.security.cert.X509Certificate partnerCertificateChain[] = getPartnerCertificate();
......@@ -223,21 +220,22 @@ public class GetTokenCertificate extends Request {
}
return sw.toString();
}
/*
@Override
public String getKey() {
// only cache user tokens, job tokens have the job ID in them and cannot
// be effectively cached
if (certificateType == TokenCertificateType.USER_CERTIFICATE)
return getEffectiveRequester().getName();
return null;
}
@Override
public long getTimeout() {
// for the same user don't generate another certificate for 10 minutes
// but return the same one
return 1000L * 60 * 10;
}*/
/*
* @Override
* public String getKey() {
* // only cache user tokens, job tokens have the job ID in them and cannot
* // be effectively cached
* if (certificateType == TokenCertificateType.USER_CERTIFICATE)
* return getEffectiveRequester().getName();
*
* return null;
* }
*
* @Override
* public long getTimeout() {
* // for the same user don't generate another certificate for 10 minutes
* // but return the same one
* return 1000L * 60 * 10;
* }
*/
}
......@@ -28,11 +28,11 @@ public class GetMatchJob extends Request {
public GetMatchJob(final AliEnPrincipal user, final HashMap<String, Object> siteMap) {
setRequestUser(user);
this.matchRequest = siteMap;
this.matchRequest.put("UserCertificate", this.getRequesterIdentity().getUserCert()[0]);
}
@Override
public void run() {
this.matchRequest.put("AliEnPrincipal", this.getEffectiveRequester());
this.match = JobBroker.getMatchJob(matchRequest);
}
......
......@@ -278,10 +278,6 @@ public class BusyBox {
boolean signal = false;
while ((sLine = br.readLine()) != null) {
if (sLine.startsWith(JShPrintWriter.degradedSignal)) {
printJCentralConnError();
break;
}
signal = true;
if (sLine.startsWith(JShPrintWriter.outputterminator))
updateEnvironment(sLine);
......@@ -331,10 +327,6 @@ public class BusyBox {
boolean signal = false;
while ((sLine = br.readLine()) != null) {
if (sLine.startsWith(JShPrintWriter.degradedSignal)) {
printJCentralConnError();
break;
}
signal = true;
......@@ -587,9 +579,4 @@ public class BusyBox {
private static void printConnError() {
JSh.printErr("Connection to JBox interrupted.");
}
private static void printJCentralConnError() {
JSh.printErr("Connection error to JCentral.");
}
}
......@@ -30,10 +30,6 @@ import lazyj.Format;
*/
public class JAliEnCOMMander extends Thread {
private final static int maxTryConnect = 3;
private int triedConnects = 0;
/**
* Logger
*/
......@@ -53,10 +49,9 @@ public class JAliEnCOMMander extends Thread {
* The commands that have a JAliEnCommand* implementation
*/
private static final String[] jAliEnCommandList = new String[] { "ls", "get", "cat", "whereis", "cp", "cd", "time", "mkdir", "find", "listFilesFromCollection", "scrlog", "submit", "motd",
"access", "commit", "packages", "pwd", "ps", "rmdir", "rm", "mv", "masterjob", "user", "touch", "type", "kill", "lfn2guid", "guid2lfn", "w", "uptime", "addFileToCollection",
"addMirror", "addTag", "addTagValue", "chgroup", "chown", "createCollection", "deleteMirror", "df", "du", "fquota", "jobinfo", "jquota", "killTransfer", "listSEDistance", "listTransfer",
"md5sum", "mirror", "queue", "queueinfo", "register", "registerOutput", "removeTag", "removeTagValue", "resubmit", "resubmitTransfer", "showTags", "showTagValue", "spy", "top", "groups",
"token" };
"access", "commit", "packages", "pwd", "ps", "rmdir", "rm", "mv", "masterjob", "user", "touch", "type", "kill", "lfn2guid", "guid2lfn", "w", "uptime", "addFileToCollection", "addMirror",
"addTag", "addTagValue", "chgroup", "chown", "createCollection", "deleteMirror", "df", "du", "fquota", "jobinfo", "jquota", "killTransfer", "listSEDistance", "listTransfer", "md5sum",
"mirror", "queue", "queueinfo", "register", "registerOutput", "removeTag", "removeTagValue", "resubmit", "resubmitTransfer", "showTags", "showTagValue", "spy", "top", "groups", "token" };
private static final String[] jAliEnAdminCommandList = new String[] { "addTrigger", "addHost", "queue", "register", "addSE", "addUser", "calculateFileQuota", "calculateJobQuota", "groupmembers" };
......@@ -98,8 +93,6 @@ public class JAliEnCOMMander extends Thread {
private final HashMap<String, File> localFileCash;
private boolean degraded = false;
private static JAliEnCOMMander lastInstance = null;
/**
......@@ -131,33 +124,30 @@ public class JAliEnCOMMander extends Thread {
this.user = (user != null) ? user : AuthorizationFactory.getDefaultUser();
this.site = (site != null) ? site : ConfigUtils.getConfig().gets("alice_close_site").trim();
myHome = UsersHelper.getHomeDir(this.user.getName());
localFileCash = new HashMap<>();
this.out = out;
degraded = false;
if (curDir == null)
initializeJCentralConnection();
else
if (this.user.isJobAgent()) {
// For job agents we do not care about directories
myHome = "";
this.curDir = curDir;
}
else {
// User directories must be set correctly
myHome = UsersHelper.getHomeDir(this.user.getName());
if (curDir == null)
try {
this.curDir = c_api.getLFN(myHome);
} catch (final Exception e) {
logger.log(Level.WARNING, "Exception initializing connection", e);
}
else
this.curDir = curDir;
}
setName("Commander");
}
private boolean initializeJCentralConnection() {
triedConnects++;
try {
curDir = c_api.getLFN(myHome);
degraded = false;
} catch (final Exception e) {
logger.log(Level.WARNING, "Exception initializing connection", e);
degraded = true;
}
return !degraded;
}
/**
* @param md5
* @param localFile
......@@ -245,10 +235,7 @@ public class JAliEnCOMMander extends Thread {
* @return LFN of the current directory
*/
public LFN getCurrentDir() {
// if (curDir == null)
// curDir = c_api.getLFN(myHome);
return curDir;
}
/**
......@@ -307,44 +294,22 @@ public class JAliEnCOMMander extends Thread {
if (kill)
break;
if (degraded) {
if (triedConnects < maxTryConnect) {
if (!initializeJCentralConnection())
flush();
if (triedConnects < maxTryConnect)
try {
Thread.sleep(triedConnects * maxTryConnect * 500);
} catch (final InterruptedException e) {
e.printStackTrace();
}
}
else {
System.out.println("Giving up...");
break;
}
}
else {
waitForCommand();
if (kill)
break;
try {
status.set(1);
try {
status.set(1);
setName("Commander: Executing: " + Arrays.toString(arg));
setName("Commander: Executing: " + Arrays.toString(arg));
execute();
} catch (final Exception e) {
logger.log(Level.WARNING, "Got exception", e);
}finally {
out = null;
execute();
} catch (final Exception e) {
logger.log(Level.WARNING, "Got exception", e);
} finally {
out = null;
setName("Commander: Idle");
setName("Commander: Idle");
status.set(0);
synchronized (status) {
status.notifyAll();
}
status.set(0);
synchronized (status) {
status.notifyAll();
}
}
}
......@@ -484,12 +449,7 @@ public class JAliEnCOMMander extends Thread {
* flush the buffer and produce status to be send to client
*/
public void flush() {
if (degraded) {
out.degraded();
out.setenv(UsersHelper.getHomeDir(user.getName()), getUsername());
}
else
out.setenv(getCurrentDirName(), getUsername());
out.setenv(getCurrentDirName(), getUsername());
out.flush();
}
......@@ -528,10 +488,9 @@ public class JAliEnCOMMander extends Thread {
} catch (@SuppressWarnings("unused") final ClassNotFoundException e) {
// System.out.println("No such command or not implemented");
return null;
}
catch (final java.lang.reflect.InvocationTargetException e) {
} catch (final java.lang.reflect.InvocationTargetException e) {
logger.log(Level.SEVERE, "Exception running command", e);
return null;
}
}
}
}
\ No newline at end of file
......@@ -133,15 +133,12 @@ public class JAliEnCommandaccess extends JAliEnBaseCommand {
else
out.setField("envelope", pfn.ticket.envelope.getSignedEnvelope());
out.setField("url", pfn.ticket.envelope.getTransactionURL());
out.setField("pfn", pfn.toString());
out.setField( "url", pfn.ticket.envelope.getTransactionURL());
out.setField( "pfn", pfn.toString());
out.setField("guid", pfn.getGuid().getName());
out.setField("se", pfn.getSE().getName());
if (accessRequest.equals(AccessType.WRITE))
out.setField("nSEs", "1");
else
out.setField("nSEs", " " + pfns.size());
out.setField( "se", pfn.getSE().getName());
out.setField("tags", pfn.getSE().qos.toString());
out.setField("nSEs", String.valueOf(pfns.size()));
out.setField("user", commander.user.getName());
}
}
......
......@@ -58,7 +58,7 @@ public class JAliEnCommandfind extends JAliEnBaseCommand {
private boolean bC = false;
/**
* marker for -l argument :limit the number of returned files per database host
* marker for -l argument : long format, optionally human readable file sizes
*/
private boolean bL = false;
......@@ -158,10 +158,11 @@ public class JAliEnCommandfind extends JAliEnBaseCommand {
else
for (final LFN lfn : lfns)
if (out.isRootPrinter()) {
out.nextResult();
if (bL) {
out.setField("perm", FileSystemUtils.getFormatedTypeAndPerm(lfn));
out.setField("owner", lfn.owner);
out.setField("permissions", FileSystemUtils.getFormatedTypeAndPerm(lfn));
out.setField("user", lfn.owner);
out.setField("group", lfn.gowner);
out.setField("size", (bH ? Format.size(lfn.size) : String.valueOf(lfn.size)));
out.setField("ctime", " " + lfn.ctime);
......@@ -175,7 +176,7 @@ public class JAliEnCommandfind extends JAliEnBaseCommand {
if (bL)
// print long
out.printOutln(FileSystemUtils.getFormatedTypeAndPerm(lfn) + padSpace(3) + padLeft(lfn.owner, 8) + padSpace(1) + padLeft(lfn.gowner, 8) + padSpace(1)
+ padLeft(bH ? Format.size(lfn.size) : String.valueOf(lfn.size), 12) + format(lfn.ctime) + padSpace(1) + padSpace(4) + lfn.getCanonicalName());
+ padLeft(bH ? Format.size(lfn.size) : String.valueOf(lfn.size), 12) + padSpace(1) + format(lfn.ctime) + padSpace(1) + padSpace(4) + lfn.getCanonicalName());
else
out.printOutln(lfn.getCanonicalName());
......@@ -195,15 +196,14 @@ public class JAliEnCommandfind extends JAliEnBaseCommand {
out.printOutln();
out.printOutln(helpUsage("find", "<path> <pattern> flags"));
out.printOutln();
// out.printOutln(helpUsage("Possible flags are coming soon..."));
out.printOutln(helpStartOptions());
out.printOutln(helpOption("-a", "show hidden .* files"));
out.printOutln(helpOption("-s", "no sorting"));
out.printOutln(helpOption("-c", "c filename"));
out.printOutln(helpOption("-c", "collection filename (put the output in a collection)"));
out.printOutln(helpOption("-y", "(FOR THE OCDB) return only the biggest version of each file"));
out.printOutln(helpOption("-x", "x collection name"));
out.printOutln(helpOption("-x", "xml collection name (return the LFN list through XmlCollection)"));
out.printOutln(helpOption("-d", "return also the directories"));
out.printOutln(helpOption("-l[h]", "long format, optionally human readable file sizes"));
out.printOutln();
......
......@@ -107,15 +107,16 @@ public class JAliEnCommandls extends JAliEnBaseCommand {
if (out.isRootPrinter()) {
out.nextResult();
if (localLFN.guid != null)
out.setField("guid", localLFN.guid.toString().toUpperCase());
out.setField("perm", FileSystemUtils.getFormatedTypeAndPerm(localLFN));
out.setField("owner", localLFN.owner);
out.setField("permissions", FileSystemUtils.getFormatedTypeAndPerm(localLFN));
out.setField("user", localLFN.owner);
out.setField("group", localLFN.gowner);
out.setField("size", String.valueOf(localLFN.size));
out.setField("ctime", String.valueOf(localLFN.ctime.getTime() / 1000));
out.setField("name", localLFN.getFileName() + (bF && localLFN.isDirectory() ? "/" : ""));
out.setField("path", localLFN.getCanonicalName() + (bF && localLFN.isDirectory() ? "/" : ""));
if (localLFN.guid != null)
out.setField("guid", localLFN.guid.toString().toUpperCase());
}
else {
String ret = "";
......
......@@ -80,10 +80,7 @@ public class JAliEnCommandps extends JAliEnBaseCommand {
out.setField("value ", tracelog);
out.setField("message: ", "not implemented yet");
}
else {
logger.log(Level.INFO, "bad getJDL, bad tracelog");
if (users.size() == 0)
users.add(commander.getUsername());
......@@ -142,11 +139,11 @@ public class JAliEnCommandps extends JAliEnBaseCommand {
}
else {
if (users.size() == 0)
users.add(commander.getUsername());
final List<Job> ps = commander.q_api.getPS(states, users, sites, nodes, mjobs, jobid, orderByKey, limit);
logger.log(Level.INFO, "ps " + ps);
if (ps != null)
for (final Job j : ps) {
......
......@@ -73,7 +73,7 @@ public class JAliEnCommandtoken extends JAliEnBaseCommand {
@Override
public void run() {
GetTokenCertificate tokenreq = new GetTokenCertificate(commander.user, requestedUser, tokentype, extension, validity, commander.user.getUserCert()[0]);
GetTokenCertificate tokenreq = new GetTokenCertificate(commander.user, requestedUser, tokentype, extension, validity);
try {
tokenreq = Dispatcher.execute(tokenreq);
......
......@@ -2,6 +2,7 @@ package alien.shell.commands;
import java.io.IOException;
import java.io.OutputStream;
import java.util.LinkedHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
......@@ -31,7 +32,7 @@ public class JSONPrintWriter extends UIPrintWriter {
private final JSONArray resultArray;
private final JSONObject metadataResult;
private JSONObject currentResult;
private LinkedHashMap<String, String> currentResult;
/**
* @param os
......@@ -107,12 +108,6 @@ public class JSONPrintWriter extends UIPrintWriter {
}
@Override
protected void degraded() {
// TODO Auto-generated method stub
}
@SuppressWarnings("unchecked")
@Override
void nextResult() {
......@@ -122,11 +117,10 @@ public class JSONPrintWriter extends UIPrintWriter {
}
}
@SuppressWarnings("unchecked")
@Override
void setField(final String key, final String value) {
if (currentResult == null)
currentResult = new JSONObject();
currentResult = new LinkedHashMap<>();
currentResult.put(key, value);
}
......
......@@ -18,11 +18,6 @@ public class JShPrintWriter extends UIPrintWriter {
*/
static transient final Logger logger = ConfigUtils.getLogger(JShPrintWriter.class.getCanonicalName());
/**
*
*/
public static final String degradedSignal = String.valueOf((char) 25);
/**
*
*/
......@@ -131,11 +126,6 @@ public class JShPrintWriter extends UIPrintWriter {
print(pendingSignal + "\n");
}
@Override
protected void degraded() {
print(degradedSignal);
}
@Override
void nextResult() {
// ignored
......
......@@ -121,11 +121,6 @@ public class PlainWriter extends UIPrintWriter {
// ignore
}
@Override
protected void degraded() {
// ignore
}
@Override
void nextResult() {
// ignored
......
......@@ -169,11 +169,6 @@ public class RootPrintWriter extends UIPrintWriter {
// ignore - not to be implemented in the root printer
}
@Override
protected void degraded() {
// ignore - not to be implemented in the root printer
}
@Override
protected void blackwhitemode() {
// ignore - not to be implemented in the root printer
......
......@@ -88,11 +88,6 @@ public abstract class UIPrintWriter {
*/
abstract protected void pending();
/**
*
*/
abstract protected void degraded();
/**
* identify the RootPrinter from above
*
......@@ -108,7 +103,7 @@ public abstract class UIPrintWriter {
*
* @param args
*/
protected void setReturnArgs(@SuppressWarnings(