use of es.bsc.compss.types.annotations.parameter.DataType in project compss by bsc-wdc.
the class COMPSsRuntimeImpl method processParameters.
/*
* ************************************************************************************************************
* PRIVATE HELPER METHODS
* ********************************************************************************************************
*/
private Parameter[] processParameters(int parameterCount, Object[] parameters) {
Parameter[] pars = new Parameter[parameterCount];
// Parameter parsing needed, object is not serializable
int i = 0;
for (int npar = 0; npar < parameterCount; ++npar) {
DataType type = (DataType) parameters[i + 1];
Direction direction = (Direction) parameters[i + 2];
Stream stream = (Stream) parameters[i + 3];
String prefix = (String) parameters[i + 4];
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(" Parameter " + (npar + 1) + " has type " + type.name());
}
switch(type) {
case FILE_T:
try {
String fileName = (String) parameters[i];
String originalName = new File(fileName).getName();
DataLocation location = createLocation((String) parameters[i]);
pars[npar] = new FileParameter(direction, stream, prefix, location, originalName);
} catch (Exception e) {
LOGGER.error(ERROR_FILE_NAME, e);
ErrorManager.fatal(ERROR_FILE_NAME, e);
}
break;
case PSCO_T:
case OBJECT_T:
pars[npar] = new ObjectParameter(direction, stream, prefix, parameters[i], oReg.newObjectParameter(parameters[i]));
break;
case EXTERNAL_OBJECT_T:
String id = (String) parameters[i];
pars[npar] = new ExternalObjectParameter(direction, stream, prefix, id, externalObjectHashcode(id));
break;
default:
/*
* Basic types (including String). The only possible direction is IN, warn otherwise
*/
if (direction != Direction.IN) {
LOGGER.warn(WARN_WRONG_DIRECTION + "Parameter " + npar + " is a basic type, therefore it must have IN direction");
}
pars[npar] = new BasicTypeParameter(type, Direction.IN, stream, prefix, parameters[i]);
break;
}
i += 5;
}
return pars;
}
use of es.bsc.compss.types.annotations.parameter.DataType in project compss by bsc-wdc.
the class GATJob method processParameters.
private void processParameters(String sandboxPath, ArrayList<String> symlinks, ArrayList<String> lArgs) {
logger.debug("Processing parameters for GAT job " + this.jobId);
lArgs.add(Boolean.toString(taskParams.hasTargetObject()));
// Add return type
if (taskParams.hasReturnValue()) {
Parameter returnParam = taskParams.getParameters()[taskParams.getParameters().length - 1];
lArgs.add(Integer.toString(returnParam.getType().ordinal()));
} else {
lArgs.add("null");
}
// Add parameters
int numParams = taskParams.getParameters().length;
if (taskParams.hasReturnValue()) {
numParams--;
}
lArgs.add(Integer.toString(numParams));
for (Parameter param : taskParams.getParameters()) {
DataType type = param.getType();
lArgs.add(Integer.toString(type.ordinal()));
lArgs.add(Integer.toString(param.getStream().ordinal()));
String prefix = param.getPrefix();
if (prefix == null || prefix.isEmpty()) {
prefix = Constants.PREFIX_EMTPY;
}
lArgs.add(param.getPrefix());
switch(type) {
case FILE_T:
DependencyParameter dFilePar = (DependencyParameter) param;
java.io.File f = new java.io.File(dFilePar.getDataTarget());
if (!f.getName().equals(dFilePar.getOriginalName())) {
// Add file to manage symlinks and renames
String originalName = sandboxPath + File.separator + dFilePar.getOriginalName();
symlinks.add(dFilePar.getDataTarget());
symlinks.add(dFilePar.getOriginalName());
lArgs.add(originalName);
} else {
// Original and target is the same nothing to do
lArgs.add(dFilePar.getDataTarget());
}
break;
case PSCO_T:
case EXTERNAL_OBJECT_T:
logger.error("GAT Adaptor does not support PSCO Types");
listener.jobFailed(this, JobEndStatus.SUBMISSION_FAILED);
break;
case OBJECT_T:
DependencyParameter dPar = (DependencyParameter) param;
DataAccessId dAccId = dPar.getDataAccessId();
lArgs.add(dPar.getDataTarget());
if (dAccId instanceof RAccessId) {
lArgs.add("R");
} else {
// for the worker to know it must write the object to disk
lArgs.add("W");
}
break;
case STRING_T:
BasicTypeParameter btParS = (BasicTypeParameter) param;
// Check spaces
String value = btParS.getValue().toString();
int numSubStrings = value.split(" ").length;
lArgs.add(Integer.toString(numSubStrings));
lArgs.add(value);
break;
default:
// Basic Types
BasicTypeParameter btParB = (BasicTypeParameter) param;
lArgs.add(btParB.getValue().toString());
break;
}
}
}
use of es.bsc.compss.types.annotations.parameter.DataType in project compss by bsc-wdc.
the class GATWorker method parseApplicationParameters.
/**
* Parses the application parameters
*
* @param args
* arg[L]: boolean is the method executed on a certain instance arg[L]: integer amount of parameters of
* the method arg[L+]: parameters of the method For each parameter: type: 0-10 (file, boolean, char,
* string, byte, short, int, long, float, double, object) [substrings: amount of substrings (only used
* when the type is string)] value: value for the parameter or the file where it is contained (for
* objects and files) [Direction: R/W (only used when the type is object)]
*/
private static void parseApplicationParameters(String[] args) {
// Variables
if (GATWorker.hasTarget) {
// The target object of the last parameter before the return value (if any)
GATWorker.types = new Class[GATWorker.numParams - 1];
GATWorker.values = new Object[GATWorker.numParams - 1];
} else {
GATWorker.types = new Class[GATWorker.numParams];
GATWorker.values = new Object[GATWorker.numParams];
}
GATWorker.streams = new Stream[GATWorker.numParams];
GATWorker.prefixes = new String[GATWorker.numParams];
GATWorker.isFile = new boolean[GATWorker.numParams];
GATWorker.mustWrite = new boolean[GATWorker.numParams];
GATWorker.renamings = new String[GATWorker.numParams];
// Parse the parameter types and values
DataType[] dataTypesEnum = DataType.values();
Stream[] dataStream = Stream.values();
int argPosition = GATWorker.initialAppParamsPosition;
for (int i = 0; i < GATWorker.numParams; i++) {
// We need to use wrapper classes for basic types, reflection will unwrap automatically
int argType_index = Integer.parseInt(args[argPosition]);
if (argType_index >= dataTypesEnum.length) {
ErrorManager.error(WARN_UNSUPPORTED_TYPE + argType_index);
}
DataType argType = dataTypesEnum[argType_index];
argPosition++;
int argStream_index = Integer.parseInt(args[argPosition]);
if (argStream_index >= dataStream.length) {
ErrorManager.error(WARN_UNSUPPORTED_STREAM + argStream_index);
}
GATWorker.streams[i] = dataStream[argStream_index];
argPosition++;
String prefix = args[argPosition];
if (prefix == null || prefix.isEmpty()) {
prefix = Constants.PREFIX_EMTPY;
}
GATWorker.prefixes[i] = prefix;
argPosition++;
switch(argType) {
case FILE_T:
GATWorker.types[i] = String.class;
GATWorker.values[i] = args[argPosition++];
break;
case OBJECT_T:
GATWorker.renamings[i] = (String) args[argPosition++];
GATWorker.mustWrite[i] = ((String) args[argPosition++]).equals("W");
retrieveObject(renamings[i], i);
break;
case PSCO_T:
GATWorker.renamings[i] = (String) args[argPosition++];
GATWorker.mustWrite[i] = ((String) args[argPosition++]).equals("W");
retrievePSCO(renamings[i], i);
break;
case EXTERNAL_OBJECT_T:
GATWorker.types[i] = String.class;
GATWorker.values[i] = args[argPosition++];
break;
case BOOLEAN_T:
GATWorker.types[i] = boolean.class;
GATWorker.values[i] = new Boolean(args[argPosition++]);
break;
case CHAR_T:
GATWorker.types[i] = char.class;
GATWorker.values[i] = new Character(args[argPosition++].charAt(0));
break;
case STRING_T:
GATWorker.types[i] = String.class;
int numSubStrings = Integer.parseInt(args[argPosition++]);
String aux = "";
for (int j = 0; j < numSubStrings; j++) {
if (j != 0) {
aux += " ";
}
aux += args[argPosition++];
}
GATWorker.values[i] = aux;
break;
case BYTE_T:
GATWorker.types[i] = byte.class;
GATWorker.values[i] = new Byte(args[argPosition++]);
break;
case SHORT_T:
GATWorker.types[i] = short.class;
GATWorker.values[i] = new Short(args[argPosition++]);
break;
case INT_T:
GATWorker.types[i] = int.class;
GATWorker.values[i] = new Integer(args[argPosition++]);
break;
case LONG_T:
GATWorker.types[i] = long.class;
GATWorker.values[i] = new Long(args[argPosition++]);
break;
case FLOAT_T:
GATWorker.types[i] = float.class;
GATWorker.values[i] = new Float(args[argPosition++]);
break;
case DOUBLE_T:
GATWorker.types[i] = double.class;
GATWorker.values[i] = new Double(args[argPosition++]);
break;
default:
ErrorManager.error(WARN_UNSUPPORTED_TYPE + argType);
return;
}
GATWorker.isFile[i] = argType.equals(DataType.FILE_T);
}
// Retrieve return renaming if existing
if (GATWorker.hasReturn) {
// +1 = StreamType, +2 = prefix, +3 = value
GATWorker.retRenaming = args[argPosition + 3];
}
}
use of es.bsc.compss.types.annotations.parameter.DataType in project compss by bsc-wdc.
the class NIOAdaptor method receivedTaskDone.
@Override
public void receivedTaskDone(Connection c, NIOTaskResult tr, boolean successful) {
int taskId = tr.getTaskId();
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Received Task done message for Task " + taskId);
}
// Update running jobs
NIOJob nj = RUNNING_JOBS.remove(taskId);
// Update information
List<DataType> taskResultTypes = tr.getParamTypes();
for (int i = 0; i < taskResultTypes.size(); ++i) {
DataType newType = taskResultTypes.get(i);
switch(newType) {
case PSCO_T:
case EXTERNAL_OBJECT_T:
String pscoId = (String) tr.getParamValue(i);
DependencyParameter dp = (DependencyParameter) nj.getTaskParams().getParameters()[i];
updateParameter(newType, pscoId, dp);
break;
default:
// We only update information about PSCOs or EXTERNAL_OBJECTS
break;
}
}
// Update NIO Job
if (nj != null) {
// Mark task as finished and release waiters
JobHistory prevJobHistory = nj.getHistory();
nj.taskFinished(successful);
// Retrieve files if required
if (WORKER_DEBUG || !successful) {
String jobOut = JOBS_DIR + "job" + nj.getJobId() + "_" + prevJobHistory + ".out";
String jobErr = JOBS_DIR + "job" + nj.getJobId() + "_" + prevJobHistory + ".err";
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Requesting JobOut " + jobOut + " for Task " + taskId);
LOGGER.debug("Requesting JobErr " + jobErr + " for Task " + taskId);
}
c.receiveDataFile(jobOut);
c.receiveDataFile(jobErr);
}
/*if (!successful) {
String jobOut = JOBS_DIR + "job" + nj.getJobId() + "_" + newJobHistory + ".out";
String jobErr = JOBS_DIR + "job" + nj.getJobId() + "_" + newJobHistory + ".err";
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Requesting JobOut " + jobOut + " for Task " + taskId);
LOGGER.debug("Requesting JobErr " + jobErr + " for Task " + taskId);
}
c.receiveDataFile(jobOut);
c.receiveDataFile(jobErr);
}*/
}
// Close connection
c.finishConnection();
}
use of es.bsc.compss.types.annotations.parameter.DataType in project compss by bsc-wdc.
the class GATJob method prepareJob.
private JobDescription prepareJob() throws Exception {
// Get the information related to the job
logger.debug("Preparing GAT Job " + this.jobId);
TaskDescription taskParams = this.taskParams;
String targetPath = getResourceNode().getInstallDir();
String targetHost = getResourceNode().getHost();
String targetUser = getResourceNode().getUser();
if (userNeeded && !targetUser.isEmpty()) {
targetUser += "@";
} else {
targetUser = "";
}
SoftwareDescription sd = new SoftwareDescription();
sd.setExecutable(targetPath + WORKER_SCRIPT_PATH + WORKER_SCRIPT_NAME);
ArrayList<String> lArgs = new ArrayList<String>();
// Common arguments: language working_dir lib_path num_obsolete [obs1... obsN] tracing [event_type task_id
// slot_id]
lArgs.add(LANG);
lArgs.add(getResourceNode().getWorkingDir());
lArgs.add(getResourceNode().getLibPath());
LogicalData[] obsoleteFiles = getResource().pollObsoletes();
if (obsoleteFiles != null) {
lArgs.add("" + obsoleteFiles.length);
for (LogicalData ld : obsoleteFiles) {
String renaming = ld.getName();
lArgs.add(renaming);
}
} else {
lArgs.add("0");
}
// Check sandbox working dir
boolean isSpecific = false;
String sandboxDir = null;
AbstractMethodImplementation absImpl = (AbstractMethodImplementation) this.impl;
switch(absImpl.getMethodType()) {
case BINARY:
BinaryImplementation binaryImpl = (BinaryImplementation) absImpl;
sandboxDir = binaryImpl.getWorkingDir();
isSpecific = true;
break;
case MPI:
MPIImplementation mpiImpl = (MPIImplementation) absImpl;
sandboxDir = mpiImpl.getWorkingDir();
isSpecific = true;
break;
case DECAF:
DecafImplementation decafImpl = (DecafImplementation) absImpl;
sandboxDir = decafImpl.getWorkingDir();
isSpecific = true;
break;
case OMPSS:
OmpSsImplementation ompssImpl = (OmpSsImplementation) absImpl;
sandboxDir = ompssImpl.getWorkingDir();
isSpecific = true;
break;
case OPENCL:
OpenCLImplementation openclImpl = (OpenCLImplementation) absImpl;
sandboxDir = openclImpl.getWorkingDir();
isSpecific = true;
break;
case METHOD:
sandboxDir = null;
break;
}
if (sandboxDir == null || sandboxDir.isEmpty() || sandboxDir.equals(Constants.UNASSIGNED)) {
sandboxDir = getResourceNode().getWorkingDir() + File.separator + "sandBox" + File.separator + "job_" + this.jobId;
isSpecific = false;
}
// Processing parameters to get symlinks pairs to create (symlinks) and how to pass parameters in the GAT
// Job(paramArgs)
ArrayList<String> symlinks = new ArrayList<>();
ArrayList<String> paramArgs = new ArrayList<>();
processParameters(sandboxDir, symlinks, paramArgs);
// Adding info to create symlinks between renamed files and original names
lArgs.add(Boolean.toString(isSpecific));
lArgs.add(sandboxDir);
if (symlinks.size() > 0) {
lArgs.add(String.valueOf(symlinks.size()));
lArgs.addAll(symlinks);
} else {
lArgs.add("0");
}
lArgs.add(Boolean.toString(Tracer.isActivated()));
lArgs.add(getHostName());
if (debug) {
logger.debug("hostName " + getHostName());
}
if (Tracer.isActivated()) {
// event type
lArgs.add(String.valueOf(Tracer.getTaskEventsType()));
// task id
lArgs.add(String.valueOf(this.taskParams.getId() + 1));
int slot = Tracer.getNextSlot(targetHost);
// slot id
lArgs.add(String.valueOf(slot));
sd.addAttribute("slot", slot);
}
// Language-dependent arguments: taskSandbox_dir app_dir classpath pythonpath debug storage_conf
// method_impl_type method_impl_params
// numSlaves [slave1,..,slaveN] numCus
// has_target num_params par_type_1 par_1 ... par_type_n par_n
lArgs.add(sandboxDir);
lArgs.add(getResourceNode().getAppDir());
lArgs.add(getClasspath());
lArgs.add(getPythonpath());
lArgs.add(String.valueOf(debug));
lArgs.add(STORAGE_CONF);
lArgs.add(String.valueOf(absImpl.getMethodType()));
switch(absImpl.getMethodType()) {
case METHOD:
MethodImplementation methodImpl = (MethodImplementation) absImpl;
lArgs.add(methodImpl.getDeclaringClass());
String methodName = methodImpl.getAlternativeMethodName();
if (methodName == null || methodName.isEmpty()) {
methodName = taskParams.getName();
}
lArgs.add(methodName);
break;
case MPI:
MPIImplementation mpiImpl = (MPIImplementation) absImpl;
lArgs.add(mpiImpl.getMpiRunner());
lArgs.add(mpiImpl.getBinary());
break;
case DECAF:
DecafImplementation decafImpl = (DecafImplementation) absImpl;
lArgs.add(targetPath + DecafImplementation.SCRIPT_PATH);
String dfScript = decafImpl.getDfScript();
if (!dfScript.startsWith(File.separator)) {
String appPath = getResourceNode().getAppDir();
dfScript = appPath + File.separator + dfScript;
}
lArgs.add(dfScript);
String dfExecutor = decafImpl.getDfExecutor();
if (dfExecutor == null || dfExecutor.isEmpty() || dfExecutor.equals(Constants.UNASSIGNED)) {
dfExecutor = "executor.sh";
}
if (!dfExecutor.startsWith(File.separator) && !dfExecutor.startsWith("./")) {
dfExecutor = "./" + dfExecutor;
}
lArgs.add(dfExecutor);
String dfLib = decafImpl.getDfLib();
if (dfLib == null || dfLib.isEmpty()) {
dfLib = Constants.UNASSIGNED;
}
lArgs.add(dfLib);
lArgs.add(decafImpl.getMpiRunner());
break;
case OMPSS:
OmpSsImplementation ompssImpl = (OmpSsImplementation) absImpl;
lArgs.add(ompssImpl.getBinary());
break;
case OPENCL:
OpenCLImplementation openclImpl = (OpenCLImplementation) absImpl;
lArgs.add(openclImpl.getKernel());
break;
case BINARY:
BinaryImplementation binaryImpl = (BinaryImplementation) absImpl;
lArgs.add(binaryImpl.getBinary());
break;
}
// Slave nodes and cus description
lArgs.add(String.valueOf(slaveWorkersNodeNames.size()));
lArgs.addAll(slaveWorkersNodeNames);
lArgs.add(String.valueOf(((MethodResourceDescription) this.impl.getRequirements()).getTotalCPUComputingUnits()));
// Add parameter arguments already processed
lArgs.addAll(paramArgs);
// Conversion vector -> array
String[] arguments = new String[lArgs.size()];
arguments = lArgs.toArray(arguments);
try {
sd.setArguments(arguments);
} catch (NullPointerException e) {
StringBuilder sb = new StringBuilder("Null argument parameter of job " + this.jobId + " " + absImpl.getMethodDefinition() + "\n");
int i = 0;
for (Parameter param : taskParams.getParameters()) {
sb.append("Parameter ").append(i).append("\n");
DataType type = param.getType();
sb.append("\t Type: ").append(param.getType()).append("\n");
if (type == DataType.FILE_T || type == DataType.OBJECT_T) {
DependencyParameter dPar = (DependencyParameter) param;
DataAccessId dAccId = dPar.getDataAccessId();
sb.append("\t Target: ").append(dPar.getDataTarget()).append("\n");
if (type == DataType.OBJECT_T) {
if (dAccId instanceof RAccessId) {
sb.append("\t Direction: " + "R").append("\n");
} else {
// for the worker to know it must write the object to disk
sb.append("\t Direction: " + "W").append("\n");
}
}
} else if (type == DataType.STRING_T) {
BasicTypeParameter btParS = (BasicTypeParameter) param;
// Check spaces
String value = btParS.getValue().toString();
int numSubStrings = value.split(" ").length;
sb.append("\t Num Substrings: " + Integer.toString(numSubStrings)).append("\n");
sb.append("\t Value:" + value).append("\n");
} else {
// Basic types
BasicTypeParameter btParB = (BasicTypeParameter) param;
sb.append("\t Value: " + btParB.getValue().toString()).append("\n");
}
i++;
}
logger.error(sb.toString());
listener.jobFailed(this, JobEndStatus.SUBMISSION_FAILED);
}
sd.addAttribute("jobId", jobId);
// JEA Changed to allow execution in MN
sd.addAttribute(SoftwareDescription.WALLTIME_MAX, absImpl.getRequirements().getWallClockLimit());
if (absImpl.getRequirements().getHostQueues().size() > 0) {
sd.addAttribute(SoftwareDescription.JOB_QUEUE, absImpl.getRequirements().getHostQueues().get(0));
}
sd.addAttribute("coreCount", absImpl.getRequirements().getTotalCPUComputingUnits());
sd.addAttribute("gpuCount", absImpl.getRequirements().getTotalGPUComputingUnits());
sd.addAttribute("fpgaCount", absImpl.getRequirements().getTotalFPGAComputingUnits());
sd.addAttribute(SoftwareDescription.MEMORY_MAX, absImpl.getRequirements().getMemorySize());
// sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, "/tmp/");
sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, getResourceNode().getWorkingDir());
sd.addAttribute(SoftwareDescription.SANDBOX_USEROOT, "true");
sd.addAttribute(SoftwareDescription.SANDBOX_DELETE, "false");
/*
* sd.addAttribute(SoftwareDescription.SANDBOX_PRESTAGE_STDIN, "false");
* sd.addAttribute(SoftwareDescription.SANDBOX_POSTSTAGE_STDOUT, "false");
* sd.addAttribute(SoftwareDescription.SANDBOX_POSTSTAGE_STDERR, "false");
*/
if (debug) {
// Set standard output file for job
File outFile = GAT.createFile(context, Protocol.ANY_URI.getSchema() + File.separator + JOBS_DIR + "job" + jobId + "_" + this.getHistory() + ".out");
sd.setStdout(outFile);
}
if (debug || usingGlobus) {
// Set standard error file for job
File errFile = GAT.createFile(context, Protocol.ANY_URI.getSchema() + File.separator + JOBS_DIR + "job" + jobId + "_" + this.getHistory() + ".err");
sd.setStderr(errFile);
}
Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put(RES_ATTR, Protocol.ANY_URI.getSchema() + targetUser + targetHost);
attributes.put("Jobname", "compss_remote_job_" + jobId);
ResourceDescription rd = new HardwareResourceDescription(attributes);
if (debug) {
logger.debug("Ready to submit job " + jobId + ":");
logger.debug(" * Host: " + targetHost);
logger.debug(" * Executable: " + sd.getExecutable());
StringBuilder sb = new StringBuilder(" - Arguments:");
for (String arg : sd.getArguments()) {
sb.append(" ").append(arg);
}
logger.debug(sb.toString());
}
JobDescription jd = new JobDescription(sd, rd);
// jd.setProcessCount(method.getRequirements().getProcessorCoreCount());
return jd;
}
Aggregations