Search in sources :

Example 1 with DecafImplementation

use of es.bsc.compss.types.implementations.DecafImplementation in project compss by bsc-wdc.

the class ITFParser method checkDefinedImplementations.

/**
 * Check all the defined implementations of the same method
 *
 * @param m
 * @param methodId
 * @param calleeMethodSignature
 * @param hasStreams
 * @param implementations
 * @param signatures
 */
private static void checkDefinedImplementations(java.lang.reflect.Method m, Integer methodId, StringBuilder calleeMethodSignature, boolean hasStreams, boolean hasPrefixes, List<Implementation> implementations, List<String> signatures) {
    /*
         * Global constraints of the method
         */
    MethodResourceDescription defaultConstraints = MethodResourceDescription.EMPTY_FOR_CONSTRAINTS.copy();
    if (m.isAnnotationPresent(Constraints.class)) {
        defaultConstraints = new MethodResourceDescription(m.getAnnotation(Constraints.class));
    }
    /*
         * Check all annotations present at the method for versioning
         */
    String methodName = m.getName();
    int implId = 0;
    /*
         * METHOD
         */
    for (Method methodAnnot : m.getAnnotationsByType(Method.class)) {
        LOGGER.debug("   * Processing @Method annotation");
        // Warning for ignoring streams
        if (hasStreams) {
            ErrorManager.warn("Java method " + methodName + " does not support stream annotations. SKIPPING stream annotation");
        }
        // Warning for ignoring prefixes
        if (hasPrefixes) {
            ErrorManager.warn("Java method " + methodName + " does not support prefix annotations. SKIPPING prefix annotation");
        }
        String declaringClass = methodAnnot.declaringClass();
        String methodSignature = calleeMethodSignature.toString() + declaringClass;
        signatures.add(methodSignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (methodAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(methodAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new MethodImplementation(declaringClass, methodName, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
    /*
         * SERVICE
         */
    for (Service serviceAnnot : m.getAnnotationsByType(Service.class)) {
        // Services don't have constraints
        LOGGER.debug("   * Processing @Service annotation");
        // Warning for ignoring streams
        if (hasStreams) {
            ErrorManager.warn("Java service " + methodName + " does not support stream annotations. SKIPPING stream annotation");
        }
        calleeMethodSignature.append(serviceAnnot.namespace()).append(',');
        calleeMethodSignature.append(serviceAnnot.name()).append(',');
        calleeMethodSignature.append(serviceAnnot.port());
        String serviceSignature = calleeMethodSignature.toString();
        signatures.add(serviceSignature);
        // Register service implementation
        Implementation impl = new ServiceImplementation(methodId, serviceAnnot.namespace(), serviceAnnot.name(), serviceAnnot.port(), serviceAnnot.operation());
        ++implId;
        implementations.add(impl);
    }
    /*
         * MPI
         */
    for (MPI mpiAnnot : m.getAnnotationsByType(MPI.class)) {
        LOGGER.debug("   * Processing @MPI annotation");
        String binary = EnvironmentLoader.loadFromEnvironment(mpiAnnot.binary());
        String workingDir = EnvironmentLoader.loadFromEnvironment(mpiAnnot.workingDir());
        String mpiRunner = EnvironmentLoader.loadFromEnvironment(mpiAnnot.mpiRunner());
        if (mpiRunner == null || mpiRunner.isEmpty()) {
            ErrorManager.error("Empty mpiRunner annotation for method " + m.getName());
        }
        if (binary == null || binary.isEmpty()) {
            ErrorManager.error("Empty binary annotation for method " + m.getName());
        }
        LOGGER.debug("Binary: " + binary);
        LOGGER.debug("mpiRunner: " + mpiRunner);
        String mpiSignature = calleeMethodSignature.toString() + LoaderUtils.MPI_SIGNATURE;
        signatures.add(mpiSignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (mpiAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(mpiAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new MPIImplementation(binary, workingDir, mpiRunner, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
    /*
         * Decaf
         */
    for (Decaf decafAnnot : m.getAnnotationsByType(Decaf.class)) {
        LOGGER.debug("   * Processing @DECAF annotation");
        String dfScript = EnvironmentLoader.loadFromEnvironment(decafAnnot.dfScript());
        String dfExecutor = EnvironmentLoader.loadFromEnvironment(decafAnnot.dfExecutor());
        String dfLib = EnvironmentLoader.loadFromEnvironment(decafAnnot.dfLib());
        String workingDir = EnvironmentLoader.loadFromEnvironment(decafAnnot.workingDir());
        String mpiRunner = EnvironmentLoader.loadFromEnvironment(decafAnnot.mpiRunner());
        if (mpiRunner == null || mpiRunner.isEmpty()) {
            ErrorManager.error("Empty mpiRunner annotation for method " + m.getName());
        }
        if (dfScript == null || dfScript.isEmpty()) {
            ErrorManager.error("Empty binary annotation for method " + m.getName());
        }
        LOGGER.debug("DF Script: " + dfScript);
        LOGGER.debug("DF Executor: " + dfExecutor);
        LOGGER.debug("DF Lib: " + dfLib);
        LOGGER.debug("mpiRunner: " + mpiRunner);
        String mpiSignature = calleeMethodSignature.toString() + LoaderUtils.DECAF_SIGNATURE;
        signatures.add(mpiSignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (decafAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(decafAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new DecafImplementation(dfScript, dfExecutor, dfLib, workingDir, mpiRunner, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
    /*
         * OMPSS
         */
    for (OmpSs ompssAnnot : m.getAnnotationsByType(OmpSs.class)) {
        LOGGER.debug("   * Processing @OmpSs annotation");
        String binary = EnvironmentLoader.loadFromEnvironment(ompssAnnot.binary());
        String workingDir = EnvironmentLoader.loadFromEnvironment(ompssAnnot.workingDir());
        if (binary == null || binary.isEmpty()) {
            ErrorManager.error("Empty binary annotation for method " + m.getName());
        }
        String ompssSignature = calleeMethodSignature.toString() + LoaderUtils.OMPSS_SIGNATURE;
        signatures.add(ompssSignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (ompssAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(ompssAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new OmpSsImplementation(binary, workingDir, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
    /*
         * OPENCL
         */
    for (OpenCL openclAnnot : m.getAnnotationsByType(OpenCL.class)) {
        LOGGER.debug("   * Processing @OpenCL annotation");
        String kernel = EnvironmentLoader.loadFromEnvironment(openclAnnot.kernel());
        String workingDir = EnvironmentLoader.loadFromEnvironment(openclAnnot.workingDir());
        if (kernel == null || kernel.isEmpty()) {
            ErrorManager.error("Empty kernel annotation for method " + m.getName());
        }
        String openclSignature = calleeMethodSignature.toString() + LoaderUtils.OPENCL_SIGNATURE;
        signatures.add(openclSignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (openclAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(openclAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new OpenCLImplementation(kernel, workingDir, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
    /*
         * BINARY
         */
    for (Binary binaryAnnot : m.getAnnotationsByType(Binary.class)) {
        LOGGER.debug("   * Processing @Binary annotation");
        String binary = EnvironmentLoader.loadFromEnvironment(binaryAnnot.binary());
        String workingDir = EnvironmentLoader.loadFromEnvironment(binaryAnnot.workingDir());
        if (binary == null || binary.isEmpty()) {
            ErrorManager.error("Empty binary annotation for method " + m.getName());
        }
        String binarySignature = calleeMethodSignature.toString() + LoaderUtils.BINARY_SIGNATURE;
        signatures.add(binarySignature);
        // Load specific method constraints if present
        MethodResourceDescription implConstraints = defaultConstraints;
        if (binaryAnnot.constraints() != null) {
            implConstraints = new MethodResourceDescription(binaryAnnot.constraints());
            implConstraints.mergeMultiConstraints(defaultConstraints);
        }
        // Register method implementation
        Implementation impl = new BinaryImplementation(binary, workingDir, methodId, implId, implConstraints);
        ++implId;
        implementations.add(impl);
    }
}
Also used : MethodImplementation(es.bsc.compss.types.implementations.MethodImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) MPI(es.bsc.compss.types.annotations.task.MPI) Service(es.bsc.compss.types.annotations.task.Service) Decaf(es.bsc.compss.types.annotations.task.Decaf) Method(es.bsc.compss.types.annotations.task.Method) OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) Implementation(es.bsc.compss.types.implementations.Implementation) ServiceImplementation(es.bsc.compss.types.implementations.ServiceImplementation) MethodImplementation(es.bsc.compss.types.implementations.MethodImplementation) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) ServiceImplementation(es.bsc.compss.types.implementations.ServiceImplementation) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) MethodResourceDescription(es.bsc.compss.types.resources.MethodResourceDescription) OpenCL(es.bsc.compss.types.annotations.task.OpenCL) Binary(es.bsc.compss.types.annotations.task.Binary) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) OmpSs(es.bsc.compss.types.annotations.task.OmpSs) MultiOmpSs(es.bsc.compss.types.annotations.task.repeatables.MultiOmpSs) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation)

Example 2 with DecafImplementation

use of es.bsc.compss.types.implementations.DecafImplementation in project compss by bsc-wdc.

the class GATJob method prepareJob.

private JobDescription prepareJob() throws Exception {
    // Get the information related to the job
    logger.debug("Preparing GAT Job " + this.jobId);
    TaskDescription taskParams = this.taskParams;
    String targetPath = getResourceNode().getInstallDir();
    String targetHost = getResourceNode().getHost();
    String targetUser = getResourceNode().getUser();
    if (userNeeded && !targetUser.isEmpty()) {
        targetUser += "@";
    } else {
        targetUser = "";
    }
    SoftwareDescription sd = new SoftwareDescription();
    sd.setExecutable(targetPath + WORKER_SCRIPT_PATH + WORKER_SCRIPT_NAME);
    ArrayList<String> lArgs = new ArrayList<String>();
    // Common arguments: language working_dir lib_path num_obsolete [obs1... obsN] tracing [event_type task_id
    // slot_id]
    lArgs.add(LANG);
    lArgs.add(getResourceNode().getWorkingDir());
    lArgs.add(getResourceNode().getLibPath());
    LogicalData[] obsoleteFiles = getResource().pollObsoletes();
    if (obsoleteFiles != null) {
        lArgs.add("" + obsoleteFiles.length);
        for (LogicalData ld : obsoleteFiles) {
            String renaming = ld.getName();
            lArgs.add(renaming);
        }
    } else {
        lArgs.add("0");
    }
    // Check sandbox working dir
    boolean isSpecific = false;
    String sandboxDir = null;
    AbstractMethodImplementation absImpl = (AbstractMethodImplementation) this.impl;
    switch(absImpl.getMethodType()) {
        case BINARY:
            BinaryImplementation binaryImpl = (BinaryImplementation) absImpl;
            sandboxDir = binaryImpl.getWorkingDir();
            isSpecific = true;
            break;
        case MPI:
            MPIImplementation mpiImpl = (MPIImplementation) absImpl;
            sandboxDir = mpiImpl.getWorkingDir();
            isSpecific = true;
            break;
        case DECAF:
            DecafImplementation decafImpl = (DecafImplementation) absImpl;
            sandboxDir = decafImpl.getWorkingDir();
            isSpecific = true;
            break;
        case OMPSS:
            OmpSsImplementation ompssImpl = (OmpSsImplementation) absImpl;
            sandboxDir = ompssImpl.getWorkingDir();
            isSpecific = true;
            break;
        case OPENCL:
            OpenCLImplementation openclImpl = (OpenCLImplementation) absImpl;
            sandboxDir = openclImpl.getWorkingDir();
            isSpecific = true;
            break;
        case METHOD:
            sandboxDir = null;
            break;
    }
    if (sandboxDir == null || sandboxDir.isEmpty() || sandboxDir.equals(Constants.UNASSIGNED)) {
        sandboxDir = getResourceNode().getWorkingDir() + File.separator + "sandBox" + File.separator + "job_" + this.jobId;
        isSpecific = false;
    }
    // Processing parameters to get symlinks pairs to create (symlinks) and how to pass parameters in the GAT
    // Job(paramArgs)
    ArrayList<String> symlinks = new ArrayList<>();
    ArrayList<String> paramArgs = new ArrayList<>();
    processParameters(sandboxDir, symlinks, paramArgs);
    // Adding info to create symlinks between renamed files and original names
    lArgs.add(Boolean.toString(isSpecific));
    lArgs.add(sandboxDir);
    if (symlinks.size() > 0) {
        lArgs.add(String.valueOf(symlinks.size()));
        lArgs.addAll(symlinks);
    } else {
        lArgs.add("0");
    }
    lArgs.add(Boolean.toString(Tracer.isActivated()));
    lArgs.add(getHostName());
    if (debug) {
        logger.debug("hostName " + getHostName());
    }
    if (Tracer.isActivated()) {
        // event type
        lArgs.add(String.valueOf(Tracer.getTaskEventsType()));
        // task id
        lArgs.add(String.valueOf(this.taskParams.getId() + 1));
        int slot = Tracer.getNextSlot(targetHost);
        // slot id
        lArgs.add(String.valueOf(slot));
        sd.addAttribute("slot", slot);
    }
    // Language-dependent arguments: taskSandbox_dir app_dir classpath pythonpath debug storage_conf
    // method_impl_type method_impl_params
    // numSlaves [slave1,..,slaveN] numCus
    // has_target num_params par_type_1 par_1 ... par_type_n par_n
    lArgs.add(sandboxDir);
    lArgs.add(getResourceNode().getAppDir());
    lArgs.add(getClasspath());
    lArgs.add(getPythonpath());
    lArgs.add(String.valueOf(debug));
    lArgs.add(STORAGE_CONF);
    lArgs.add(String.valueOf(absImpl.getMethodType()));
    switch(absImpl.getMethodType()) {
        case METHOD:
            MethodImplementation methodImpl = (MethodImplementation) absImpl;
            lArgs.add(methodImpl.getDeclaringClass());
            String methodName = methodImpl.getAlternativeMethodName();
            if (methodName == null || methodName.isEmpty()) {
                methodName = taskParams.getName();
            }
            lArgs.add(methodName);
            break;
        case MPI:
            MPIImplementation mpiImpl = (MPIImplementation) absImpl;
            lArgs.add(mpiImpl.getMpiRunner());
            lArgs.add(mpiImpl.getBinary());
            break;
        case DECAF:
            DecafImplementation decafImpl = (DecafImplementation) absImpl;
            lArgs.add(targetPath + DecafImplementation.SCRIPT_PATH);
            String dfScript = decafImpl.getDfScript();
            if (!dfScript.startsWith(File.separator)) {
                String appPath = getResourceNode().getAppDir();
                dfScript = appPath + File.separator + dfScript;
            }
            lArgs.add(dfScript);
            String dfExecutor = decafImpl.getDfExecutor();
            if (dfExecutor == null || dfExecutor.isEmpty() || dfExecutor.equals(Constants.UNASSIGNED)) {
                dfExecutor = "executor.sh";
            }
            if (!dfExecutor.startsWith(File.separator) && !dfExecutor.startsWith("./")) {
                dfExecutor = "./" + dfExecutor;
            }
            lArgs.add(dfExecutor);
            String dfLib = decafImpl.getDfLib();
            if (dfLib == null || dfLib.isEmpty()) {
                dfLib = Constants.UNASSIGNED;
            }
            lArgs.add(dfLib);
            lArgs.add(decafImpl.getMpiRunner());
            break;
        case OMPSS:
            OmpSsImplementation ompssImpl = (OmpSsImplementation) absImpl;
            lArgs.add(ompssImpl.getBinary());
            break;
        case OPENCL:
            OpenCLImplementation openclImpl = (OpenCLImplementation) absImpl;
            lArgs.add(openclImpl.getKernel());
            break;
        case BINARY:
            BinaryImplementation binaryImpl = (BinaryImplementation) absImpl;
            lArgs.add(binaryImpl.getBinary());
            break;
    }
    // Slave nodes and cus description
    lArgs.add(String.valueOf(slaveWorkersNodeNames.size()));
    lArgs.addAll(slaveWorkersNodeNames);
    lArgs.add(String.valueOf(((MethodResourceDescription) this.impl.getRequirements()).getTotalCPUComputingUnits()));
    // Add parameter arguments already processed
    lArgs.addAll(paramArgs);
    // Conversion vector -> array
    String[] arguments = new String[lArgs.size()];
    arguments = lArgs.toArray(arguments);
    try {
        sd.setArguments(arguments);
    } catch (NullPointerException e) {
        StringBuilder sb = new StringBuilder("Null argument parameter of job " + this.jobId + " " + absImpl.getMethodDefinition() + "\n");
        int i = 0;
        for (Parameter param : taskParams.getParameters()) {
            sb.append("Parameter ").append(i).append("\n");
            DataType type = param.getType();
            sb.append("\t Type: ").append(param.getType()).append("\n");
            if (type == DataType.FILE_T || type == DataType.OBJECT_T) {
                DependencyParameter dPar = (DependencyParameter) param;
                DataAccessId dAccId = dPar.getDataAccessId();
                sb.append("\t Target: ").append(dPar.getDataTarget()).append("\n");
                if (type == DataType.OBJECT_T) {
                    if (dAccId instanceof RAccessId) {
                        sb.append("\t Direction: " + "R").append("\n");
                    } else {
                        // for the worker to know it must write the object to disk
                        sb.append("\t Direction: " + "W").append("\n");
                    }
                }
            } else if (type == DataType.STRING_T) {
                BasicTypeParameter btParS = (BasicTypeParameter) param;
                // Check spaces
                String value = btParS.getValue().toString();
                int numSubStrings = value.split(" ").length;
                sb.append("\t Num Substrings: " + Integer.toString(numSubStrings)).append("\n");
                sb.append("\t Value:" + value).append("\n");
            } else {
                // Basic types
                BasicTypeParameter btParB = (BasicTypeParameter) param;
                sb.append("\t Value: " + btParB.getValue().toString()).append("\n");
            }
            i++;
        }
        logger.error(sb.toString());
        listener.jobFailed(this, JobEndStatus.SUBMISSION_FAILED);
    }
    sd.addAttribute("jobId", jobId);
    // JEA Changed to allow execution in MN
    sd.addAttribute(SoftwareDescription.WALLTIME_MAX, absImpl.getRequirements().getWallClockLimit());
    if (absImpl.getRequirements().getHostQueues().size() > 0) {
        sd.addAttribute(SoftwareDescription.JOB_QUEUE, absImpl.getRequirements().getHostQueues().get(0));
    }
    sd.addAttribute("coreCount", absImpl.getRequirements().getTotalCPUComputingUnits());
    sd.addAttribute("gpuCount", absImpl.getRequirements().getTotalGPUComputingUnits());
    sd.addAttribute("fpgaCount", absImpl.getRequirements().getTotalFPGAComputingUnits());
    sd.addAttribute(SoftwareDescription.MEMORY_MAX, absImpl.getRequirements().getMemorySize());
    // sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, "/tmp/");
    sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, getResourceNode().getWorkingDir());
    sd.addAttribute(SoftwareDescription.SANDBOX_USEROOT, "true");
    sd.addAttribute(SoftwareDescription.SANDBOX_DELETE, "false");
    /*
         * sd.addAttribute(SoftwareDescription.SANDBOX_PRESTAGE_STDIN, "false");
         * sd.addAttribute(SoftwareDescription.SANDBOX_POSTSTAGE_STDOUT, "false");
         * sd.addAttribute(SoftwareDescription.SANDBOX_POSTSTAGE_STDERR, "false");
         */
    if (debug) {
        // Set standard output file for job
        File outFile = GAT.createFile(context, Protocol.ANY_URI.getSchema() + File.separator + JOBS_DIR + "job" + jobId + "_" + this.getHistory() + ".out");
        sd.setStdout(outFile);
    }
    if (debug || usingGlobus) {
        // Set standard error file for job
        File errFile = GAT.createFile(context, Protocol.ANY_URI.getSchema() + File.separator + JOBS_DIR + "job" + jobId + "_" + this.getHistory() + ".err");
        sd.setStderr(errFile);
    }
    Map<String, Object> attributes = new HashMap<String, Object>();
    attributes.put(RES_ATTR, Protocol.ANY_URI.getSchema() + targetUser + targetHost);
    attributes.put("Jobname", "compss_remote_job_" + jobId);
    ResourceDescription rd = new HardwareResourceDescription(attributes);
    if (debug) {
        logger.debug("Ready to submit job " + jobId + ":");
        logger.debug("  * Host: " + targetHost);
        logger.debug("  * Executable: " + sd.getExecutable());
        StringBuilder sb = new StringBuilder("  - Arguments:");
        for (String arg : sd.getArguments()) {
            sb.append(" ").append(arg);
        }
        logger.debug(sb.toString());
    }
    JobDescription jd = new JobDescription(sd, rd);
    // jd.setProcessCount(method.getRequirements().getProcessorCoreCount());
    return jd;
}
Also used : MethodImplementation(es.bsc.compss.types.implementations.MethodImplementation) AbstractMethodImplementation(es.bsc.compss.types.implementations.AbstractMethodImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) RAccessId(es.bsc.compss.types.data.DataAccessId.RAccessId) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DependencyParameter(es.bsc.compss.types.parameter.DependencyParameter) BasicTypeParameter(es.bsc.compss.types.parameter.BasicTypeParameter) OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) JobDescription(org.gridlab.gat.resources.JobDescription) TaskDescription(es.bsc.compss.types.TaskDescription) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) DataType(es.bsc.compss.types.annotations.parameter.DataType) MethodResourceDescription(es.bsc.compss.types.resources.MethodResourceDescription) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) AbstractMethodImplementation(es.bsc.compss.types.implementations.AbstractMethodImplementation) HardwareResourceDescription(org.gridlab.gat.resources.HardwareResourceDescription) SoftwareDescription(org.gridlab.gat.resources.SoftwareDescription) LogicalData(es.bsc.compss.types.data.LogicalData) MethodResourceDescription(es.bsc.compss.types.resources.MethodResourceDescription) HardwareResourceDescription(org.gridlab.gat.resources.HardwareResourceDescription) ResourceDescription(org.gridlab.gat.resources.ResourceDescription) Parameter(es.bsc.compss.types.parameter.Parameter) DependencyParameter(es.bsc.compss.types.parameter.DependencyParameter) BasicTypeParameter(es.bsc.compss.types.parameter.BasicTypeParameter) File(org.gridlab.gat.io.File) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation) DataAccessId(es.bsc.compss.types.data.DataAccessId)

Example 3 with DecafImplementation

use of es.bsc.compss.types.implementations.DecafImplementation in project compss by bsc-wdc.

the class Executor method createTaskSandbox.

/**
 * Creates a sandbox for a task
 *
 * @param nt
 *            task description
 * @return Sandbox dir
 * @throws IOException
 * @throws Exception
 */
private TaskWorkingDir createTaskSandbox(NIOTask nt) throws IOException {
    // Check if an specific working dir is provided
    String specificWD = null;
    switch(nt.getMethodType()) {
        case BINARY:
            BinaryImplementation binaryImpl = (BinaryImplementation) nt.getMethodImplementation();
            specificWD = binaryImpl.getWorkingDir();
            break;
        case MPI:
            MPIImplementation mpiImpl = (MPIImplementation) nt.getMethodImplementation();
            specificWD = mpiImpl.getWorkingDir();
            break;
        case DECAF:
            DecafImplementation decafImpl = (DecafImplementation) nt.getMethodImplementation();
            specificWD = decafImpl.getWorkingDir();
            break;
        case OMPSS:
            OmpSsImplementation ompssImpl = (OmpSsImplementation) nt.getMethodImplementation();
            specificWD = ompssImpl.getWorkingDir();
            break;
        case OPENCL:
            OpenCLImplementation openclImpl = (OpenCLImplementation) nt.getMethodImplementation();
            specificWD = openclImpl.getWorkingDir();
            break;
        case METHOD:
            specificWD = null;
            break;
    }
    TaskWorkingDir taskWD;
    if (specificWD != null && !specificWD.isEmpty() && !specificWD.equals(Constants.UNASSIGNED)) {
        // Binary has an specific working dir, set it
        File workingDir = new File(specificWD);
        taskWD = new TaskWorkingDir(workingDir, true);
        // Create structures
        Files.createDirectories(workingDir.toPath());
    } else {
        // No specific working dir provided, set default sandbox
        String completePath = this.nw.getWorkingDir() + "sandBox" + File.separator + "job_" + nt.getJobId();
        File workingDir = new File(completePath);
        taskWD = new TaskWorkingDir(workingDir, false);
        // Clean-up previous versions if any
        if (workingDir.exists()) {
            LOGGER.debug("Deleting folder " + workingDir.toString());
            if (!workingDir.delete()) {
                LOGGER.warn("Cannot delete working dir folder: " + workingDir.toString());
            }
        }
        // Create structures
        Files.createDirectories(workingDir.toPath());
    }
    return taskWD;
}
Also used : OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) File(java.io.File) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation)

Example 4 with DecafImplementation

use of es.bsc.compss.types.implementations.DecafImplementation in project compss by bsc-wdc.

the class CERegistration method process.

@Override
public void process(TaskScheduler ts) {
    // Register the coreElement
    Integer coreId = CoreManager.registerNewCoreElement(coreElementSignature);
    if (coreId == null) {
        // CoreElement already exists, retrieve its id
        coreId = CoreManager.getCoreId(coreElementSignature);
    }
    // Retrieve the new implementation number
    int implId = CoreManager.getNumberCoreImplementations(coreId);
    // Create the implementation
    Implementation m = null;
    switch(implType) {
        case METHOD:
            if (this.implTypeArgs.length != 2) {
                ErrorManager.error("Incorrect parameters for type METHOD on " + this.coreElementSignature);
            }
            String declaringClass = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String methodName = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            if (declaringClass == null || declaringClass.isEmpty()) {
                ErrorManager.error("Empty declaringClass annotation for method " + this.coreElementSignature);
            }
            if (methodName == null || methodName.isEmpty()) {
                ErrorManager.error("Empty methodName annotation for method " + this.coreElementSignature);
            }
            m = new MethodImplementation(declaringClass, methodName, coreId, implId, implConstraints);
            break;
        case MPI:
            if (this.implTypeArgs.length != 3) {
                ErrorManager.error("Incorrect parameters for type MPI on " + this.coreElementSignature);
            }
            String mpiBinary = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String mpiWorkingDir = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            String mpiRunner = EnvironmentLoader.loadFromEnvironment(implTypeArgs[2]);
            if (mpiRunner == null || mpiRunner.isEmpty()) {
                ErrorManager.error("Empty mpiRunner annotation for MPI method " + this.coreElementSignature);
            }
            if (mpiBinary == null || mpiBinary.isEmpty()) {
                ErrorManager.error("Empty binary annotation for MPI method " + this.coreElementSignature);
            }
            m = new MPIImplementation(mpiBinary, mpiWorkingDir, mpiRunner, coreId, implId, this.implConstraints);
            break;
        case DECAF:
            if (this.implTypeArgs.length != 5) {
                ErrorManager.error("Incorrect parameters for type DECAF on " + this.coreElementSignature);
            }
            String dfScript = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String dfExecutor = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            String dfLib = EnvironmentLoader.loadFromEnvironment(implTypeArgs[2]);
            String decafWorkingDir = EnvironmentLoader.loadFromEnvironment(implTypeArgs[3]);
            String decafRunner = EnvironmentLoader.loadFromEnvironment(implTypeArgs[4]);
            if (decafRunner == null || decafRunner.isEmpty()) {
                ErrorManager.error("Empty mpiRunner annotation for DECAF method " + this.coreElementSignature);
            }
            if (dfScript == null || dfScript.isEmpty()) {
                ErrorManager.error("Empty dfScript annotation for DECAF method " + this.coreElementSignature);
            }
            m = new DecafImplementation(dfScript, dfExecutor, dfLib, decafWorkingDir, decafRunner, coreId, implId, this.implConstraints);
            break;
        case BINARY:
            if (this.implTypeArgs.length != 2) {
                ErrorManager.error("Incorrect parameters for type BINARY on " + this.coreElementSignature);
            }
            String binary = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String binaryWorkingDir = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            if (binary == null || binary.isEmpty()) {
                ErrorManager.error("Empty binary annotation for BINARY method " + this.coreElementSignature);
            }
            m = new BinaryImplementation(binary, binaryWorkingDir, coreId, implId, implConstraints);
            break;
        case OMPSS:
            if (this.implTypeArgs.length != 2) {
                ErrorManager.error("Incorrect parameters for type OMPSS on " + this.coreElementSignature);
            }
            String ompssBinary = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String ompssWorkingDir = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            if (ompssBinary == null || ompssBinary.isEmpty()) {
                ErrorManager.error("Empty binary annotation for OmpSs method " + this.coreElementSignature);
            }
            m = new OmpSsImplementation(ompssBinary, ompssWorkingDir, coreId, implId, implConstraints);
            break;
        case OPENCL:
            if (this.implTypeArgs.length != 2) {
                ErrorManager.error("Incorrect parameters for type OPENCL on " + this.coreElementSignature);
            }
            String openclKernel = EnvironmentLoader.loadFromEnvironment(implTypeArgs[0]);
            String openclWorkingDir = EnvironmentLoader.loadFromEnvironment(implTypeArgs[1]);
            if (openclKernel == null || openclKernel.isEmpty()) {
                ErrorManager.error("Empty kernel annotation for OpenCL method " + this.coreElementSignature);
            }
            m = new OpenCLImplementation(openclKernel, openclWorkingDir, coreId, implId, implConstraints);
            break;
        default:
            ErrorManager.error("Unrecognised implementation type");
            break;
    }
    // Register the new implementation
    List<Implementation> newImpls = new LinkedList<>();
    newImpls.add(m);
    List<String> newSignatures = new LinkedList<>();
    newSignatures.add(implSignature);
    CoreManager.registerNewImplementations(coreId, newImpls, newSignatures);
    // Update the Resources structures
    LinkedList<Integer> newCores = new LinkedList<>();
    newCores.add(coreId);
    ResourceManager.coreElementUpdates(newCores);
    // Update the Scheduler structures
    ts.coreElementsUpdated();
    LOGGER.debug("Data structures resized and CE-resources links updated");
    sem.release();
}
Also used : MethodImplementation(es.bsc.compss.types.implementations.MethodImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) MPIImplementation(es.bsc.compss.types.implementations.MPIImplementation) OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) Implementation(es.bsc.compss.types.implementations.Implementation) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation) MethodImplementation(es.bsc.compss.types.implementations.MethodImplementation) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) LinkedList(java.util.LinkedList) OpenCLImplementation(es.bsc.compss.types.implementations.OpenCLImplementation) DecafImplementation(es.bsc.compss.types.implementations.DecafImplementation) BinaryImplementation(es.bsc.compss.types.implementations.BinaryImplementation) OmpSsImplementation(es.bsc.compss.types.implementations.OmpSsImplementation)

Aggregations

BinaryImplementation (es.bsc.compss.types.implementations.BinaryImplementation)4 DecafImplementation (es.bsc.compss.types.implementations.DecafImplementation)4 MPIImplementation (es.bsc.compss.types.implementations.MPIImplementation)4 OmpSsImplementation (es.bsc.compss.types.implementations.OmpSsImplementation)4 OpenCLImplementation (es.bsc.compss.types.implementations.OpenCLImplementation)4 MethodImplementation (es.bsc.compss.types.implementations.MethodImplementation)3 Implementation (es.bsc.compss.types.implementations.Implementation)2 MethodResourceDescription (es.bsc.compss.types.resources.MethodResourceDescription)2 TaskDescription (es.bsc.compss.types.TaskDescription)1 DataType (es.bsc.compss.types.annotations.parameter.DataType)1 Binary (es.bsc.compss.types.annotations.task.Binary)1 Decaf (es.bsc.compss.types.annotations.task.Decaf)1 MPI (es.bsc.compss.types.annotations.task.MPI)1 Method (es.bsc.compss.types.annotations.task.Method)1 OmpSs (es.bsc.compss.types.annotations.task.OmpSs)1 OpenCL (es.bsc.compss.types.annotations.task.OpenCL)1 Service (es.bsc.compss.types.annotations.task.Service)1 MultiOmpSs (es.bsc.compss.types.annotations.task.repeatables.MultiOmpSs)1 DataAccessId (es.bsc.compss.types.data.DataAccessId)1 RAccessId (es.bsc.compss.types.data.DataAccessId.RAccessId)1