Examples with NIOParam es.bsc.compss.nio.NIOParam used on opensource projects

Example 1 with NIOParam

use of es.bsc.compss.nio.NIOParam in project compss by bsc-wdc.

the class NIOJob method addParams.

private LinkedList<NIOParam> addParams() {
    LinkedList<NIOParam> params = new LinkedList<>();
    for (Parameter param : taskParams.getParameters()) {
        DataType type = param.getType();
        NIOParam np;
        switch(type) {
            case FILE_T:
            case OBJECT_T:
            case PSCO_T:
            case EXTERNAL_OBJECT_T:
                DependencyParameter dPar = (DependencyParameter) param;
                DataAccessId dAccId = dPar.getDataAccessId();
                Object value = dPar.getDataTarget();
                boolean preserveSourceData = true;
                if (dAccId instanceof RAccessId) {
                    // Parameter is a R, has sources
                    preserveSourceData = ((RAccessId) dAccId).isPreserveSourceData();
                } else if (dAccId instanceof RWAccessId) {
                    // Parameter is a RW, has sources
                    preserveSourceData = ((RWAccessId) dAccId).isPreserveSourceData();
                } else {
                    // Parameter is a W, it has no sources
                    preserveSourceData = false;
                }
                // Workaround for Python PSCOs in return
                // Check if the parameter has a valid PSCO and change its type
                String renaming;
                DataAccessId faId = dPar.getDataAccessId();
                if (faId instanceof WAccessId) {
                    // Write mode
                    WAccessId waId = (WAccessId) faId;
                    renaming = waId.getWrittenDataInstance().getRenaming();
                } else if (faId instanceof RWAccessId) {
                    // Read write mode
                    RWAccessId rwaId = (RWAccessId) faId;
                    renaming = rwaId.getWrittenDataInstance().getRenaming();
                } else {
                    // Read only mode
                    RAccessId raId = (RAccessId) faId;
                    renaming = raId.getReadDataInstance().getRenaming();
                }
                String pscoId = Comm.getData(renaming).getId();
                if (pscoId != null && type.equals(DataType.FILE_T)) {
                    param.setType(DataType.EXTERNAL_OBJECT_T);
                    type = param.getType();
                }
                // Create the NIO Param
                // Only store W and RW
                boolean writeFinalValue = !(dAccId instanceof RAccessId);
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, (Data) dPar.getDataSource(), dPar.getOriginalName());
                break;
            default:
                BasicTypeParameter btParB = (BasicTypeParameter) param;
                value = btParB.getValue();
                // Basic parameters are not preserved on Worker
                preserveSourceData = false;
                // Basic parameters are not stored on Worker
                writeFinalValue = false;
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, null, DependencyParameter.NO_NAME);
                break;
        }
        params.add(np);
    }
    return params;
}

Example 2 with NIOParam

use of es.bsc.compss.nio.NIOParam in project compss by bsc-wdc.

the class NIOJob method prepareJob.

public NIOTask prepareJob() {
    AbstractMethodImplementation absMethodImpl = (AbstractMethodImplementation) this.impl;
    // This is a workarround for Python
    if (absMethodImpl.getMethodType().equals(MethodType.METHOD)) {
        MethodImplementation mImpl = (MethodImplementation) absMethodImpl;
        String methodName = mImpl.getAlternativeMethodName();
        if (methodName == null || methodName.isEmpty()) {
            mImpl.setAlternativeMethodName(taskParams.getName());
        }
    }
    boolean hasTarget = taskParams.hasTargetObject();
    boolean hasReturn = taskParams.hasReturnValue();
    LinkedList<NIOParam> params = addParams();
    MethodResourceDescription reqs = absMethodImpl.getRequirements();
    int numParams = params.size();
    if (taskParams.hasReturnValue()) {
        numParams--;
    }
    // Create NIOTask
    NIOTask nt = new NIOTask(LANG, debug, absMethodImpl, hasTarget, hasReturn, params, numParams, reqs, this.slaveWorkersNodeNames, this.taskId, this.taskParams.getId(), this.jobId, this.history, this.transferId);
    return nt;
}

Example 3 with NIOParam

use of es.bsc.compss.nio.NIOParam in project compss by bsc-wdc.

the class Executor method removeOriginalFilenames.

/**
 * Undo symbolic links and renames done with the original names in task sandbox to the renamed file
 *
 * @param nt
 *            task description
 * @throws IOException
 * @throws JobExecutionException
 * @throws Exception
 *             returns exception is an unexpected case is found.
 */
private void removeOriginalFilenames(NIOTask nt) throws IOException, JobExecutionException {
    for (NIOParam param : nt.getParams()) {
        if (param.getType().equals(DataType.FILE_T)) {
            String renamedFilePath = (String) param.getValue();
            String newOriginalFilePath = param.getOriginalName();
            LOGGER.debug("Treating file " + renamedFilePath);
            if (!renamedFilePath.equals(newOriginalFilePath)) {
                File newOrigFile = new File(newOriginalFilePath);
                File renamedFile = new File(renamedFilePath);
                if (renamedFile.exists()) {
                    // IN, INOUT
                    if (newOrigFile.exists()) {
                        if (Files.isSymbolicLink(newOrigFile.toPath())) {
                            // If a symbolic link is created remove it
                            LOGGER.debug("Deleting symlink " + newOrigFile.toPath());
                            Files.delete(newOrigFile.toPath());
                        } else {
                            // Rewrite inout param by moving the new file to the renaming
                            move(newOrigFile.toPath(), renamedFile.toPath());
                        }
                    } else {
                        // Both files exist and are updated
                        LOGGER.debug("Repeated data for " + renamedFilePath + ". Nothing to do");
                    }
                } else {
                    // OUT
                    if (newOrigFile.exists()) {
                        if (Files.isSymbolicLink(newOrigFile.toPath())) {
                            // Unexpected case
                            String msg = "ERROR: Unexpected case. A Problem occurred with File " + renamedFilePath + ". Either this file or the original name " + newOriginalFilePath + " do not exist.";
                            LOGGER.error(msg);
                            System.err.println(msg);
                            throw new JobExecutionException(msg);
                        } else {
                            // If an output file is created move to the renamed path (OUT Case)
                            move(newOrigFile.toPath(), renamedFile.toPath());
                        }
                    } else {
                        // Error output file does not exist
                        String msg = "ERROR: Output file " + newOriginalFilePath + " does not exist";
                        // Unexpected case (except for C binding when not serializing outputs)
                        if (Lang.valueOf(nt.getLang().toUpperCase()) != Lang.C) {
                            LOGGER.error(msg);
                            System.err.println(msg);
                            throw new JobExecutionException(msg);
                        }
                    }
                }
            }
        }
    }
}

Example 4 with NIOParam

use of es.bsc.compss.nio.NIOParam in project compss by bsc-wdc.

the class Invoker method serializeBinaryExitValue.

public void serializeBinaryExitValue() throws JobExecutionException {
    LOGGER.debug("Checking binary exit value serialization");
    NIOParam lastParam = nt.getParams().getLast();
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("- Param Type: " + lastParam.getType().name());
        LOGGER.debug("- Preserve source data: " + lastParam.isPreserveSourceData());
        LOGGER.debug("- Write final value: " + lastParam.isWriteFinalValue());
        LOGGER.debug("- Prefix: " + lastParam.getPrefix());
    }
    // Last parameter is a FILE, direction OUT, with skip prefix => return in Python
    if (lastParam.getType().equals(DataType.FILE_T) && !lastParam.isPreserveSourceData() && lastParam.isWriteFinalValue() && lastParam.getPrefix().equals(Constants.PREFIX_SKIP)) {
        // Write exit value to the file
        String renaming = lastParam.getOriginalName();
        LOGGER.info("Writing Binary Exit Value (" + this.retValue.toString() + ") to " + renaming);
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(renaming))) {
            String value = "I" + this.retValue.toString() + "\n.\n";
            writer.write(value);
            writer.flush();
        } catch (IOException ioe) {
            throw new JobExecutionException("ERROR: Cannot serialize binary exit value for bindings", ioe);
        }
    }
}

Example 5 with NIOParam

use of es.bsc.compss.nio.NIOParam in project compss by bsc-wdc.

the class NIOWorker method receivedNewTask.

@Override
public void receivedNewTask(NIONode master, NIOTask task, List<String> obsoleteFiles) {
    WORKER_LOGGER.info("Received Job " + task);
    if (NIOTracer.isActivated()) {
        NIOTracer.emitEvent(NIOTracer.Event.WORKER_RECEIVED_NEW_TASK.getId(), NIOTracer.Event.WORKER_RECEIVED_NEW_TASK.getType());
    }
    // Remove obsolete
    if (obsoleteFiles != null) {
        removeObsolete(obsoleteFiles);
    }
    // Demand files
    WORKER_LOGGER.info("Checking parameters");
    TransferringTask tt = new TransferringTask(task);
    int i = 0;
    for (NIOParam param : task.getParams()) {
        i++;
        if (param.getData() != null) {
            // Parameter has associated data
            WORKER_LOGGER.debug("- Checking transfers for data of parameter " + (String) param.getValue());
            switch(param.getType()) {
                case OBJECT_T:
                    askForObject(param, i, tt);
                    break;
                case PSCO_T:
                    askForPSCO(param);
                    break;
                case EXTERNAL_OBJECT_T:
                    // Nothing to do since external parameters send their ID directly
                    break;
                case FILE_T:
                    askForFile(param, i, tt);
                    break;
                default:
                    // In any case, there is nothing to do for these type of parameters
                    break;
            }
        } else {
            // OUT parameter. Has no associated data. Decrease the parameter counter (we already have it)
            tt.decreaseParams();
        }
    }
    // Request the transfers
    if (NIOTracer.isActivated()) {
        NIOTracer.emitEvent(tt.getTask().getTaskId(), NIOTracer.getTaskTransfersType());
    }
    requestTransfers();
    if (NIOTracer.isActivated()) {
        NIOTracer.emitEvent(NIOTracer.EVENT_END, NIOTracer.getTaskTransfersType());
    }
    if (tt.getParams() == 0) {
        executeTask(tt.getTask());
    }
    if (NIOTracer.isActivated()) {
        NIOTracer.emitEvent(NIOTracer.EVENT_END, NIOTracer.Event.WORKER_RECEIVED_NEW_TASK.getType());
    }
}