Examples with DependencyParameter es.bsc.compss.types.parameter.DependencyParameter used on opensource projects

Example 1 with DependencyParameter

use of es.bsc.compss.types.parameter.DependencyParameter in project compss by bsc-wdc.

the class ScoresTest method testResourceScores.

@SuppressWarnings("unchecked")
@Test
public void testResourceScores() throws BlockedActionException, UnassignedActionException, Exception {
    drs1.clear();
    FakeAllocatableAction action1 = new FakeAllocatableAction(fao, 1, 0, CoreManager.getCoreImplementations(0));
    DataInstanceId d1v1 = new DataInstanceId(1, 1);
    Comm.registerData(d1v1.getRenaming());
    DataInstanceId d2v2 = new DataInstanceId(2, 2);
    Comm.registerData(d2v2.getRenaming());
    DependencyParameter dpD1V1 = new DependencyParameter(DataType.FILE_T, Direction.IN, Stream.UNSPECIFIED, Constants.PREFIX_EMTPY);
    dpD1V1.setDataAccessId(new RAccessId(1, 1));
    DependencyParameter dpD2V2 = new DependencyParameter(DataType.FILE_T, Direction.IN, Stream.UNSPECIFIED, Constants.PREFIX_EMTPY);
    dpD2V2.setDataAccessId(new RAccessId(2, 2));
    TaskDescription params = new TaskDescription("task", false, Constants.SINGLE_NODE, false, false, false, false, new Parameter[] { dpD1V1, dpD2V2 });
    FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation> actionScore = (FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation>) ds.generateActionScore(action1);
    FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation> score1 = (FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation>) drs1.generateResourceScore(action1, params, actionScore);
    Verifiers.verifyScore(score1, 0, 2 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY, 0, 0, 2 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY);
    Comm.registerLocation(d1v1.getRenaming(), DataLocation.createLocation(drs1.getResource(), new SimpleURI("/home/test/a")));
    score1 = (FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation>) drs1.generateResourceScore(action1, params, actionScore);
    Verifiers.verifyScore(score1, 0, 1 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY, 0, 0, 1 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY);
    Comm.registerLocation(d2v2.getRenaming(), DataLocation.createLocation(drs1.getResource(), new SimpleURI("/home/test/b")));
    score1 = (FullGraphScore<FakeProfile, FakeResourceDescription, FakeImplementation>) drs1.generateResourceScore(action1, params, actionScore);
    Verifiers.verifyScore(score1, 0, 0 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY, 0, 0, 0 * FullGraphResourceScheduler.DATA_TRANSFER_DELAY);
    Comm.removeData(d1v1.getRenaming());
    Comm.removeData(d2v2.getRenaming());
}

Example 2 with DependencyParameter

use of es.bsc.compss.types.parameter.DependencyParameter in project compss by bsc-wdc.

the class NIOJob method addParams.

private LinkedList<NIOParam> addParams() {
    LinkedList<NIOParam> params = new LinkedList<>();
    for (Parameter param : taskParams.getParameters()) {
        DataType type = param.getType();
        NIOParam np;
        switch(type) {
            case FILE_T:
            case OBJECT_T:
            case PSCO_T:
            case EXTERNAL_OBJECT_T:
                DependencyParameter dPar = (DependencyParameter) param;
                DataAccessId dAccId = dPar.getDataAccessId();
                Object value = dPar.getDataTarget();
                boolean preserveSourceData = true;
                if (dAccId instanceof RAccessId) {
                    // Parameter is a R, has sources
                    preserveSourceData = ((RAccessId) dAccId).isPreserveSourceData();
                } else if (dAccId instanceof RWAccessId) {
                    // Parameter is a RW, has sources
                    preserveSourceData = ((RWAccessId) dAccId).isPreserveSourceData();
                } else {
                    // Parameter is a W, it has no sources
                    preserveSourceData = false;
                }
                // Workaround for Python PSCOs in return
                // Check if the parameter has a valid PSCO and change its type
                String renaming;
                DataAccessId faId = dPar.getDataAccessId();
                if (faId instanceof WAccessId) {
                    // Write mode
                    WAccessId waId = (WAccessId) faId;
                    renaming = waId.getWrittenDataInstance().getRenaming();
                } else if (faId instanceof RWAccessId) {
                    // Read write mode
                    RWAccessId rwaId = (RWAccessId) faId;
                    renaming = rwaId.getWrittenDataInstance().getRenaming();
                } else {
                    // Read only mode
                    RAccessId raId = (RAccessId) faId;
                    renaming = raId.getReadDataInstance().getRenaming();
                }
                String pscoId = Comm.getData(renaming).getId();
                if (pscoId != null && type.equals(DataType.FILE_T)) {
                    param.setType(DataType.EXTERNAL_OBJECT_T);
                    type = param.getType();
                }
                // Create the NIO Param
                // Only store W and RW
                boolean writeFinalValue = !(dAccId instanceof RAccessId);
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, (Data) dPar.getDataSource(), dPar.getOriginalName());
                break;
            default:
                BasicTypeParameter btParB = (BasicTypeParameter) param;
                value = btParB.getValue();
                // Basic parameters are not preserved on Worker
                preserveSourceData = false;
                // Basic parameters are not stored on Worker
                writeFinalValue = false;
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, null, DependencyParameter.NO_NAME);
                break;
        }
        params.add(np);
    }
    return params;
}

Example 3 with DependencyParameter

use of es.bsc.compss.types.parameter.DependencyParameter in project compss by bsc-wdc.

the class TaskAnalyser method endTask.

/**
 * Registers the end of execution of task @task
 *
 * @param task
 */
public void endTask(Task task) {
    int taskId = task.getId();
    boolean isFree = task.isFree();
    TaskState taskState = task.getStatus();
    LOGGER.info("Notification received for task " + taskId + " with end status " + taskState);
    // Check status
    if (!isFree) {
        LOGGER.debug("Task " + taskId + " is not registered as free. Waiting for other executions to end");
        return;
    }
    if (taskState == TaskState.FAILED) {
        ErrorManager.error(TASK_FAILED + task);
        return;
    }
    /*
         * Treat end of task
         */
    LOGGER.debug("Ending task " + taskId);
    // Free dependencies
    Long appId = task.getAppId();
    Integer taskCount = appIdToTaskCount.get(appId) - 1;
    appIdToTaskCount.put(appId, taskCount);
    if (taskCount == 0) {
        // Remove the appId from the barrier flags (if existant, otherwise do nothing)
        appIdBarrierFlags.remove(appId);
        Semaphore sem = appIdToSemaphore.remove(appId);
        if (sem != null) {
            // App was synchornized on a barrier flag or a no more tasks
            // Release the application semaphore
            appIdToTaskCount.remove(appId);
            sem.release();
        }
    }
    // Check if task is being waited
    List<Semaphore> sems = waitedTasks.remove(task);
    if (sems != null) {
        for (Semaphore sem : sems) {
            sem.release();
        }
    }
    for (Parameter param : task.getTaskDescription().getParameters()) {
        DataType type = param.getType();
        if (type == DataType.FILE_T || type == DataType.OBJECT_T || type == DataType.PSCO_T || type == DataType.EXTERNAL_OBJECT_T) {
            DependencyParameter dPar = (DependencyParameter) param;
            DataAccessId dAccId = dPar.getDataAccessId();
            LOGGER.debug("Treating that data " + dAccId + " has been accessed at " + dPar.getDataTarget());
            DIP.dataHasBeenAccessed(dAccId);
        }
    }
    // Task generation is finished if we are on noMoreTasks but we are not on a barrier
    if (appIdToSemaphore.get(appId) != null && !appIdBarrierFlags.contains(appId)) {
        checkResultFileTransfer(task);
    }
    // Release data dependent tasks
    task.releaseDataDependents();
}

Example 4 with DependencyParameter

use of es.bsc.compss.types.parameter.DependencyParameter in project compss by bsc-wdc.

the class TaskAnalyser method processTask.

/**
 * Process the dependencies of a new task @currentTask
 *
 * @param currentTask
 */
public void processTask(Task currentTask) {
    TaskDescription params = currentTask.getTaskDescription();
    LOGGER.info("New " + (params.getType() == TaskType.METHOD ? "method" : "service") + " task(" + params.getName() + "), ID = " + currentTask.getId());
    if (drawGraph) {
        addNewTask(currentTask);
    }
    // Update task count
    Integer methodId = params.getId();
    Integer actualCount = currentTaskCount.get(methodId);
    if (actualCount == null) {
        actualCount = 0;
    }
    currentTaskCount.put(methodId, actualCount + 1);
    // Update app id task count
    Long appId = currentTask.getAppId();
    Integer taskCount = appIdToTaskCount.get(appId);
    if (taskCount == null) {
        taskCount = 0;
    }
    taskCount++;
    appIdToTaskCount.put(appId, taskCount);
    Integer totalTaskCount = appIdToTotalTaskCount.get(appId);
    if (totalTaskCount == null) {
        totalTaskCount = 0;
    }
    totalTaskCount++;
    appIdToTotalTaskCount.put(appId, totalTaskCount);
    // Check scheduling enforcing data
    int constrainingParam = -1;
    if (params.getType() == TaskType.SERVICE && params.hasTargetObject()) {
        if (params.hasReturnValue()) {
            constrainingParam = params.getParameters().length - 2;
        } else {
            constrainingParam = params.getParameters().length - 1;
        }
    }
    Parameter[] parameters = params.getParameters();
    for (int paramIdx = 0; paramIdx < parameters.length; paramIdx++) {
        Parameter p = parameters[paramIdx];
        if (DEBUG) {
            LOGGER.debug("* Parameter : " + p);
        }
        // Conversion: direction -> access mode
        AccessMode am = AccessMode.R;
        switch(p.getDirection()) {
            case IN:
                am = AccessMode.R;
                break;
            case OUT:
                am = AccessMode.W;
                break;
            case INOUT:
                am = AccessMode.RW;
                break;
        }
        // Inform the Data Manager about the new accesses
        DataAccessId daId;
        switch(p.getType()) {
            case FILE_T:
                FileParameter fp = (FileParameter) p;
                daId = DIP.registerFileAccess(am, fp.getLocation());
                break;
            case PSCO_T:
                ObjectParameter pscop = (ObjectParameter) p;
                // Check if its PSCO class and persisted to infer its type
                pscop.setType(DataType.PSCO_T);
                daId = DIP.registerObjectAccess(am, pscop.getValue(), pscop.getCode());
                break;
            case EXTERNAL_OBJECT_T:
                ExternalObjectParameter externalPSCOparam = (ExternalObjectParameter) p;
                // Check if its PSCO class and persisted to infer its type
                externalPSCOparam.setType(DataType.EXTERNAL_OBJECT_T);
                daId = DIP.registerExternalObjectAccess(am, externalPSCOparam.getId(), externalPSCOparam.getCode());
                break;
            case OBJECT_T:
                ObjectParameter op = (ObjectParameter) p;
                // Check if its PSCO class and persisted to infer its type
                if (op.getValue() instanceof StubItf && ((StubItf) op.getValue()).getID() != null) {
                    op.setType(DataType.PSCO_T);
                }
                daId = DIP.registerObjectAccess(am, op.getValue(), op.getCode());
                break;
            default:
                /*
                     * Basic types (including String). The only possible access mode is R (already checked by the API)
                     */
                continue;
        }
        // Add dependencies to the graph and register output values for future dependencies
        DependencyParameter dp = (DependencyParameter) p;
        dp.setDataAccessId(daId);
        switch(am) {
            case R:
                checkDependencyForRead(currentTask, dp);
                if (paramIdx == constrainingParam) {
                    DataAccessId.RAccessId raId = (DataAccessId.RAccessId) dp.getDataAccessId();
                    DataInstanceId dependingDataId = raId.getReadDataInstance();
                    if (dependingDataId != null) {
                        if (dependingDataId.getVersionId() > 1) {
                            currentTask.setEnforcingTask(writers.get(dependingDataId.getDataId()));
                        }
                    }
                }
                break;
            case RW:
                checkDependencyForRead(currentTask, dp);
                if (paramIdx == constrainingParam) {
                    DataAccessId.RWAccessId raId = (DataAccessId.RWAccessId) dp.getDataAccessId();
                    DataInstanceId dependingDataId = raId.getReadDataInstance();
                    if (dependingDataId != null) {
                        if (dependingDataId.getVersionId() > 1) {
                            currentTask.setEnforcingTask(writers.get(dependingDataId.getDataId()));
                        }
                    }
                }
                registerOutputValues(currentTask, dp);
                break;
            case W:
                registerOutputValues(currentTask, dp);
                break;
        }
    }
}

Example 5 with DependencyParameter

use of es.bsc.compss.types.parameter.DependencyParameter in project compss by bsc-wdc.

the class GATJob method processParameters.

private void processParameters(String sandboxPath, ArrayList<String> symlinks, ArrayList<String> lArgs) {
    logger.debug("Processing parameters for GAT job " + this.jobId);
    lArgs.add(Boolean.toString(taskParams.hasTargetObject()));
    // Add return type
    if (taskParams.hasReturnValue()) {
        Parameter returnParam = taskParams.getParameters()[taskParams.getParameters().length - 1];
        lArgs.add(Integer.toString(returnParam.getType().ordinal()));
    } else {
        lArgs.add("null");
    }
    // Add parameters
    int numParams = taskParams.getParameters().length;
    if (taskParams.hasReturnValue()) {
        numParams--;
    }
    lArgs.add(Integer.toString(numParams));
    for (Parameter param : taskParams.getParameters()) {
        DataType type = param.getType();
        lArgs.add(Integer.toString(type.ordinal()));
        lArgs.add(Integer.toString(param.getStream().ordinal()));
        String prefix = param.getPrefix();
        if (prefix == null || prefix.isEmpty()) {
            prefix = Constants.PREFIX_EMTPY;
        }
        lArgs.add(param.getPrefix());
        switch(type) {
            case FILE_T:
                DependencyParameter dFilePar = (DependencyParameter) param;
                java.io.File f = new java.io.File(dFilePar.getDataTarget());
                if (!f.getName().equals(dFilePar.getOriginalName())) {
                    // Add file to manage symlinks and renames
                    String originalName = sandboxPath + File.separator + dFilePar.getOriginalName();
                    symlinks.add(dFilePar.getDataTarget());
                    symlinks.add(dFilePar.getOriginalName());
                    lArgs.add(originalName);
                } else {
                    // Original and target is the same nothing to do
                    lArgs.add(dFilePar.getDataTarget());
                }
                break;
            case PSCO_T:
            case EXTERNAL_OBJECT_T:
                logger.error("GAT Adaptor does not support PSCO Types");
                listener.jobFailed(this, JobEndStatus.SUBMISSION_FAILED);
                break;
            case OBJECT_T:
                DependencyParameter dPar = (DependencyParameter) param;
                DataAccessId dAccId = dPar.getDataAccessId();
                lArgs.add(dPar.getDataTarget());
                if (dAccId instanceof RAccessId) {
                    lArgs.add("R");
                } else {
                    // for the worker to know it must write the object to disk
                    lArgs.add("W");
                }
                break;
            case STRING_T:
                BasicTypeParameter btParS = (BasicTypeParameter) param;
                // Check spaces
                String value = btParS.getValue().toString();
                int numSubStrings = value.split(" ").length;
                lArgs.add(Integer.toString(numSubStrings));
                lArgs.add(value);
                break;
            default:
                // Basic Types
                BasicTypeParameter btParB = (BasicTypeParameter) param;
                lArgs.add(btParB.getValue().toString());
                break;
        }
    }
}