Examples with Data es.bsc.compss.nio.commands.Data used on opensource projects

Example 1 with Data

use of es.bsc.compss.nio.commands.Data in project compss by bsc-wdc.

the class NIOJob method addParams.

private LinkedList<NIOParam> addParams() {
    LinkedList<NIOParam> params = new LinkedList<>();
    for (Parameter param : taskParams.getParameters()) {
        DataType type = param.getType();
        NIOParam np;
        switch(type) {
            case FILE_T:
            case OBJECT_T:
            case PSCO_T:
            case EXTERNAL_OBJECT_T:
                DependencyParameter dPar = (DependencyParameter) param;
                DataAccessId dAccId = dPar.getDataAccessId();
                Object value = dPar.getDataTarget();
                boolean preserveSourceData = true;
                if (dAccId instanceof RAccessId) {
                    // Parameter is a R, has sources
                    preserveSourceData = ((RAccessId) dAccId).isPreserveSourceData();
                } else if (dAccId instanceof RWAccessId) {
                    // Parameter is a RW, has sources
                    preserveSourceData = ((RWAccessId) dAccId).isPreserveSourceData();
                } else {
                    // Parameter is a W, it has no sources
                    preserveSourceData = false;
                }
                // Workaround for Python PSCOs in return
                // Check if the parameter has a valid PSCO and change its type
                String renaming;
                DataAccessId faId = dPar.getDataAccessId();
                if (faId instanceof WAccessId) {
                    // Write mode
                    WAccessId waId = (WAccessId) faId;
                    renaming = waId.getWrittenDataInstance().getRenaming();
                } else if (faId instanceof RWAccessId) {
                    // Read write mode
                    RWAccessId rwaId = (RWAccessId) faId;
                    renaming = rwaId.getWrittenDataInstance().getRenaming();
                } else {
                    // Read only mode
                    RAccessId raId = (RAccessId) faId;
                    renaming = raId.getReadDataInstance().getRenaming();
                }
                String pscoId = Comm.getData(renaming).getId();
                if (pscoId != null && type.equals(DataType.FILE_T)) {
                    param.setType(DataType.EXTERNAL_OBJECT_T);
                    type = param.getType();
                }
                // Create the NIO Param
                // Only store W and RW
                boolean writeFinalValue = !(dAccId instanceof RAccessId);
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, (Data) dPar.getDataSource(), dPar.getOriginalName());
                break;
            default:
                BasicTypeParameter btParB = (BasicTypeParameter) param;
                value = btParB.getValue();
                // Basic parameters are not preserved on Worker
                preserveSourceData = false;
                // Basic parameters are not stored on Worker
                writeFinalValue = false;
                np = new NIOParam(type, param.getStream(), param.getPrefix(), preserveSourceData, writeFinalValue, value, null, DependencyParameter.NO_NAME);
                break;
        }
        params.add(np);
    }
    return params;
}

Example 2 with Data

use of es.bsc.compss.nio.commands.Data in project compss by bsc-wdc.

the class NIOWorkerNode method sendData.

@Override
public void sendData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    if (target.getHosts().contains(Comm.getAppHost())) {
        // Order petition directly
        if (tgtData != null) {
            MultiURI u = ld.alreadyAvailable(Comm.getAppHost());
            if (u != null) {
                // Already present at the master
                reason.setDataTarget(u.getPath());
                listener.notifyEnd(null);
                return;
            }
        }
        Copy c = new DeferredCopy(ld, null, target, tgtData, reason, listener);
        Data d = new Data(ld);
        if (source != null) {
            for (MultiURI uri : source.getURIs()) {
                try {
                    NIOURI nURI = (NIOURI) uri.getInternalURI(NIOAdaptor.ID);
                    if (nURI != null) {
                        d.getSources().add(nURI);
                    }
                } catch (UnstartedNodeException une) {
                // Ignore internal URI
                }
            }
        }
        String path = target.getURIInHost(Comm.getAppHost()).getPath();
        c.setFinalTarget(path);
        ld.startCopy(c, c.getTargetLoc());
        DataRequest dr = new MasterDataRequest(c, reason.getType(), d, path);
        commManager.addTransferRequest(dr);
        commManager.requestTransfers();
    } else {
        // Request to any other
        orderCopy(new DeferredCopy(ld, source, target, tgtData, reason, listener));
    }
}

Example 3 with Data

use of es.bsc.compss.nio.commands.Data in project compss by bsc-wdc.

the class NIOAgent method requestTransfers.

/**
 * Check if receive slots available
 */
public void requestTransfers() {
    DataRequest dr = null;
    synchronized (pendingRequests) {
        if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
            dr = pendingRequests.remove();
        }
    }
    while (dr != null) {
        Data source = dr.getSource();
        NIOURI uri = source.getFirstURI();
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(source.getName());
        }
        NIONode nn = uri.getHost();
        if (nn.getIp() == null) {
            nn = masterNode;
        }
        Connection c = null;
        try {
            c = TM.startConnection(nn);
            LOGGER.debug("Connection " + c.hashCode() + " will be used to acquire data " + dr.getTarget() + " stored in " + nn + " with name " + dr.getSource().getName());
            Data remoteData = new Data(source.getName(), uri);
            CommandDataDemand cdd = new CommandDataDemand(this, remoteData, tracingID);
            ongoingTransfers.put(c, dr.getSource().getName());
            c.sendCommand(cdd);
            if (NIOTracer.isActivated()) {
                c.receive();
            }
            if (dr.getType() == DataType.FILE_T) {
                c.receiveDataFile(dr.getTarget());
            } else {
                c.receiveDataObject();
            }
        } catch (Exception e) {
            e.printStackTrace(System.err);
        } finally {
            if (c != null) {
                c.finishConnection();
            }
        }
        synchronized (pendingRequests) {
            if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
                dr = pendingRequests.remove();
            } else {
                dr = null;
            }
        }
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(NIOTracer.TRANSFER_END);
        }
    }
}

Example 4 with Data

use of es.bsc.compss.nio.commands.Data in project compss by bsc-wdc.

the class NIOWorkerNode method orderCopy.

private void orderCopy(DeferredCopy c) {
    LOGGER.info("Order Copy for " + c.getSourceData());
    Resource tgtRes = ((LinkedList<Resource>) c.getTargetLoc().getHosts()).getFirst();
    LogicalData ld = c.getSourceData();
    String path;
    synchronized (ld) {
        if (c.getTargetData() != null) {
            MultiURI u = ld.alreadyAvailable(tgtRes);
            if (u != null) {
                path = u.getPath();
            } else {
                path = c.getTargetLoc().getURIInHost(tgtRes).getPath();
            }
        } else {
            path = c.getTargetLoc().getURIInHost(tgtRes).getPath();
        }
        c.setProposedSource(new Data(ld));
        LOGGER.debug("Setting final target in deferred copy " + path);
        c.setFinalTarget(path);
        // TODO: MISSING CHECK IF FILE IS ALREADY BEEN COPIED IN A SHARED LOCATION
        ld.startCopy(c, c.getTargetLoc());
        commManager.registerCopy(c);
    }
    c.end(DataOperation.OpEndState.OP_OK);
}