Examples with DataRequest es.bsc.compss.nio.dataRequest.DataRequest used on opensource projects

Example 1 with DataRequest

use of es.bsc.compss.nio.dataRequest.DataRequest in project compss by bsc-wdc.

the class NIOWorkerNode method sendData.

@Override
public void sendData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    if (target.getHosts().contains(Comm.getAppHost())) {
        // Order petition directly
        if (tgtData != null) {
            MultiURI u = ld.alreadyAvailable(Comm.getAppHost());
            if (u != null) {
                // Already present at the master
                reason.setDataTarget(u.getPath());
                listener.notifyEnd(null);
                return;
            }
        }
        Copy c = new DeferredCopy(ld, null, target, tgtData, reason, listener);
        Data d = new Data(ld);
        if (source != null) {
            for (MultiURI uri : source.getURIs()) {
                try {
                    NIOURI nURI = (NIOURI) uri.getInternalURI(NIOAdaptor.ID);
                    if (nURI != null) {
                        d.getSources().add(nURI);
                    }
                } catch (UnstartedNodeException une) {
                // Ignore internal URI
                }
            }
        }
        String path = target.getURIInHost(Comm.getAppHost()).getPath();
        c.setFinalTarget(path);
        ld.startCopy(c, c.getTargetLoc());
        DataRequest dr = new MasterDataRequest(c, reason.getType(), d, path);
        commManager.addTransferRequest(dr);
        commManager.requestTransfers();
    } else {
        // Request to any other
        orderCopy(new DeferredCopy(ld, source, target, tgtData, reason, listener));
    }
}

Example 2 with DataRequest

use of es.bsc.compss.nio.dataRequest.DataRequest in project compss by bsc-wdc.

the class NIOWorker method askForTransfer.

private void askForTransfer(boolean askTransfer, NIOParam param, int index, TransferringTask tt) {
    // Request the transfer if needed
    if (askTransfer) {
        WORKER_LOGGER.info("- Parameter " + index + "(" + (String) param.getValue() + ") does not exist, requesting data transfer");
        DataRequest dr = new WorkerDataRequest(tt, param.getType(), param.getData(), (String) param.getValue());
        addTransferRequest(dr);
    } else {
        // If no transfer, decrease the parameter counter
        // (we already have it)
        WORKER_LOGGER.info("- Parameter " + index + "(" + (String) param.getValue() + ") already exists.");
        tt.decreaseParams();
    }
}

Example 3 with DataRequest

use of es.bsc.compss.nio.dataRequest.DataRequest in project compss by bsc-wdc.

the class NIOAgent method receivedData.

/**
 * Received Data
 *
 * @param c
 * @param t
 */
public void receivedData(Connection c, Transfer t) {
    String dataId = ongoingTransfers.remove(c);
    if (dataId == null) {
        // execution
        return;
    }
    releaseReceiveSlot();
    List<DataRequest> requests = dataToRequests.remove(dataId);
    Map<String, List<DataRequest>> byTarget = new HashMap<>();
    for (DataRequest req : requests) {
        List<DataRequest> sameTarget = byTarget.get(req.getTarget());
        if (sameTarget == null) {
            sameTarget = new LinkedList<DataRequest>();
            byTarget.put(req.getTarget(), sameTarget);
        }
        sameTarget.add(req);
    }
    if (NIOTracer.isActivated()) {
        int tag = abs(dataId.hashCode());
        NIOTracer.emitDataTransferEvent(dataId);
        NIOTracer.emitCommEvent(false, connection2Partner.get(c), tag, t.getSize());
        connection2Partner.remove(c);
    }
    if (byTarget.size() == 1) {
        String targetName = requests.get(0).getTarget();
        receivedValue(t.getDestination(), targetName, t.getObject(), requests);
    } else {
        if (t.isFile()) {
            receivedValue(t.getDestination(), t.getFileName(), t.getObject(), byTarget.remove(t.getFileName()));
        } else {
            receivedValue(t.getDestination(), dataId, t.getObject(), byTarget.remove(dataId));
        }
        for (Entry<String, List<DataRequest>> entry : byTarget.entrySet()) {
            String targetName = entry.getKey();
            List<DataRequest> reqs = entry.getValue();
            try {
                if (t.isFile()) {
                    Files.copy((new File(t.getFileName())).toPath(), (new File(targetName)).toPath());
                    receivedValue(t.getDestination(), targetName, t.getObject(), byTarget.remove(targetName));
                } else {
                    Object o = Serializer.deserialize(t.getArray());
                    receivedValue(t.getDestination(), targetName, o, reqs);
                }
            } catch (IOException | ClassNotFoundException e) {
                LOGGER.warn("Can not replicate received Data", e);
            }
        }
    }
    requestTransfers();
    // Check if shutdown and ready
    if (finish == true && !hasPendingTransfers()) {
        shutdown(closingConnection);
    }
}

Example 4 with DataRequest

use of es.bsc.compss.nio.dataRequest.DataRequest in project compss by bsc-wdc.

the class NIOAdaptor method handleRequestedDataNotAvailableError.

@Override
public void handleRequestedDataNotAvailableError(List<DataRequest> failedRequests, String dataId) {
    for (DataRequest dr : failedRequests) {
        MasterDataRequest mdr = (MasterDataRequest) dr;
        Copy c = (Copy) mdr.getOperation();
        c.getSourceData().finishedCopy(c);
        // Notify the copy has failed
        c.end(DataOperation.OpEndState.OP_FAILED);
    }
}

Example 5 with DataRequest

use of es.bsc.compss.nio.dataRequest.DataRequest in project compss by bsc-wdc.

the class NIOAgent method requestTransfers.

/**
 * Check if receive slots available
 */
public void requestTransfers() {
    DataRequest dr = null;
    synchronized (pendingRequests) {
        if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
            dr = pendingRequests.remove();
        }
    }
    while (dr != null) {
        Data source = dr.getSource();
        NIOURI uri = source.getFirstURI();
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(source.getName());
        }
        NIONode nn = uri.getHost();
        if (nn.getIp() == null) {
            nn = masterNode;
        }
        Connection c = null;
        try {
            c = TM.startConnection(nn);
            LOGGER.debug("Connection " + c.hashCode() + " will be used to acquire data " + dr.getTarget() + " stored in " + nn + " with name " + dr.getSource().getName());
            Data remoteData = new Data(source.getName(), uri);
            CommandDataDemand cdd = new CommandDataDemand(this, remoteData, tracingID);
            ongoingTransfers.put(c, dr.getSource().getName());
            c.sendCommand(cdd);
            if (NIOTracer.isActivated()) {
                c.receive();
            }
            if (dr.getType() == DataType.FILE_T) {
                c.receiveDataFile(dr.getTarget());
            } else {
                c.receiveDataObject();
            }
        } catch (Exception e) {
            e.printStackTrace(System.err);
        } finally {
            if (c != null) {
                c.finishConnection();
            }
        }
        synchronized (pendingRequests) {
            if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
                dr = pendingRequests.remove();
            } else {
                dr = null;
            }
        }
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(NIOTracer.TRANSFER_END);
        }
    }
}