Examples with Copy es.bsc.compss.types.data.operation.copy.Copy used on opensource projects

Example 1 with Copy

use of es.bsc.compss.types.data.operation.copy.Copy in project compss by bsc-wdc.

the class COMPSsMaster method obtainData.

@Override
public void obtainData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    LOGGER.info("Obtain Data " + ld.getName());
    /*
         * PSCO transfers are always available, if any SourceLocation is PSCO, don't transfer
         */
    for (DataLocation loc : ld.getLocations()) {
        if (loc.getProtocol().equals(Protocol.PERSISTENT_URI)) {
            LOGGER.debug("Object in Persistent Storage. Set dataTarget to " + loc.getPath());
            reason.setDataTarget(loc.getPath());
            listener.notifyEnd(null);
            return;
        }
    }
    /*
         * Otherwise the data is a file or an object that can be already in the master memory, in the master disk or
         * being transfered
         */
    String targetPath = target.getURIInHost(Comm.getAppHost()).getPath();
    // Check if data is in memory (no need to check if it is PSCO since previous case avoids it)
    if (ld.isInMemory()) {
        // Serialize value to file
        try {
            Serializer.serialize(ld.getValue(), targetPath);
        } catch (IOException ex) {
            ErrorManager.warn("Error copying file from memory to " + targetPath, ex);
        }
        if (tgtData != null) {
            tgtData.addLocation(target);
        }
        LOGGER.debug("Object in memory. Set dataTarget to " + targetPath);
        reason.setDataTarget(targetPath);
        listener.notifyEnd(null);
        return;
    }
    // Check if there are current copies in progress
    if (DEBUG) {
        LOGGER.debug("Data " + ld.getName() + " not in memory. Checking if there is a copy to the master in progress");
    }
    ld.lockHostRemoval();
    Collection<Copy> copiesInProgress = ld.getCopiesInProgress();
    if (copiesInProgress != null && !copiesInProgress.isEmpty()) {
        for (Copy copy : copiesInProgress) {
            if (copy != null) {
                if (copy.getTargetLoc() != null && copy.getTargetLoc().getHosts().contains(Comm.getAppHost())) {
                    if (DEBUG) {
                        LOGGER.debug("Copy in progress tranfering " + ld.getName() + "to master. Waiting for finishing");
                    }
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied set dataTarget " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copying file " + copy.getFinalTarget() + " from master to " + targetPath + " with replacing", ex);
                    }
                } else if (copy.getTargetData() != null && copy.getTargetData().getAllHosts().contains(Comm.getAppHost())) {
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied. Set data target to " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copy from " + copy.getFinalTarget() + " to " + targetPath + " with replacing", ex);
                    }
                } else {
                    if (DEBUG) {
                        LOGGER.debug("Current copies are not transfering " + ld.getName() + " to master. Ignoring at this moment");
                    }
                }
            }
        }
    }
    // Checking if file is already in master
    if (DEBUG) {
        LOGGER.debug("Checking if " + ld.getName() + " is at master (" + Comm.getAppHost().getName() + ").");
    }
    for (MultiURI u : ld.getURIs()) {
        if (DEBUG) {
            String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
            LOGGER.debug(ld.getName() + " is at " + u.toString() + "(" + hostname + ")");
        }
        if (u.getHost() == Comm.getAppHost()) {
            try {
                if (DEBUG) {
                    LOGGER.debug("Master local copy " + ld.getName() + " from " + u.getHost().getName() + " to " + targetPath);
                }
                Files.copy((new File(u.getPath())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                if (tgtData != null) {
                    tgtData.addLocation(target);
                }
                LOGGER.debug("File copied. Set data target to " + targetPath);
                reason.setDataTarget(targetPath);
                listener.notifyEnd(null);
                ld.releaseHostRemoval();
                return;
            } catch (IOException ex) {
                ErrorManager.warn("Error master local copy file from " + u.getPath() + " to " + targetPath + " with replacing", ex);
            }
        } else {
            if (DEBUG) {
                String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
                LOGGER.debug("Data " + ld.getName() + " copy in " + hostname + " not evaluated now");
            }
        }
    }
    // Ask the transfer from an specific source
    if (source != null) {
        for (Resource sourceRes : source.getHosts()) {
            COMPSsNode node = sourceRes.getNode();
            String sourcePath = source.getURIInHost(sourceRes).getPath();
            if (node != this) {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Sending data " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    node.sendData(ld, source, target, tgtData, reason, listener);
                } catch (Exception e) {
                    ErrorManager.warn("Not possible to sending data master to " + targetPath, e);
                    continue;
                }
                LOGGER.debug("Data " + ld.getName() + " sent.");
                ld.releaseHostRemoval();
                return;
            } else {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Local copy " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    Files.copy(new File(sourcePath).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                    LOGGER.debug("File copied. Set data target to " + targetPath);
                    reason.setDataTarget(targetPath);
                    listener.notifyEnd(null);
                    ld.releaseHostRemoval();
                    return;
                } catch (IOException ex) {
                    ErrorManager.warn("Error master local copy file from " + sourcePath + " to " + targetPath, ex);
                }
            }
        }
    } else {
        LOGGER.debug("Source data location is null. Trying other alternatives");
    }
    // Preferred source is null or copy has failed. Trying to retrieve data from any host
    for (Resource sourceRes : ld.getAllHosts()) {
        COMPSsNode node = sourceRes.getNode();
        if (node != this) {
            try {
                LOGGER.debug("Sending data " + ld.getName() + " from " + sourceRes.getName() + " to " + targetPath);
                node.sendData(ld, source, target, tgtData, reason, listener);
            } catch (Exception e) {
                LOGGER.error("Error: exception sending data", e);
                continue;
            }
            LOGGER.debug("Data " + ld.getName() + " sent.");
            ld.releaseHostRemoval();
            return;
        } else {
            if (DEBUG) {
                LOGGER.debug("Data " + ld.getName() + " copy in " + sourceRes.getName() + " not evaluated now. Should have been evaluated before");
            }
        }
    }
    // If we have not exited before, any copy method was successful. Raise warning
    ErrorManager.warn("Error file " + ld.getName() + " not transferred to " + targetPath);
    ld.releaseHostRemoval();
}

Example 2 with Copy

use of es.bsc.compss.types.data.operation.copy.Copy in project compss by bsc-wdc.

the class NIOWorkerNode method sendData.

@Override
public void sendData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    if (target.getHosts().contains(Comm.getAppHost())) {
        // Order petition directly
        if (tgtData != null) {
            MultiURI u = ld.alreadyAvailable(Comm.getAppHost());
            if (u != null) {
                // Already present at the master
                reason.setDataTarget(u.getPath());
                listener.notifyEnd(null);
                return;
            }
        }
        Copy c = new DeferredCopy(ld, null, target, tgtData, reason, listener);
        Data d = new Data(ld);
        if (source != null) {
            for (MultiURI uri : source.getURIs()) {
                try {
                    NIOURI nURI = (NIOURI) uri.getInternalURI(NIOAdaptor.ID);
                    if (nURI != null) {
                        d.getSources().add(nURI);
                    }
                } catch (UnstartedNodeException une) {
                // Ignore internal URI
                }
            }
        }
        String path = target.getURIInHost(Comm.getAppHost()).getPath();
        c.setFinalTarget(path);
        ld.startCopy(c, c.getTargetLoc());
        DataRequest dr = new MasterDataRequest(c, reason.getType(), d, path);
        commManager.addTransferRequest(dr);
        commManager.requestTransfers();
    } else {
        // Request to any other
        orderCopy(new DeferredCopy(ld, source, target, tgtData, reason, listener));
    }
}

Example 3 with Copy

use of es.bsc.compss.types.data.operation.copy.Copy in project compss by bsc-wdc.

the class GATWorkerNode method obtainData.

@Override
public void obtainData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    Copy c = new GATCopy(ld, source, target, tgtData, reason, listener);
    GATAdaptor.enqueueCopy(c);
}

Example 4 with Copy

use of es.bsc.compss.types.data.operation.copy.Copy in project compss by bsc-wdc.

the class GATWorkerNode method sendData.

@Override
public void sendData(LogicalData srcData, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    Copy c = new GATCopy(srcData, source, target, tgtData, reason, listener);
    GATAdaptor.enqueueCopy(c);
}

Example 5 with Copy

use of es.bsc.compss.types.data.operation.copy.Copy in project compss by bsc-wdc.

the class NIOAdaptor method handleRequestedDataNotAvailableError.

@Override
public void handleRequestedDataNotAvailableError(List<DataRequest> failedRequests, String dataId) {
    for (DataRequest dr : failedRequests) {
        MasterDataRequest mdr = (MasterDataRequest) dr;
        Copy c = (Copy) mdr.getOperation();
        c.getSourceData().finishedCopy(c);
        // Notify the copy has failed
        c.end(DataOperation.OpEndState.OP_FAILED);
    }
}