Examples with MultiURI es.bsc.compss.types.uri.MultiURI used on opensource projects

Example 1 with MultiURI

use of es.bsc.compss.types.uri.MultiURI in project compss by bsc-wdc.

the class Comm method removeData.

/**
 * Removes the data with id @renaming
 *
 * @param renaming
 */
public static synchronized void removeData(String renaming) {
    LOGGER.debug("Remove data " + renaming);
    LogicalData ld = data.remove(renaming);
    ld.isObsolete();
    for (DataLocation dl : ld.getLocations()) {
        MultiURI uri = dl.getURIInHost(appHost);
        if (uri != null) {
            File f = new File(uri.getPath());
            if (f.exists()) {
                LOGGER.info("Deleting file " + f.getAbsolutePath());
                if (!f.delete()) {
                    LOGGER.error("Cannot delete file " + f.getAbsolutePath());
                }
            }
        }
    }
}

Example 2 with MultiURI

use of es.bsc.compss.types.uri.MultiURI in project compss by bsc-wdc.

the class COMPSsMaster method obtainData.

@Override
public void obtainData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    LOGGER.info("Obtain Data " + ld.getName());
    /*
         * PSCO transfers are always available, if any SourceLocation is PSCO, don't transfer
         */
    for (DataLocation loc : ld.getLocations()) {
        if (loc.getProtocol().equals(Protocol.PERSISTENT_URI)) {
            LOGGER.debug("Object in Persistent Storage. Set dataTarget to " + loc.getPath());
            reason.setDataTarget(loc.getPath());
            listener.notifyEnd(null);
            return;
        }
    }
    /*
         * Otherwise the data is a file or an object that can be already in the master memory, in the master disk or
         * being transfered
         */
    String targetPath = target.getURIInHost(Comm.getAppHost()).getPath();
    // Check if data is in memory (no need to check if it is PSCO since previous case avoids it)
    if (ld.isInMemory()) {
        // Serialize value to file
        try {
            Serializer.serialize(ld.getValue(), targetPath);
        } catch (IOException ex) {
            ErrorManager.warn("Error copying file from memory to " + targetPath, ex);
        }
        if (tgtData != null) {
            tgtData.addLocation(target);
        }
        LOGGER.debug("Object in memory. Set dataTarget to " + targetPath);
        reason.setDataTarget(targetPath);
        listener.notifyEnd(null);
        return;
    }
    // Check if there are current copies in progress
    if (DEBUG) {
        LOGGER.debug("Data " + ld.getName() + " not in memory. Checking if there is a copy to the master in progress");
    }
    ld.lockHostRemoval();
    Collection<Copy> copiesInProgress = ld.getCopiesInProgress();
    if (copiesInProgress != null && !copiesInProgress.isEmpty()) {
        for (Copy copy : copiesInProgress) {
            if (copy != null) {
                if (copy.getTargetLoc() != null && copy.getTargetLoc().getHosts().contains(Comm.getAppHost())) {
                    if (DEBUG) {
                        LOGGER.debug("Copy in progress tranfering " + ld.getName() + "to master. Waiting for finishing");
                    }
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied set dataTarget " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copying file " + copy.getFinalTarget() + " from master to " + targetPath + " with replacing", ex);
                    }
                } else if (copy.getTargetData() != null && copy.getTargetData().getAllHosts().contains(Comm.getAppHost())) {
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied. Set data target to " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copy from " + copy.getFinalTarget() + " to " + targetPath + " with replacing", ex);
                    }
                } else {
                    if (DEBUG) {
                        LOGGER.debug("Current copies are not transfering " + ld.getName() + " to master. Ignoring at this moment");
                    }
                }
            }
        }
    }
    // Checking if file is already in master
    if (DEBUG) {
        LOGGER.debug("Checking if " + ld.getName() + " is at master (" + Comm.getAppHost().getName() + ").");
    }
    for (MultiURI u : ld.getURIs()) {
        if (DEBUG) {
            String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
            LOGGER.debug(ld.getName() + " is at " + u.toString() + "(" + hostname + ")");
        }
        if (u.getHost() == Comm.getAppHost()) {
            try {
                if (DEBUG) {
                    LOGGER.debug("Master local copy " + ld.getName() + " from " + u.getHost().getName() + " to " + targetPath);
                }
                Files.copy((new File(u.getPath())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                if (tgtData != null) {
                    tgtData.addLocation(target);
                }
                LOGGER.debug("File copied. Set data target to " + targetPath);
                reason.setDataTarget(targetPath);
                listener.notifyEnd(null);
                ld.releaseHostRemoval();
                return;
            } catch (IOException ex) {
                ErrorManager.warn("Error master local copy file from " + u.getPath() + " to " + targetPath + " with replacing", ex);
            }
        } else {
            if (DEBUG) {
                String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
                LOGGER.debug("Data " + ld.getName() + " copy in " + hostname + " not evaluated now");
            }
        }
    }
    // Ask the transfer from an specific source
    if (source != null) {
        for (Resource sourceRes : source.getHosts()) {
            COMPSsNode node = sourceRes.getNode();
            String sourcePath = source.getURIInHost(sourceRes).getPath();
            if (node != this) {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Sending data " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    node.sendData(ld, source, target, tgtData, reason, listener);
                } catch (Exception e) {
                    ErrorManager.warn("Not possible to sending data master to " + targetPath, e);
                    continue;
                }
                LOGGER.debug("Data " + ld.getName() + " sent.");
                ld.releaseHostRemoval();
                return;
            } else {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Local copy " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    Files.copy(new File(sourcePath).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                    LOGGER.debug("File copied. Set data target to " + targetPath);
                    reason.setDataTarget(targetPath);
                    listener.notifyEnd(null);
                    ld.releaseHostRemoval();
                    return;
                } catch (IOException ex) {
                    ErrorManager.warn("Error master local copy file from " + sourcePath + " to " + targetPath, ex);
                }
            }
        }
    } else {
        LOGGER.debug("Source data location is null. Trying other alternatives");
    }
    // Preferred source is null or copy has failed. Trying to retrieve data from any host
    for (Resource sourceRes : ld.getAllHosts()) {
        COMPSsNode node = sourceRes.getNode();
        if (node != this) {
            try {
                LOGGER.debug("Sending data " + ld.getName() + " from " + sourceRes.getName() + " to " + targetPath);
                node.sendData(ld, source, target, tgtData, reason, listener);
            } catch (Exception e) {
                LOGGER.error("Error: exception sending data", e);
                continue;
            }
            LOGGER.debug("Data " + ld.getName() + " sent.");
            ld.releaseHostRemoval();
            return;
        } else {
            if (DEBUG) {
                LOGGER.debug("Data " + ld.getName() + " copy in " + sourceRes.getName() + " not evaluated now. Should have been evaluated before");
            }
        }
    }
    // If we have not exited before, any copy method was successful. Raise warning
    ErrorManager.warn("Error file " + ld.getName() + " not transferred to " + targetPath);
    ld.releaseHostRemoval();
}

Example 3 with MultiURI

use of es.bsc.compss.types.uri.MultiURI in project compss by bsc-wdc.

the class NIOWorkerNode method sendData.

@Override
public void sendData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    if (target.getHosts().contains(Comm.getAppHost())) {
        // Order petition directly
        if (tgtData != null) {
            MultiURI u = ld.alreadyAvailable(Comm.getAppHost());
            if (u != null) {
                // Already present at the master
                reason.setDataTarget(u.getPath());
                listener.notifyEnd(null);
                return;
            }
        }
        Copy c = new DeferredCopy(ld, null, target, tgtData, reason, listener);
        Data d = new Data(ld);
        if (source != null) {
            for (MultiURI uri : source.getURIs()) {
                try {
                    NIOURI nURI = (NIOURI) uri.getInternalURI(NIOAdaptor.ID);
                    if (nURI != null) {
                        d.getSources().add(nURI);
                    }
                } catch (UnstartedNodeException une) {
                // Ignore internal URI
                }
            }
        }
        String path = target.getURIInHost(Comm.getAppHost()).getPath();
        c.setFinalTarget(path);
        ld.startCopy(c, c.getTargetLoc());
        DataRequest dr = new MasterDataRequest(c, reason.getType(), d, path);
        commManager.addTransferRequest(dr);
        commManager.requestTransfers();
    } else {
        // Request to any other
        orderCopy(new DeferredCopy(ld, source, target, tgtData, reason, listener));
    }
}

Example 4 with MultiURI

use of es.bsc.compss.types.uri.MultiURI in project compss by bsc-wdc.

the class GATCopy method specificCopy.

@Override
public void specificCopy() throws CopyException {
    LOGGER.debug(DBG_PREFIX + "Performing GAT Specific Copy for " + getName());
    // Fetch valid destination URIs
    List<MultiURI> targetURIs = tgtLoc.getURIs();
    List<URI> selectedTargetURIs = new LinkedList<>();
    for (MultiURI uri : targetURIs) {
        try {
            URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
            if (internalURI != null) {
                selectedTargetURIs.add(internalURI);
            }
        } catch (UnstartedNodeException une) {
            throw new GATCopyException(une);
        }
    }
    if (selectedTargetURIs.isEmpty()) {
        LOGGER.error(DBG_PREFIX + ERR_NO_TGT_URI);
        throw new GATCopyException(ERR_NO_TGT_URI);
    }
    LOGGER.debug(DBG_PREFIX + "Selected target URIs");
    // Fetch valid source URIs
    List<MultiURI> sourceURIs;
    List<URI> selectedSourceURIs = new LinkedList<>();
    synchronized (srcData) {
        if (srcLoc != null) {
            sourceURIs = srcLoc.getURIs();
            for (MultiURI uri : sourceURIs) {
                try {
                    URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                    if (internalURI != null) {
                        selectedSourceURIs.add(internalURI);
                    }
                } catch (UnstartedNodeException une) {
                    LOGGER.error(DBG_PREFIX + "Exception selecting source URI");
                    throw new GATCopyException(une);
                }
            }
        }
        sourceURIs = srcData.getURIs();
        for (MultiURI uri : sourceURIs) {
            try {
                URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                if (internalURI != null) {
                    selectedSourceURIs.add(internalURI);
                }
            } catch (UnstartedNodeException une) {
                LOGGER.error(DBG_PREFIX + "Exception selecting source URI for " + getName());
                throw new GATCopyException(une);
            }
        }
        if (selectedSourceURIs.isEmpty()) {
            if (srcData.isInMemory()) {
                LOGGER.debug("Data for " + getName() + " is in memory");
                try {
                    srcData.writeToStorage();
                    sourceURIs = srcData.getURIs();
                    for (MultiURI uri : sourceURIs) {
                        URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                        if (internalURI != null) {
                            selectedSourceURIs.add(internalURI);
                        }
                    }
                } catch (Exception e) {
                    LOGGER.fatal("Exception writing object to file.", e);
                    throw new GATCopyException(ERR_NO_SRC_URI);
                }
            } else {
                LOGGER.error(DBG_PREFIX + ERR_NO_SRC_URI);
                throw new GATCopyException(ERR_NO_SRC_URI);
            }
        }
    }
    GATInvocationException exception = new GATInvocationException("default logical file");
    for (URI src : selectedSourceURIs) {
        for (URI tgt : selectedTargetURIs) {
            try {
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("GATCopy From: " + src + " to " + tgt);
                }
                // Source and target URIs contain Runtime information (schema)
                // Convert it to GAT format
                URI gatSrc = new URI(DataLocation.Protocol.ANY_URI.getSchema() + src.getHost() + "/" + src.getPath());
                URI gatTgt = new URI(DataLocation.Protocol.ANY_URI.getSchema() + tgt.getHost() + "/" + tgt.getPath());
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("GATCopy From: " + gatSrc + " to " + gatTgt);
                }
                doCopy(gatSrc, gatTgt);
            // Try to copy from each location until successful
            } catch (Exception e) {
                exception.add("default logical file", e);
                LOGGER.warn("Error copying file", e);
                continue;
            }
            return;
        }
    }
    if (!(this.reason instanceof WorkersDebugInfoCopyTransferable)) {
        ErrorManager.error("File '" + srcData.getName() + "' could not be copied because it does not exist.", exception);
    }
    throw new GATCopyException(exception);
}

Example 5 with MultiURI

use of es.bsc.compss.types.uri.MultiURI in project compss by bsc-wdc.

the class PersistentLocation method getURIs.

@Override
public List<MultiURI> getURIs() {
    // Retrieve locations from Back-end
    List<String> locations = null;
    try {
        locations = StorageItf.getLocations(this.id);
    } catch (StorageException e) {
        ErrorManager.error("ERROR: Cannot retrieve locations of " + this.id + " from Storage Back-end");
    }
    if (locations == null) {
        ErrorManager.error("ERROR: Cannot retrieve locations of " + this.id + " from Storage Back-end");
    }
    // Retrieve URIs from hosts
    LinkedList<MultiURI> uris = new LinkedList<>();
    for (String hostName : locations) {
        Resource host = Resource.getResource(hostName);
        if (host != null) {
            uris.add(new MultiURI(Protocol.PERSISTENT_URI, host, this.id));
        } else {
            LOGGER.warn("Storage Back-End returned non-registered host " + hostName + ". Skipping URI in host");
        }
    }
    return uris;
}