Examples with Resource es.bsc.compss.types.resources.Resource used on opensource projects

Example 1 with Resource

use of es.bsc.compss.types.resources.Resource in project compss by bsc-wdc.

the class ResourceOptimizer method getBestDestruction.

/**
 * Given a set of resources, it checks every possible modification of the resource and returns the one that better
 * fits with the destruction recommendations.
 *
 * The decision-making algorithm tries to minimize the number of affected CE that weren't recommended to be
 * modified, minimize the number of slots that weren't requested to be destroyed and maximize the number of slots
 * that can be removed and they were requested for.
 *
 * @param resourceSet
 *            set of resources
 * @param destroyRecommendations
 *            number of slots to be removed for each CE
 * @return an object array defining the best solution.
 *
 *         0-> (Resource) selected Resource.
 *
 *         1->(CloudTypeInstanceDescription) Type to be destroyed to be destroyed.
 *
 *         2-> (int[]) record of the #CE with removed slots and that they shouldn't be modified, #slots that will be
 *         destroyed and they weren't recommended, #slots that will be removed and they were asked to be.
 */
private Object[] getBestDestruction(Collection<CloudMethodWorker> resourceSet, float[] destroyRecommendations) {
    CloudProvider cp;
    float[] bestRecord = new float[3];
    bestRecord[0] = Float.MAX_VALUE;
    bestRecord[1] = Float.MAX_VALUE;
    bestRecord[2] = Float.MIN_VALUE;
    Resource bestResource = null;
    CloudInstanceTypeDescription bestType = null;
    for (CloudMethodWorker res : resourceSet) {
        cp = res.getProvider();
        if (cp == null) {
            // it's not a cloud machine
            if (DEBUG) {
                RUNTIME_LOGGER.debug("Resource " + res.getName() + " is not cloud. Skipping...");
            }
            continue;
        }
        Map<CloudInstanceTypeDescription, float[]> typeToPoints = getPossibleReductions(res, destroyRecommendations);
        for (Map.Entry<CloudInstanceTypeDescription, float[]> destruction : typeToPoints.entrySet()) {
            CloudInstanceTypeDescription type = destruction.getKey();
            float[] values = destruction.getValue();
            if (DEBUG) {
                RUNTIME_LOGGER.debug("Type: " + type.getName() + " value 0: " + values[0] + " (" + bestRecord[0] + ") " + " value 1: " + values[1] + " (" + bestRecord[1] + ") " + " value 2: " + values[2] + " (" + bestRecord[2] + ")");
            }
            if (bestRecord[0] == values[0]) {
                if (bestRecord[1] == values[1]) {
                    if (bestRecord[2] < values[2]) {
                        bestRecord = values;
                        bestResource = res;
                        bestType = type;
                    }
                } else if (bestRecord[1] > values[1]) {
                    bestRecord = values;
                    bestResource = res;
                    bestType = type;
                }
            } else if (bestRecord[0] > values[0]) {
                bestRecord = values;
                bestResource = res;
                bestType = type;
            }
        }
    }
    if (bestResource != null) {
        Object[] ret = new Object[3];
        ret[0] = bestResource;
        ret[1] = bestType;
        ret[2] = bestRecord;
        return ret;
    } else {
        RUNTIME_LOGGER.warn("Best resource to remove not found");
        return null;
    }
}

Example 2 with Resource

use of es.bsc.compss.types.resources.Resource in project compss by bsc-wdc.

the class COMPSsMaster method obtainData.

@Override
public void obtainData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    LOGGER.info("Obtain Data " + ld.getName());
    /*
         * PSCO transfers are always available, if any SourceLocation is PSCO, don't transfer
         */
    for (DataLocation loc : ld.getLocations()) {
        if (loc.getProtocol().equals(Protocol.PERSISTENT_URI)) {
            LOGGER.debug("Object in Persistent Storage. Set dataTarget to " + loc.getPath());
            reason.setDataTarget(loc.getPath());
            listener.notifyEnd(null);
            return;
        }
    }
    /*
         * Otherwise the data is a file or an object that can be already in the master memory, in the master disk or
         * being transfered
         */
    String targetPath = target.getURIInHost(Comm.getAppHost()).getPath();
    // Check if data is in memory (no need to check if it is PSCO since previous case avoids it)
    if (ld.isInMemory()) {
        // Serialize value to file
        try {
            Serializer.serialize(ld.getValue(), targetPath);
        } catch (IOException ex) {
            ErrorManager.warn("Error copying file from memory to " + targetPath, ex);
        }
        if (tgtData != null) {
            tgtData.addLocation(target);
        }
        LOGGER.debug("Object in memory. Set dataTarget to " + targetPath);
        reason.setDataTarget(targetPath);
        listener.notifyEnd(null);
        return;
    }
    // Check if there are current copies in progress
    if (DEBUG) {
        LOGGER.debug("Data " + ld.getName() + " not in memory. Checking if there is a copy to the master in progress");
    }
    ld.lockHostRemoval();
    Collection<Copy> copiesInProgress = ld.getCopiesInProgress();
    if (copiesInProgress != null && !copiesInProgress.isEmpty()) {
        for (Copy copy : copiesInProgress) {
            if (copy != null) {
                if (copy.getTargetLoc() != null && copy.getTargetLoc().getHosts().contains(Comm.getAppHost())) {
                    if (DEBUG) {
                        LOGGER.debug("Copy in progress tranfering " + ld.getName() + "to master. Waiting for finishing");
                    }
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied set dataTarget " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copying file " + copy.getFinalTarget() + " from master to " + targetPath + " with replacing", ex);
                    }
                } else if (copy.getTargetData() != null && copy.getTargetData().getAllHosts().contains(Comm.getAppHost())) {
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied. Set data target to " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copy from " + copy.getFinalTarget() + " to " + targetPath + " with replacing", ex);
                    }
                } else {
                    if (DEBUG) {
                        LOGGER.debug("Current copies are not transfering " + ld.getName() + " to master. Ignoring at this moment");
                    }
                }
            }
        }
    }
    // Checking if file is already in master
    if (DEBUG) {
        LOGGER.debug("Checking if " + ld.getName() + " is at master (" + Comm.getAppHost().getName() + ").");
    }
    for (MultiURI u : ld.getURIs()) {
        if (DEBUG) {
            String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
            LOGGER.debug(ld.getName() + " is at " + u.toString() + "(" + hostname + ")");
        }
        if (u.getHost() == Comm.getAppHost()) {
            try {
                if (DEBUG) {
                    LOGGER.debug("Master local copy " + ld.getName() + " from " + u.getHost().getName() + " to " + targetPath);
                }
                Files.copy((new File(u.getPath())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                if (tgtData != null) {
                    tgtData.addLocation(target);
                }
                LOGGER.debug("File copied. Set data target to " + targetPath);
                reason.setDataTarget(targetPath);
                listener.notifyEnd(null);
                ld.releaseHostRemoval();
                return;
            } catch (IOException ex) {
                ErrorManager.warn("Error master local copy file from " + u.getPath() + " to " + targetPath + " with replacing", ex);
            }
        } else {
            if (DEBUG) {
                String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
                LOGGER.debug("Data " + ld.getName() + " copy in " + hostname + " not evaluated now");
            }
        }
    }
    // Ask the transfer from an specific source
    if (source != null) {
        for (Resource sourceRes : source.getHosts()) {
            COMPSsNode node = sourceRes.getNode();
            String sourcePath = source.getURIInHost(sourceRes).getPath();
            if (node != this) {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Sending data " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    node.sendData(ld, source, target, tgtData, reason, listener);
                } catch (Exception e) {
                    ErrorManager.warn("Not possible to sending data master to " + targetPath, e);
                    continue;
                }
                LOGGER.debug("Data " + ld.getName() + " sent.");
                ld.releaseHostRemoval();
                return;
            } else {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Local copy " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    Files.copy(new File(sourcePath).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                    LOGGER.debug("File copied. Set data target to " + targetPath);
                    reason.setDataTarget(targetPath);
                    listener.notifyEnd(null);
                    ld.releaseHostRemoval();
                    return;
                } catch (IOException ex) {
                    ErrorManager.warn("Error master local copy file from " + sourcePath + " to " + targetPath, ex);
                }
            }
        }
    } else {
        LOGGER.debug("Source data location is null. Trying other alternatives");
    }
    // Preferred source is null or copy has failed. Trying to retrieve data from any host
    for (Resource sourceRes : ld.getAllHosts()) {
        COMPSsNode node = sourceRes.getNode();
        if (node != this) {
            try {
                LOGGER.debug("Sending data " + ld.getName() + " from " + sourceRes.getName() + " to " + targetPath);
                node.sendData(ld, source, target, tgtData, reason, listener);
            } catch (Exception e) {
                LOGGER.error("Error: exception sending data", e);
                continue;
            }
            LOGGER.debug("Data " + ld.getName() + " sent.");
            ld.releaseHostRemoval();
            return;
        } else {
            if (DEBUG) {
                LOGGER.debug("Data " + ld.getName() + " copy in " + sourceRes.getName() + " not evaluated now. Should have been evaluated before");
            }
        }
    }
    // If we have not exited before, any copy method was successful. Raise warning
    ErrorManager.warn("Error file " + ld.getName() + " not transferred to " + targetPath);
    ld.releaseHostRemoval();
}

Example 3 with Resource

use of es.bsc.compss.types.resources.Resource in project compss by bsc-wdc.

the class COMPSsRuntimeImpl method createLocation.

private DataLocation createLocation(String fileName) throws IOException {
    // Check if fileName contains schema
    SimpleURI uri = new SimpleURI(fileName);
    if (uri.getSchema().isEmpty()) {
        // Add default File scheme and wrap local paths
        String canonicalPath = new File(fileName).getCanonicalPath();
        uri = new SimpleURI(Protocol.FILE_URI.getSchema() + canonicalPath);
    }
    // Check host
    Resource host = Comm.getAppHost();
    String hostName = uri.getHost();
    if (hostName != null && !hostName.isEmpty()) {
        host = Resource.getResource(hostName);
        if (host == null) {
            ErrorManager.error("Host " + hostName + " not found when creating data location.");
        }
    }
    // Create location
    return DataLocation.createLocation(host, uri);
}

Example 4 with Resource

use of es.bsc.compss.types.resources.Resource in project compss by bsc-wdc.

the class NIOAdaptor method submitTask.

protected static void submitTask(NIOJob job) throws Exception {
    LOGGER.debug("NIO submitting new job " + job.getJobId());
    Resource res = job.getResource();
    NIOWorkerNode worker = (NIOWorkerNode) res.getNode();
    LogicalData[] obsoletes = res.pollObsoletes();
    List<String> obsoleteRenamings = new LinkedList<>();
    for (LogicalData ld : obsoletes) {
        obsoleteRenamings.add(worker.getWorkingDir() + File.separator + ld.getName());
    }
    RUNNING_JOBS.put(job.getJobId(), job);
    worker.submitTask(job, obsoleteRenamings);
}

Example 5 with Resource

use of es.bsc.compss.types.resources.Resource in project compss by bsc-wdc.

the class PersistentLocation method getURIs.

@Override
public List<MultiURI> getURIs() {
    // Retrieve locations from Back-end
    List<String> locations = null;
    try {
        locations = StorageItf.getLocations(this.id);
    } catch (StorageException e) {
        ErrorManager.error("ERROR: Cannot retrieve locations of " + this.id + " from Storage Back-end");
    }
    if (locations == null) {
        ErrorManager.error("ERROR: Cannot retrieve locations of " + this.id + " from Storage Back-end");
    }
    // Retrieve URIs from hosts
    LinkedList<MultiURI> uris = new LinkedList<>();
    for (String hostName : locations) {
        Resource host = Resource.getResource(hostName);
        if (host != null) {
            uris.add(new MultiURI(Protocol.PERSISTENT_URI, host, this.id));
        } else {
            LOGGER.warn("Storage Back-End returned non-registered host " + hostName + ". Skipping URI in host");
        }
    }
    return uris;
}