Examples with DataLocation es.bsc.compss.types.data.location.DataLocation used on opensource projects

Example 1 with DataLocation

use of es.bsc.compss.types.data.location.DataLocation in project compss by bsc-wdc.

the class Comm method registerValue.

/**
 * Registers a new value @value for the data with id @dataId dataId must exist
 *
 * @param dataId
 * @param value
 * @return
 */
public static synchronized LogicalData registerValue(String dataId, Object value) {
    LOGGER.debug("Register value " + value + " for data " + dataId);
    String targetPath = Protocol.OBJECT_URI.getSchema() + dataId;
    DataLocation location = null;
    try {
        SimpleURI uri = new SimpleURI(targetPath);
        location = DataLocation.createLocation(appHost, uri);
    } catch (IOException e) {
        ErrorManager.error(DataLocation.ERROR_INVALID_LOCATION + " " + targetPath, e);
    }
    LogicalData logicalData = data.get(dataId);
    logicalData.addLocation(location);
    logicalData.setValue(value);
    // Register PSCO Location if needed it's PSCO and it's persisted
    if (value instanceof StubItf) {
        String id = ((StubItf) value).getID();
        if (id != null) {
            Comm.registerPSCO(dataId, id);
        }
    }
    return logicalData;
}

Example 2 with DataLocation

use of es.bsc.compss.types.data.location.DataLocation in project compss by bsc-wdc.

the class Comm method registerPSCO.

/**
 * Registers a new PSCO id @id for the data with id @dataId dataId must exist
 *
 * @param dataId
 * @param id
 * @return
 */
public static synchronized LogicalData registerPSCO(String dataId, String id) {
    String targetPath = Protocol.PERSISTENT_URI.getSchema() + id;
    DataLocation location = null;
    try {
        SimpleURI uri = new SimpleURI(targetPath);
        location = DataLocation.createLocation(appHost, uri);
    } catch (IOException ioe) {
        ErrorManager.error(DataLocation.ERROR_INVALID_LOCATION + " " + targetPath, ioe);
    }
    LogicalData logicalData = data.get(dataId);
    logicalData.addLocation(location);
    return logicalData;
}

Example 3 with DataLocation

use of es.bsc.compss.types.data.location.DataLocation in project compss by bsc-wdc.

the class Comm method removeData.

/**
 * Removes the data with id @renaming
 *
 * @param renaming
 */
public static synchronized void removeData(String renaming) {
    LOGGER.debug("Remove data " + renaming);
    LogicalData ld = data.remove(renaming);
    ld.isObsolete();
    for (DataLocation dl : ld.getLocations()) {
        MultiURI uri = dl.getURIInHost(appHost);
        if (uri != null) {
            File f = new File(uri.getPath());
            if (f.exists()) {
                LOGGER.info("Deleting file " + f.getAbsolutePath());
                if (!f.delete()) {
                    LOGGER.error("Cannot delete file " + f.getAbsolutePath());
                }
            }
        }
    }
}

Example 4 with DataLocation

use of es.bsc.compss.types.data.location.DataLocation in project compss by bsc-wdc.

the class COMPSsMaster method obtainData.

@Override
public void obtainData(LogicalData ld, DataLocation source, DataLocation target, LogicalData tgtData, Transferable reason, EventListener listener) {
    LOGGER.info("Obtain Data " + ld.getName());
    /*
         * PSCO transfers are always available, if any SourceLocation is PSCO, don't transfer
         */
    for (DataLocation loc : ld.getLocations()) {
        if (loc.getProtocol().equals(Protocol.PERSISTENT_URI)) {
            LOGGER.debug("Object in Persistent Storage. Set dataTarget to " + loc.getPath());
            reason.setDataTarget(loc.getPath());
            listener.notifyEnd(null);
            return;
        }
    }
    /*
         * Otherwise the data is a file or an object that can be already in the master memory, in the master disk or
         * being transfered
         */
    String targetPath = target.getURIInHost(Comm.getAppHost()).getPath();
    // Check if data is in memory (no need to check if it is PSCO since previous case avoids it)
    if (ld.isInMemory()) {
        // Serialize value to file
        try {
            Serializer.serialize(ld.getValue(), targetPath);
        } catch (IOException ex) {
            ErrorManager.warn("Error copying file from memory to " + targetPath, ex);
        }
        if (tgtData != null) {
            tgtData.addLocation(target);
        }
        LOGGER.debug("Object in memory. Set dataTarget to " + targetPath);
        reason.setDataTarget(targetPath);
        listener.notifyEnd(null);
        return;
    }
    // Check if there are current copies in progress
    if (DEBUG) {
        LOGGER.debug("Data " + ld.getName() + " not in memory. Checking if there is a copy to the master in progress");
    }
    ld.lockHostRemoval();
    Collection<Copy> copiesInProgress = ld.getCopiesInProgress();
    if (copiesInProgress != null && !copiesInProgress.isEmpty()) {
        for (Copy copy : copiesInProgress) {
            if (copy != null) {
                if (copy.getTargetLoc() != null && copy.getTargetLoc().getHosts().contains(Comm.getAppHost())) {
                    if (DEBUG) {
                        LOGGER.debug("Copy in progress tranfering " + ld.getName() + "to master. Waiting for finishing");
                    }
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied set dataTarget " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copying file " + copy.getFinalTarget() + " from master to " + targetPath + " with replacing", ex);
                    }
                } else if (copy.getTargetData() != null && copy.getTargetData().getAllHosts().contains(Comm.getAppHost())) {
                    waitForCopyTofinish(copy);
                    try {
                        if (DEBUG) {
                            LOGGER.debug("Master local copy " + ld.getName() + " from " + copy.getFinalTarget() + " to " + targetPath);
                        }
                        Files.copy((new File(copy.getFinalTarget())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                        if (tgtData != null) {
                            tgtData.addLocation(target);
                        }
                        LOGGER.debug("File copied. Set data target to " + targetPath);
                        reason.setDataTarget(targetPath);
                        listener.notifyEnd(null);
                        ld.releaseHostRemoval();
                        return;
                    } catch (IOException ex) {
                        ErrorManager.warn("Error master local copy from " + copy.getFinalTarget() + " to " + targetPath + " with replacing", ex);
                    }
                } else {
                    if (DEBUG) {
                        LOGGER.debug("Current copies are not transfering " + ld.getName() + " to master. Ignoring at this moment");
                    }
                }
            }
        }
    }
    // Checking if file is already in master
    if (DEBUG) {
        LOGGER.debug("Checking if " + ld.getName() + " is at master (" + Comm.getAppHost().getName() + ").");
    }
    for (MultiURI u : ld.getURIs()) {
        if (DEBUG) {
            String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
            LOGGER.debug(ld.getName() + " is at " + u.toString() + "(" + hostname + ")");
        }
        if (u.getHost() == Comm.getAppHost()) {
            try {
                if (DEBUG) {
                    LOGGER.debug("Master local copy " + ld.getName() + " from " + u.getHost().getName() + " to " + targetPath);
                }
                Files.copy((new File(u.getPath())).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                if (tgtData != null) {
                    tgtData.addLocation(target);
                }
                LOGGER.debug("File copied. Set data target to " + targetPath);
                reason.setDataTarget(targetPath);
                listener.notifyEnd(null);
                ld.releaseHostRemoval();
                return;
            } catch (IOException ex) {
                ErrorManager.warn("Error master local copy file from " + u.getPath() + " to " + targetPath + " with replacing", ex);
            }
        } else {
            if (DEBUG) {
                String hostname = (u.getHost() != null) ? u.getHost().getName() : "null";
                LOGGER.debug("Data " + ld.getName() + " copy in " + hostname + " not evaluated now");
            }
        }
    }
    // Ask the transfer from an specific source
    if (source != null) {
        for (Resource sourceRes : source.getHosts()) {
            COMPSsNode node = sourceRes.getNode();
            String sourcePath = source.getURIInHost(sourceRes).getPath();
            if (node != this) {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Sending data " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    node.sendData(ld, source, target, tgtData, reason, listener);
                } catch (Exception e) {
                    ErrorManager.warn("Not possible to sending data master to " + targetPath, e);
                    continue;
                }
                LOGGER.debug("Data " + ld.getName() + " sent.");
                ld.releaseHostRemoval();
                return;
            } else {
                try {
                    if (DEBUG) {
                        LOGGER.debug("Local copy " + ld.getName() + " from " + sourcePath + " to " + targetPath);
                    }
                    Files.copy(new File(sourcePath).toPath(), new File(targetPath).toPath(), StandardCopyOption.REPLACE_EXISTING);
                    LOGGER.debug("File copied. Set data target to " + targetPath);
                    reason.setDataTarget(targetPath);
                    listener.notifyEnd(null);
                    ld.releaseHostRemoval();
                    return;
                } catch (IOException ex) {
                    ErrorManager.warn("Error master local copy file from " + sourcePath + " to " + targetPath, ex);
                }
            }
        }
    } else {
        LOGGER.debug("Source data location is null. Trying other alternatives");
    }
    // Preferred source is null or copy has failed. Trying to retrieve data from any host
    for (Resource sourceRes : ld.getAllHosts()) {
        COMPSsNode node = sourceRes.getNode();
        if (node != this) {
            try {
                LOGGER.debug("Sending data " + ld.getName() + " from " + sourceRes.getName() + " to " + targetPath);
                node.sendData(ld, source, target, tgtData, reason, listener);
            } catch (Exception e) {
                LOGGER.error("Error: exception sending data", e);
                continue;
            }
            LOGGER.debug("Data " + ld.getName() + " sent.");
            ld.releaseHostRemoval();
            return;
        } else {
            if (DEBUG) {
                LOGGER.debug("Data " + ld.getName() + " copy in " + sourceRes.getName() + " not evaluated now. Should have been evaluated before");
            }
        }
    }
    // If we have not exited before, any copy method was successful. Raise warning
    ErrorManager.warn("Error file " + ld.getName() + " not transferred to " + targetPath);
    ld.releaseHostRemoval();
}

Example 5 with DataLocation

use of es.bsc.compss.types.data.location.DataLocation in project compss by bsc-wdc.

the class COMPSsRuntimeImpl method openFile.

/*
     * ***********************************************************************************************************
     * COMMON IN BOTH APIs
     * ********************************************************************************************************
     */
/**
 * Returns the renaming of the file version opened
 */
@Override
public String openFile(String fileName, Direction mode) {
    if (Tracer.isActivated()) {
        Tracer.emitEvent(Tracer.Event.OPEN_FILE.getId(), Tracer.Event.OPEN_FILE.getType());
    }
    LOGGER.info("Opening " + fileName + " in mode " + mode);
    // Parse arguments to internal structures
    DataLocation loc;
    try {
        loc = createLocation(fileName);
    } catch (IOException ioe) {
        ErrorManager.fatal(ERROR_FILE_NAME, ioe);
        return null;
    }
    AccessMode am = null;
    switch(mode) {
        case IN:
            am = AccessMode.R;
            break;
        case OUT:
            am = AccessMode.W;
            break;
        case INOUT:
            am = AccessMode.RW;
            break;
    }
    // Request AP that the application wants to access a FILE or a EXTERNAL_PSCO
    String finalPath;
    switch(loc.getType()) {
        case PRIVATE:
        case SHARED:
            finalPath = mainAccessToFile(fileName, loc, am, null);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("File target Location: " + finalPath);
            }
            break;
        case PERSISTENT:
            finalPath = mainAccessToExternalObject(fileName, loc);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("External Object target Location: " + finalPath);
            }
            break;
        default:
            finalPath = null;
            ErrorManager.error("ERROR: Unrecognised protocol requesting openFile " + fileName);
    }
    if (Tracer.isActivated()) {
        Tracer.emitEvent(Tracer.EVENT_END, Tracer.getRuntimeEventsType());
    }
    return finalPath;
}