Examples with StoredObject org.javaswift.joss.model.StoredObject used on opensource projects

Example 1 with StoredObject

use of org.javaswift.joss.model.StoredObject in project stocator by SparkTC.

the class SwiftAPIClient method isJobSuccessful.

/**
 * Checks if container/object contains
 * container/object/_SUCCESS
 * If so, this object was created by successful Hadoop job
 *
 * @param objectName
 * @return boolean if job is successful
 */
private boolean isJobSuccessful(String objectName) {
    LOG.trace("Checking if job completed successfull for {}", objectName);
    if (cachedSparkJobsStatus.containsKey(objectName)) {
        return cachedSparkJobsStatus.get(objectName).booleanValue();
    }
    String obj = objectName;
    Account account = mJossAccount.getAccount();
    LOG.trace("HEAD {}", obj + "/" + HADOOP_SUCCESS);
    StoredObject so = account.getContainer(container).getObject(obj + "/" + HADOOP_SUCCESS);
    Boolean isJobOK = Boolean.FALSE;
    if (so.exists()) {
        LOG.debug("{} exists", obj + "/" + HADOOP_SUCCESS);
        isJobOK = Boolean.TRUE;
    }
    cachedSparkJobsStatus.put(objectName, isJobOK);
    return isJobOK.booleanValue();
}

Example 2 with StoredObject

use of org.javaswift.joss.model.StoredObject in project stocator by SparkTC.

the class SwiftAPIClient method isSparkOrigin.

/**
 * Checks if container/object exists and verifies
 * that it contains Data-Origin=stocator metadata
 * If so, object was created by Spark.
 *
 * @param objectName
 * @return boolean if object was created by Spark
 */
private boolean isSparkOrigin(String objectName) {
    LOG.trace("Check if created by Stocator: {}", objectName);
    if (cachedSparkOriginated.containsKey(objectName)) {
        return cachedSparkOriginated.get(objectName).booleanValue();
    }
    String obj = objectName;
    Boolean sparkOriginated = Boolean.FALSE;
    StoredObject so = mJossAccount.getAccount().getContainer(container).getObject(obj);
    if (so != null && so.exists()) {
        Object sparkOrigin = so.getMetadata("Data-Origin");
        if (sparkOrigin != null) {
            String tmp = (String) sparkOrigin;
            if (tmp.equals("stocator")) {
                sparkOriginated = Boolean.TRUE;
                LOG.trace("Object {} was created by Stocator", objectName);
            }
        }
    }
    cachedSparkOriginated.put(objectName, sparkOriginated);
    return sparkOriginated.booleanValue();
}

Example 3 with StoredObject

use of org.javaswift.joss.model.StoredObject in project stocator by SparkTC.

the class SwiftAPIClient method delete.

@Override
public boolean delete(String hostName, Path path, boolean recursive) throws IOException {
    String obj = path.toString();
    if (path.toString().startsWith(hostName)) {
        obj = getObjName(hostName, path);
    }
    LOG.debug("Object name to delete {}. Path {}", obj, path.toString());
    try {
        StoredObject so = mJossAccount.getAccount().getContainer(container).getObject(obj);
        if (so.exists()) {
            so.delete();
            objectCache.remove(obj);
        }
    } catch (Exception e) {
        LOG.warn(e.getMessage());
        LOG.warn("Delete on {} resulted in FileNotFound exception", path);
        return false;
    }
    return true;
}

Example 4 with StoredObject

use of org.javaswift.joss.model.StoredObject in project dataverse by IQSS.

the class SwiftAccessIO method initializeSwiftFileObject.

private StoredObject initializeSwiftFileObject(boolean writeAccess, String auxItemTag) throws IOException {
    String swiftEndPoint = null;
    String swiftContainerName = null;
    String swiftFileName = null;
    StoredObject fileObject;
    List<String> auxFiles = null;
    String storageIdentifier = dvObject.getStorageIdentifier();
    if (dvObject instanceof DataFile) {
        Dataset owner = this.getDataFile().getOwner();
        if (storageIdentifier.startsWith("swift://")) {
            // This is a call on an already existing swift object.
            String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);
            if (swiftStorageTokens.length != 3) {
                // bad storage identifier
                throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
            }
            swiftEndPoint = swiftStorageTokens[0];
            swiftContainerName = swiftStorageTokens[1];
            swiftFileName = swiftStorageTokens[2];
            if (StringUtil.isEmpty(swiftEndPoint) || StringUtil.isEmpty(swiftContainerName) || StringUtil.isEmpty(swiftFileName)) {
                // identifier.
                throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
            }
            if (auxItemTag != null) {
                swiftFileName = swiftFileName.concat("." + auxItemTag);
            }
        } else if (this.isReadAccess) {
            // object!
            throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object.");
        } else if (this.isWriteAccess) {
            Properties p = getSwiftProperties();
            swiftEndPoint = p.getProperty("swift.default.endpoint");
            // swiftFolderPath = dataFile.getOwner().getDisplayName();
            String swiftFolderPathSeparator = "-";
            String authorityNoSlashes = owner.getAuthority().replace(owner.getDoiSeparator(), swiftFolderPathSeparator);
            swiftFolderPath = owner.getProtocol() + swiftFolderPathSeparator + authorityNoSlashes.replace(".", swiftFolderPathSeparator) + swiftFolderPathSeparator + owner.getIdentifier();
            swiftFileName = storageIdentifier;
            // setSwiftContainerName(swiftFolderPath);
            // swiftFileName = dataFile.getDisplayName();
            // Storage Identifier is now updated after the object is uploaded on Swift.
            dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath + ":" + swiftFileName);
        } else {
            throw new IOException("SwiftAccessIO: unknown access mode.");
        }
    } else if (dvObject instanceof Dataset) {
        Dataset dataset = this.getDataset();
        if (storageIdentifier.startsWith("swift://")) {
            // This is a call on an already existing swift object.
            // TODO: determine how storage identifer will give us info
            String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);
            // number of tokens should be two because there is not main file
            if (swiftStorageTokens.length != 2) {
                // bad storage identifier
                throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
            }
            swiftEndPoint = swiftStorageTokens[0];
            swiftContainerName = swiftStorageTokens[1];
            // We will not have a file name, just an aux tag
            if (auxItemTag != null) {
                swiftFileName = auxItemTag;
            } else {
                throw new IOException("Dataset related auxillary files require an auxItemTag");
            }
            if (StringUtil.isEmpty(swiftEndPoint) || StringUtil.isEmpty(swiftContainerName) || StringUtil.isEmpty(swiftFileName)) {
                // identifier.1
                throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
            }
        } else if (this.isReadAccess) {
            // object!
            throw new IOException("IO driver mismatch: SwiftAccessIO called on a non-swift stored object.");
        } else if (this.isWriteAccess) {
            Properties p = getSwiftProperties();
            swiftEndPoint = p.getProperty("swift.default.endpoint");
            String swiftFolderPathSeparator = "-";
            String authorityNoSlashes = dataset.getAuthority().replace(dataset.getDoiSeparator(), swiftFolderPathSeparator);
            swiftFolderPath = dataset.getProtocol() + swiftFolderPathSeparator + authorityNoSlashes.replace(".", swiftFolderPathSeparator) + swiftFolderPathSeparator + dataset.getIdentifier();
            swiftFileName = auxItemTag;
            dvObject.setStorageIdentifier("swift://" + swiftEndPoint + ":" + swiftFolderPath);
        } else {
            throw new IOException("SwiftAccessIO: unknown access mode.");
        }
    } else {
        // for future scope, if dataverse is decided to be stored in swift storage containersopen
        throw new FileNotFoundException("Error initializing swift object");
    }
    if (this.account == null) {
        account = authenticateWithSwift(swiftEndPoint);
    }
    /*
        The containers created is swiftEndPoint concatenated with the swiftContainerName
        property. Creating container with certain names throws 'Unable to create
        container' error on Openstack. 
        Any datafile with http://rdgw storage identifier i.e present on Object 
        store service endpoint already only needs to look-up for container using
        just swiftContainerName which is the concatenated name.
        In future, a container for the endpoint can be created and for every
        other swiftContainerName Object Store pseudo-folder can be created, which is
        not provide by the joss Java swift library as of yet.
         */
    if (storageIdentifier.startsWith("swift://")) {
        // An existing swift object; the container must already exist as well.
        this.swiftContainer = account.getContainer(swiftContainerName);
    } else {
        // This is a new object being created.
        // changed from swiftendpoint
        this.swiftContainer = account.getContainer(swiftFolderPath);
    }
    if (!this.swiftContainer.exists()) {
        if (writeAccess) {
            // creates a private data container
            swiftContainer.create();
        // try {
        // //creates a public data container
        // this.swiftContainer.makePublic();
        // }
        // catch (Exception e){
        // //e.printStackTrace();
        // logger.warning("Caught exception "+e.getClass()+" while creating a swift container (it's likely not fatal!)");
        // }
        } else {
            // read an existing object!
            throw new IOException("SwiftAccessIO: container " + swiftContainerName + " does not exist.");
        }
    }
    fileObject = this.swiftContainer.getObject(swiftFileName);
    // object for a primary file), we also set the file download url here:
    if (auxItemTag == null && dvObject instanceof DataFile) {
        setRemoteUrl(getSwiftFileURI(fileObject));
        if (!this.isWriteAccess && !this.getDataFile().isIngestInProgress()) {
            // otherwise this gets called a bunch on upload
            setTemporarySwiftUrl(generateTemporarySwiftUrl(swiftEndPoint, swiftContainerName, swiftFileName, TEMP_URL_EXPIRES));
            setTempUrlSignature(generateTempUrlSignature(swiftEndPoint, swiftContainerName, swiftFileName, TEMP_URL_EXPIRES));
            setTempUrlExpiry(generateTempUrlExpiry(TEMP_URL_EXPIRES, System.currentTimeMillis()));
        }
        setSwiftFileName(swiftFileName);
        logger.fine(getRemoteUrl() + " success; write mode: " + writeAccess);
    } else {
        logger.fine("sucessfully opened AUX object " + auxItemTag + " , write mode: " + writeAccess);
    }
    if (!writeAccess && !fileObject.exists()) {
        throw new FileNotFoundException("SwiftAccessIO: DvObject " + swiftFileName + " does not exist (Dataverse dvObject id: " + dvObject.getId());
    }
    auxFiles = null;
    return fileObject;
}

Example 5 with StoredObject

use of org.javaswift.joss.model.StoredObject in project dataverse by IQSS.

the class SwiftAccessIO method backupAsAux.

@Override
public void backupAsAux(String auxItemTag) throws IOException {
    if (swiftFileObject == null || swiftContainer == null) {
        open();
    }
    try {
        StoredObject swiftAuxObject = openSwiftAuxFile(true, auxItemTag);
        swiftFileObject.copyObject(swiftContainer, swiftAuxObject);
    // I'm assuming we don't need to delete the main object here - ?
    // swiftFileObject.delete();
    } catch (IOException ioex) {
        String failureMsg = ioex.getMessage();
        if (failureMsg == null) {
            failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject";
        }
        throw new IOException(failureMsg);
    }
}