Examples with ObjectMetadata com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata used on opensource projects

Example 36 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project stocator by SparkTC.

the class COSAPIClient method getFileStatusKeyBased.

private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
    LOG.trace("internal method - get file status by key {}, path {}", key, path);
    FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
    if (cachedFS != null) {
        return cachedFS;
    }
    ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
    String sparkOrigin = meta.getUserMetaDataOf("data-origin");
    boolean stocatorCreated = false;
    if (sparkOrigin != null) {
        String tmp = (String) sparkOrigin;
        if (tmp.equals("stocator")) {
            stocatorCreated = true;
        }
    }
    mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
    FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
    memoryCache.putFileStatus(path.toString(), fs);
    return fs;
}

Example 37 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project stocator by SparkTC.

the class COSAPIClient method isSparkOrigin.

/**
 * Checks if container/object exists and verifies that it contains
 * Data-Origin=stocator metadata If so, object was created by Spark.
 *
 * @param objectKey the key of the object
 * @param path the object path
 * @return boolean if object was created by Spark
 */
private boolean isSparkOrigin(String objectKey, String path) {
    LOG.debug("check spark origin for {}", objectKey);
    if (!objectKey.endsWith("/")) {
        LOG.debug("Key {} has no slash. Return false", objectKey);
        return false;
    } else {
        objectKey = objectKey.substring(0, objectKey.length() - 1);
    }
    if (mCachedSparkOriginated.containsKey(objectKey)) {
        boolean res = mCachedSparkOriginated.get(objectKey).booleanValue();
        LOG.debug("found cached for spark origin for {}. Status {}", objectKey, res);
        return res;
    }
    String key = getRealKey(objectKey);
    Boolean sparkOriginated = Boolean.FALSE;
    ObjectMetadata objMetadata = getObjectMetadata(key);
    if (objMetadata != null) {
        Object sparkOrigin = objMetadata.getUserMetaDataOf("data-origin");
        if (sparkOrigin != null) {
            String tmp = (String) sparkOrigin;
            if (tmp.equals("stocator")) {
                sparkOriginated = Boolean.TRUE;
            }
        }
    }
    mCachedSparkOriginated.put(key, sparkOriginated);
    LOG.debug("spark origin for {} is {} non cached", objectKey, sparkOriginated.booleanValue());
    return sparkOriginated.booleanValue();
}

Example 38 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project stocator by SparkTC.

the class COSAPIClient method isJobSuccessful.

/**
 * Checks if container/object contains container/object/_SUCCESS If so, this
 * object was created by successful Hadoop job
 *
 * @param objectKey
 * @return boolean if job is successful
 */
private boolean isJobSuccessful(String objectKey) {
    LOG.trace("isJobSuccessful: for {}", objectKey);
    if (mCachedSparkJobsStatus.containsKey(objectKey)) {
        LOG.trace("isJobSuccessful: {} found cached", objectKey);
        return mCachedSparkJobsStatus.get(objectKey).booleanValue();
    }
    String key = getRealKey(objectKey);
    Path p = new Path(key, HADOOP_SUCCESS);
    ObjectMetadata statusMetadata = getObjectMetadata(p.toString());
    Boolean isJobOK = Boolean.FALSE;
    if (statusMetadata != null) {
        isJobOK = Boolean.TRUE;
    }
    LOG.debug("isJobSuccessful: not cached {}. Status is {}", objectKey, isJobOK);
    mCachedSparkJobsStatus.put(objectKey, isJobOK);
    return isJobOK.booleanValue();
}

Example 39 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project stocator by SparkTC.

the class COSOutputStream method close.

@Override
public void close() throws IOException {
    if (closed.getAndSet(true)) {
        return;
    }
    mBackupOutputStream.close();
    LOG.debug("OutputStream for key '{}' closed. Now beginning upload", mKey);
    try {
        final ObjectMetadata om = new ObjectMetadata();
        om.setContentLength(mBackupFile.length());
        om.setContentType(mContentType);
        om.setUserMetadata(mMetadata);
        PutObjectRequest putObjectRequest = new PutObjectRequest(mBucketName, mKey, mBackupFile);
        putObjectRequest.setMetadata(om);
        Upload upload = transfers.upload(putObjectRequest);
        upload.waitForUploadResult();
    } catch (InterruptedException e) {
        throw (InterruptedIOException) new InterruptedIOException(e.toString()).initCause(e);
    } catch (AmazonClientException e) {
        throw new IOException(String.format("saving output %s %s", mKey, e));
    } finally {
        if (!mBackupFile.delete()) {
            LOG.warn("Could not delete temporary cos file: {}", mBackupOutputStream);
        }
        super.close();
    }
    LOG.debug("OutputStream for key '{}' upload complete", mKey);
}

Example 40 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project apex-malhar by apache.

the class S3Reconciler method processCommittedData.

/**
 * Uploads the file on Amazon S3 using putObject API from S3 client
 */
@Override
protected void processCommittedData(FSRecordCompactionOperator.OutputMetaData outputMetaData) {
    try {
        Path path = new Path(outputMetaData.getPath());
        if (fs.exists(path) == false) {
            logger.debug("Ignoring non-existent path assuming replay : {}", path);
            return;
        }
        FSDataInputStream fsinput = fs.open(path);
        ObjectMetadata omd = new ObjectMetadata();
        omd.setContentLength(outputMetaData.getSize());
        String keyName = directoryName + Path.SEPARATOR + outputMetaData.getFileName();
        PutObjectRequest request = new PutObjectRequest(bucketName, keyName, fsinput, omd);
        if (outputMetaData.getSize() < Integer.MAX_VALUE) {
            request.getRequestClientOptions().setReadLimit((int) outputMetaData.getSize());
        } else {
            throw new RuntimeException("PutRequestSize greater than Integer.MAX_VALUE");
        }
        if (fs.exists(path)) {
            PutObjectResult result = s3client.putObject(request);
            logger.debug("File {} Uploaded at {}", keyName, result.getETag());
        }
    } catch (FileNotFoundException e) {
        logger.debug("Ignoring non-existent path assuming replay : {}", outputMetaData.getPath());
    } catch (IOException e) {
        logger.error("Unable to create Stream: {}", e.getMessage());
    }
}