Search in sources :

Example 1 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project elasticsearch by elastic.

the class DefaultS3OutputStream method doUpload.

protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length, boolean serverSideEncryption) throws AmazonS3Exception {
    ObjectMetadata md = new ObjectMetadata();
    if (serverSideEncryption) {
        md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    }
    md.setContentLength(length);
    InputStream inputStream = is;
    // We try to compute a MD5 while reading it
    MessageDigest messageDigest;
    try {
        messageDigest = MessageDigest.getInstance("MD5");
        inputStream = new DigestInputStream(is, messageDigest);
    } catch (NoSuchAlgorithmException impossible) {
        // Every implementation of the Java platform is required to support MD5 (see MessageDigest)
        throw new RuntimeException(impossible);
    }
    PutObjectRequest putRequest = new PutObjectRequest(bucketName, blobName, inputStream, md).withStorageClass(blobStore.getStorageClass()).withCannedAcl(blobStore.getCannedACL());
    PutObjectResult putObjectResult = blobStore.client().putObject(putRequest);
    String localMd5 = Base64.encodeAsString(messageDigest.digest());
    String remoteMd5 = putObjectResult.getContentMd5();
    if (!localMd5.equals(remoteMd5)) {
        logger.debug("MD5 local [{}], remote [{}] are not equal...", localMd5, remoteMd5);
        throw new AmazonS3Exception("MD5 local [" + localMd5 + "], remote [" + remoteMd5 + "] are not equal...");
    }
}
Also used : DigestInputStream(java.security.DigestInputStream) PutObjectResult(com.amazonaws.services.s3.model.PutObjectResult) ByteArrayInputStream(java.io.ByteArrayInputStream) DigestInputStream(java.security.DigestInputStream) InputStream(java.io.InputStream) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) MessageDigest(java.security.MessageDigest) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest)

Example 2 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project elasticsearch by elastic.

the class S3BlobContainer method move.

@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
    try {
        CopyObjectRequest request = new CopyObjectRequest(blobStore.bucket(), buildKey(sourceBlobName), blobStore.bucket(), buildKey(targetBlobName));
        if (blobStore.serverSideEncryption()) {
            ObjectMetadata objectMetadata = new ObjectMetadata();
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
            request.setNewObjectMetadata(objectMetadata);
        }
        SocketAccess.doPrivilegedVoid(() -> {
            blobStore.client().copyObject(request);
            blobStore.client().deleteObject(blobStore.bucket(), buildKey(sourceBlobName));
        });
    } catch (AmazonS3Exception e) {
        throw new IOException(e);
    }
}
Also used : CopyObjectRequest(com.amazonaws.services.s3.model.CopyObjectRequest) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) IOException(java.io.IOException) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 3 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project hadoop by apache.

the class S3AFileSystem method getObjectMetadata.

/**
   * Request object metadata; increments counters in the process.
   * @param key key
   * @return the metadata
   */
protected ObjectMetadata getObjectMetadata(String key) {
    incrementStatistic(OBJECT_METADATA_REQUESTS);
    GetObjectMetadataRequest request = new GetObjectMetadataRequest(bucket, key);
    //SSE-C requires to be filled in if enabled for object metadata
    if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) && StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))) {
        request.setSSECustomerKey(generateSSECustomerKey());
    }
    ObjectMetadata meta = s3.getObjectMetadata(request);
    incrementReadOperations();
    return meta;
}
Also used : GetObjectMetadataRequest(com.amazonaws.services.s3.model.GetObjectMetadataRequest) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 4 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project hadoop by apache.

the class S3AFileSystem method copyFile.

/**
   * Copy a single object in the bucket via a COPY operation.
   * @param srcKey source object path
   * @param dstKey destination object path
   * @param size object size
   * @throws AmazonClientException on failures inside the AWS SDK
   * @throws InterruptedIOException the operation was interrupted
   * @throws IOException Other IO problems
   */
private void copyFile(String srcKey, String dstKey, long size) throws IOException, InterruptedIOException, AmazonClientException {
    LOG.debug("copyFile {} -> {} ", srcKey, dstKey);
    try {
        ObjectMetadata srcom = getObjectMetadata(srcKey);
        ObjectMetadata dstom = cloneObjectMetadata(srcom);
        setOptionalObjectMetadata(dstom);
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
        setOptionalCopyObjectRequestParameters(copyObjectRequest);
        copyObjectRequest.setCannedAccessControlList(cannedACL);
        copyObjectRequest.setNewObjectMetadata(dstom);
        ProgressListener progressListener = new ProgressListener() {

            public void progressChanged(ProgressEvent progressEvent) {
                switch(progressEvent.getEventType()) {
                    case TRANSFER_PART_COMPLETED_EVENT:
                        incrementWriteOperations();
                        break;
                    default:
                        break;
                }
            }
        };
        Copy copy = transfers.copy(copyObjectRequest);
        copy.addProgressListener(progressListener);
        try {
            copy.waitForCopyResult();
            incrementWriteOperations();
            instrumentation.filesCopied(1, size);
        } catch (InterruptedException e) {
            throw new InterruptedIOException("Interrupted copying " + srcKey + " to " + dstKey + ", cancelling");
        }
    } catch (AmazonClientException e) {
        throw translateException("copyFile(" + srcKey + ", " + dstKey + ")", srcKey, e);
    }
}
Also used : InterruptedIOException(java.io.InterruptedIOException) CopyObjectRequest(com.amazonaws.services.s3.model.CopyObjectRequest) ProgressListener(com.amazonaws.event.ProgressListener) Copy(com.amazonaws.services.s3.transfer.Copy) AmazonClientException(com.amazonaws.AmazonClientException) ProgressEvent(com.amazonaws.event.ProgressEvent) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 5 with ObjectMetadata

use of com.talend.shaded.com.amazonaws.services.s3.model.ObjectMetadata in project hadoop by apache.

the class ITestS3AEncryptionSSEKMSDefaultKey method assertEncrypted.

@Override
protected void assertEncrypted(Path path) throws IOException {
    ObjectMetadata md = getFileSystem().getObjectMetadata(path);
    assertEquals("aws:kms", md.getSSEAlgorithm());
    assertThat(md.getSSEAwsKmsKeyId(), containsString("arn:aws:kms:"));
}
Also used : ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Aggregations

ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)194 PutObjectRequest (com.amazonaws.services.s3.model.PutObjectRequest)80 Test (org.junit.Test)60 ByteArrayInputStream (java.io.ByteArrayInputStream)54 IOException (java.io.IOException)39 AmazonClientException (com.amazonaws.AmazonClientException)31 File (java.io.File)28 AmazonServiceException (com.amazonaws.AmazonServiceException)26 Date (java.util.Date)23 InputStream (java.io.InputStream)21 S3FileTransferRequestParamsDto (org.finra.herd.model.dto.S3FileTransferRequestParamsDto)21 DataStoreException (org.apache.jackrabbit.core.data.DataStoreException)18 PutObjectResult (com.amazonaws.services.s3.model.PutObjectResult)16 Upload (com.amazonaws.services.s3.transfer.Upload)15 CopyObjectRequest (com.amazonaws.services.s3.model.CopyObjectRequest)13 S3TestUtils.buildMockedS3FileSystem (org.apache.beam.sdk.io.aws.s3.S3TestUtils.buildMockedS3FileSystem)12 GetObjectMetadataRequest (com.amazonaws.services.s3.model.GetObjectMetadataRequest)11 S3Object (com.amazonaws.services.s3.model.S3Object)11 AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)10 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)9