Search in sources :

Example 31 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project aws-doc-sdk-examples by awsdocs.

the class ImportSegment method uploadToS3.

private static void uploadToS3(File endpointsFile, String s3BucketName) {
    // Initializes Amazon S3 client.
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    // Checks whether the specified bucket exists. If not, attempts to create one.
    if (!s3.doesBucketExistV2(s3BucketName)) {
        try {
            s3.createBucket(s3BucketName);
            System.out.format("Created S3 bucket %s.\n", s3BucketName);
        } catch (AmazonS3Exception e) {
            System.err.println(e.getErrorMessage());
            System.exit(1);
        }
    }
    // Uploads the endpoints file to the bucket.
    String endpointsFileName = endpointsFile.getName();
    System.out.format("Uploading %s to S3 bucket %s . . .\n", endpointsFileName, s3BucketName);
    try {
        s3.putObject(s3BucketName, "imports/" + endpointsFileName, endpointsFile);
        System.out.println("Finished uploading to S3.");
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
}
Also used : AmazonS3(com.amazonaws.services.s3.AmazonS3) AmazonServiceException(com.amazonaws.AmazonServiceException) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception)

Example 32 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project aws-doc-sdk-examples by awsdocs.

the class S3EncryptV2 method main.

public static void main(String[] args) throws NoSuchAlgorithmException {
    AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // of this example
    try {
        s3Client.createBucket(bucketName);
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    putEncryptedData1();
    putEncryptedData2();
    putEncryptedData3_Kms();
    try {
        ObjectListing objectListing = s3Client.listObjects(bucketName);
        while (true) {
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3Client.deleteObject(bucketName, summary.getKey());
            }
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    // delete test bucket
    try {
        s3Client.deleteBucket(bucketName);
    } catch (AmazonS3Exception e) {
        System.err.println(e.getErrorMessage());
    }
    s3Client.shutdown();
    System.out.println("Done");
}
Also used : AmazonS3(com.amazonaws.services.s3.AmazonS3) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception)

Example 33 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project stocator by SparkTC.

the class COSAPIClient method getFileStatusKeyBased.

private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
    LOG.trace("internal method - get file status by key {}, path {}", key, path);
    FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
    if (cachedFS != null) {
        return cachedFS;
    }
    ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
    String sparkOrigin = meta.getUserMetaDataOf("data-origin");
    boolean stocatorCreated = false;
    if (sparkOrigin != null) {
        String tmp = (String) sparkOrigin;
        if (tmp.equals("stocator")) {
            stocatorCreated = true;
        }
    }
    mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
    FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
    LOG.trace("getFileStatusKeyBased: key {} fs.path {}", key, fs.getPath());
    memoryCache.putFileStatus(path.toString(), fs);
    return fs;
}
Also used : FileStatus(org.apache.hadoop.fs.FileStatus) LocatedFileStatus(org.apache.hadoop.fs.LocatedFileStatus) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 34 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project elasticsearch by elastic.

the class TestAmazonS3 method uploadPart.

@Override
public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException {
    if (shouldFail(request.getBucketName(), request.getKey(), writeFailureRate)) {
        long length = request.getPartSize();
        long partToRead = (long) (length * randomDouble());
        byte[] buffer = new byte[1024];
        for (long cur = 0; cur < partToRead; cur += buffer.length) {
            try (InputStream input = request.getInputStream()) {
                input.read(buffer, 0, (int) (partToRead - cur > buffer.length ? buffer.length : partToRead - cur));
            } catch (IOException ex) {
                throw new ElasticsearchException("cannot read input stream", ex);
            }
        }
        logger.info("--> random write failure on uploadPart method: throwing an exception for [bucket={}, key={}]", request.getBucketName(), request.getKey());
        AmazonS3Exception ex = new AmazonS3Exception("Random S3 write exception");
        ex.setStatusCode(400);
        ex.setErrorCode("RequestTimeout");
        throw ex;
    } else {
        return super.uploadPart(request);
    }
}
Also used : InputStream(java.io.InputStream) IOException(java.io.IOException) ElasticsearchException(org.elasticsearch.ElasticsearchException) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception)

Example 35 with AmazonS3Exception

use of com.amazonaws.services.s3.model.AmazonS3Exception in project elasticsearch by elastic.

the class MockAmazonS3 method listObjects.

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);
    String blobName;
    String prefix = listObjectsRequest.getPrefix();
    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();
    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();
        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);
            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " + "failed for blob [" + blob.getKey() + "]");
            }
            mockObjectSummaries.add(objectSummary);
        }
    }
    list.setObjectSummaries(mockObjectSummaries);
    return list;
}
Also used : DigestInputStream(java.security.DigestInputStream) S3ObjectInputStream(com.amazonaws.services.s3.model.S3ObjectInputStream) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) IOException(java.io.IOException) AmazonS3Exception(com.amazonaws.services.s3.model.AmazonS3Exception) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

AmazonS3Exception (com.amazonaws.services.s3.model.AmazonS3Exception)62 IOException (java.io.IOException)23 Test (org.junit.Test)13 FileNotFoundException (java.io.FileNotFoundException)10 ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)9 S3ObjectSummary (com.amazonaws.services.s3.model.S3ObjectSummary)9 AmazonServiceException (com.amazonaws.AmazonServiceException)8 ObjectListing (com.amazonaws.services.s3.model.ObjectListing)7 GetObjectMetadataRequest (com.amazonaws.services.s3.model.GetObjectMetadataRequest)6 S3Object (com.amazonaws.services.s3.model.S3Object)6 S3TestUtils.buildMockedS3FileSystem (org.apache.beam.sdk.io.aws.s3.S3TestUtils.buildMockedS3FileSystem)6 AmazonClientException (com.amazonaws.AmazonClientException)5 InterruptedIOException (java.io.InterruptedIOException)5 ArrayList (java.util.ArrayList)5 Path (org.apache.hadoop.fs.Path)5 ClientConfiguration (com.amazonaws.ClientConfiguration)4 AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)4 GetObjectRequest (com.amazonaws.services.s3.model.GetObjectRequest)4 PutObjectRequest (com.amazonaws.services.s3.model.PutObjectRequest)4 URI (java.net.URI)4