Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 36 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project druid by druid-io.

the class StaticS3FirehoseFactory method openObjectStream.

@Override
protected InputStream openObjectStream(URI object) throws IOException {
    try {
        // Get data of the given object and open an input stream
        final String bucket = object.getAuthority();
        final String key = S3Utils.extractS3Key(object);
        final S3Object s3Object = s3Client.getObject(bucket, key);
        if (s3Object == null) {
            throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", bucket, key);
        }
        return s3Object.getObjectContent();
    } catch (AmazonS3Exception e) {
        throw new IOException(e);
    }
}

Example 37 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project druid by druid-io.

the class S3InputSourceTest method expectSdkClientException.

// Setup mocks for invoquing the resettable condition for the S3Entity:
private static void expectSdkClientException(URI uri) throws IOException {
    final String s3Bucket = uri.getAuthority();
    final String key = S3Utils.extractS3Key(uri);
    S3ObjectInputStream someInputStream = EasyMock.createMock(S3ObjectInputStream.class);
    EasyMock.expect(someInputStream.read(EasyMock.anyObject(), EasyMock.anyInt(), EasyMock.anyInt())).andThrow(new SdkClientException("Data read has a different length than the expected")).anyTimes();
    someInputStream.close();
    expectLastCall().andVoid().anyTimes();
    S3Object someObject = EasyMock.createMock(S3Object.class);
    EasyMock.expect(someObject.getBucketName()).andReturn(s3Bucket).anyTimes();
    EasyMock.expect(someObject.getKey()).andReturn(key).anyTimes();
    EasyMock.expect(someObject.getObjectContent()).andReturn(someInputStream).anyTimes();
    EasyMock.expect(S3_CLIENT.getObject(EasyMock.anyObject(GetObjectRequest.class))).andReturn(someObject).anyTimes();
    EasyMock.replay(someObject);
    EasyMock.replay(someInputStream);
}

Example 38 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project druid by druid-io.

the class S3InputSourceTest method expectGetObject.

private static void expectGetObject(URI uri) {
    final String s3Bucket = uri.getAuthority();
    final String key = S3Utils.extractS3Key(uri);
    S3Object someObject = new S3Object();
    someObject.setBucketName(s3Bucket);
    someObject.setKey(key);
    someObject.setObjectContent(new ByteArrayInputStream(CONTENT));
    EasyMock.expect(S3_CLIENT.getObject(EasyMock.anyObject(GetObjectRequest.class))).andReturn(someObject).once();
}

Example 39 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project druid by druid-io.

the class S3DataSegmentPullerTest method testS3ObjectStream.

@Test
public void testS3ObjectStream() throws IOException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final ServerSideEncryptingAmazonS3 s3Client = EasyMock.createStrictMock(ServerSideEncryptingAmazonS3.class);
    final byte[] value = bucket.getBytes(StandardCharsets.UTF_8);
    final File tmpFile = temporaryFolder.newFile("testObjectFile");
    try (OutputStream outputStream = new FileOutputStream(tmpFile)) {
        outputStream.write(value);
    }
    final S3Object object0 = new S3Object();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/test-object");
    object0.getObjectMetadata().setLastModified(new Date(0));
    object0.setObjectContent(new FileInputStream(tmpFile));
    EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))).andReturn(object0).once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
    EasyMock.replay(s3Client);
    InputStream stream = puller.buildFileObject(URI.create(StringUtils.format("s3://%s/%s", bucket, object0.getKey()))).openInputStream();
    EasyMock.verify(s3Client);
    Assert.assertEquals(bucket, IOUtils.toString(stream, StandardCharsets.UTF_8));
}

Example 40 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project beam by apache.

the class S3FileSystem method expandGlob.

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));
    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix, wildcardRegexp);
    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;
    do {
        ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(glob.getBucket()).prefix(keyPrefix).continuationToken(continuationToken).build();
        ListObjectsV2Response response;
        try {
            response = s3Client.get().listObjectsV2(request);
        } catch (SdkServiceException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = response.nextContinuationToken();
        List<S3Object> contents = response.contents();
        contents.stream().filter(s3Object -> wildcardRegexp.matcher(s3Object.key()).matches()).forEach(s3Object -> {
            S3ResourceId expandedPath = S3ResourceId.fromComponents(glob.getScheme(), glob.getBucket(), s3Object.key()).withSize(s3Object.size()).withLastModified(Date.from(s3Object.lastModified()));
            LOG.debug("Expanded S3 object path {}", expandedPath);
            expandedPaths.add(expandedPath);
        });
    } while (continuationToken != null);
    return ExpandedGlob.create(glob, expandedPaths.build());
}