Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 31 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project presto by prestodb.

the class TestPrestoS3FileSystem method testPrestoS3InputStreamEOS.

@Test
public void testPrestoS3InputStreamEOS() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        AtomicInteger readableBytes = new AtomicInteger(1);
        MockAmazonS3 s3 = new MockAmazonS3() {

            @Override
            public S3Object getObject(GetObjectRequest req) {
                return new S3Object() {

                    @Override
                    public S3ObjectInputStream getObjectContent() {
                        return new S3ObjectInputStream(new ByteArrayInputStream(new byte[readableBytes.get()]), null);
                    }
                };
            }
        };
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            assertEquals(inputStream.read(0, new byte[2], 0, 2), 1);
            readableBytes.set(0);
            assertEquals(inputStream.read(0, new byte[1], 0, 1), -1);
        }
    }
}

Example 32 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project flink by apache.

the class AWSServicesTestUtils method listBucketObjects.

public static List<S3Object> listBucketObjects(S3AsyncClient s3, String bucketName) throws ExecutionException, InterruptedException {
    ListObjectsRequest listObjects = ListObjectsRequest.builder().bucket(bucketName).build();
    CompletableFuture<ListObjectsResponse> res = s3.listObjects(listObjects);
    return res.get().contents();
}

Example 33 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project crate by crate.

the class S3FileReadingCollectorTest method createBatchIterator.

private BatchIterator<Row> createBatchIterator(Collection<String> fileUris, String compression, final S3ObjectInputStream s3InputStream, boolean collectSourceUriFailure) {
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(txnCtx, FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = new ArrayList<>(2);
    Reference raw = createReference(SourceLineExpression.COLUMN_NAME, DataTypes.STRING);
    inputs.add(ctx.add(raw));
    if (collectSourceUriFailure) {
        Reference sourceUriFailure = createReference(SourceUriFailureExpression.COLUMN_NAME, DataTypes.STRING);
        // noinspection unchecked
        sourceUriFailureInput = (Input<String>) ctx.add(sourceUriFailure);
        inputs.add(sourceUriFailureInput);
    }
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, Map.of(S3FileInputFactory.NAME, new FileInputFactory() {

        @Override
        public FileInput create(URI uri, Settings withClauseOptions) throws IOException {
            return new S3FileInput(new S3ClientHelper() {

                @Override
                protected AmazonS3 initClient(String accessKey, String secretKey, String endpoint, String protocol) throws IOException {
                    AmazonS3 client = mock(AmazonS3Client.class);
                    ObjectListing objectListing = mock(ObjectListing.class);
                    S3ObjectSummary summary = mock(S3ObjectSummary.class);
                    S3Object s3Object = mock(S3Object.class);
                    when(client.listObjects(anyString(), anyString())).thenReturn(objectListing);
                    when(objectListing.getObjectSummaries()).thenReturn(Collections.singletonList(summary));
                    when(summary.getKey()).thenReturn("foo");
                    when(client.getObject("fakebucket", "foo")).thenReturn(s3Object);
                    when(s3Object.getObjectContent()).thenReturn(s3InputStream);
                    when(client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
                    when(objectListing.isTruncated()).thenReturn(false);
                    return client;
                }
            }, uri, "https");
        }
    }), false, 1, 0, CopyFromParserProperties.DEFAULT, FileUriCollectPhase.InputFormat.JSON, Settings.EMPTY);
}

Example 34 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project crate by crate.

the class S3FileInput method getStream.

@Override
public InputStream getStream(URI uri) throws IOException {
    S3URI s3URI = S3URI.toS3URI(uri);
    if (client == null) {
        client = clientBuilder.client(s3URI, protocolSetting);
    }
    S3Object object = client.getObject(s3URI.bucket(), s3URI.key());
    if (object != null) {
        return object.getObjectContent();
    }
    throw new IOException("Failed to load S3 URI: " + uri.toString());
}

Example 35 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project druid by druid-io.

the class S3Entity method readFrom.

@Override
protected InputStream readFrom(long offset) throws IOException {
    final GetObjectRequest request = new GetObjectRequest(object.getBucket(), object.getPath());
    request.setRange(offset);
    try {
        final S3Object s3Object = s3Client.getObject(request);
        if (s3Object == null) {
            throw new ISE("Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", object.getBucket(), object.getPath(), offset);
        }
        return s3Object.getObjectContent();
    } catch (AmazonS3Exception e) {
        throw new IOException(e);
    }
}