Examples with S3ObjectInputStream com.amazonaws.services.s3.model.S3ObjectInputStream used on opensource projects

Example 1 with S3ObjectInputStream

use of com.amazonaws.services.s3.model.S3ObjectInputStream in project crate by crate.

the class FileReadingCollectorTest method createBatchIterator.

private BatchIterator createBatchIterator(Collection<String> fileUris, String compression, final S3ObjectInputStream s3InputStream) {
    Reference raw = createReference("_raw", DataTypes.STRING);
    InputFactory.Context<LineCollectorExpression<?>> ctx = inputFactory.ctxForRefs(FileLineReferenceResolver::getImplementation);
    List<Input<?>> inputs = Collections.singletonList(ctx.add(raw));
    return FileReadingIterator.newInstance(fileUris, inputs, ctx.expressions(), compression, ImmutableMap.of(LocalFsFileInputFactory.NAME, new LocalFsFileInputFactory(), S3FileInputFactory.NAME, () -> new S3FileInput(new S3ClientHelper() {

        @Override
        protected AmazonS3 initClient(String accessKey, String secretKey) throws IOException {
            AmazonS3 client = mock(AmazonS3Client.class);
            ObjectListing objectListing = mock(ObjectListing.class);
            S3ObjectSummary summary = mock(S3ObjectSummary.class);
            S3Object s3Object = mock(S3Object.class);
            when(client.listObjects(anyString(), anyString())).thenReturn(objectListing);
            when(objectListing.getObjectSummaries()).thenReturn(Arrays.asList(summary));
            when(summary.getKey()).thenReturn("foo");
            when(client.getObject("fakebucket", "foo")).thenReturn(s3Object);
            when(s3Object.getObjectContent()).thenReturn(s3InputStream);
            when(client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
            when(objectListing.isTruncated()).thenReturn(false);
            return client;
        }
    })), false, 1, 0);
}

Example 2 with S3ObjectInputStream

use of com.amazonaws.services.s3.model.S3ObjectInputStream in project elasticsearch by elastic.

the class MockAmazonS3 method getObject.

@Override
public S3Object getObject(GetObjectRequest getObjectRequest) throws AmazonClientException, AmazonServiceException {
    // in ESBlobStoreContainerTestCase.java, the prefix is empty,
    // so the key and blobName are equivalent to each other
    String blobName = getObjectRequest.getKey();
    if (!blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] does not exist.");
    }
    // the HTTP request attribute is irrelevant for reading
    S3ObjectInputStream stream = new S3ObjectInputStream(blobs.get(blobName), null, false);
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(stream);
    return s3Object;
}

Example 3 with S3ObjectInputStream

use of com.amazonaws.services.s3.model.S3ObjectInputStream in project aws-doc-sdk-examples by awsdocs.

the class GetObject method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket and object to\n" + "download from it.\n" + "\n" + "Ex: GetObject <bucketname> <filename>\n";
    if (args.length < 2) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    String key_name = args[1];
    System.out.format("Downloading %s from S3 bucket %s...\n", key_name, bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    try {
        S3Object o = s3.getObject(bucket_name, key_name);
        S3ObjectInputStream s3is = o.getObjectContent();
        FileOutputStream fos = new FileOutputStream(new File(key_name));
        byte[] read_buf = new byte[1024];
        int read_len = 0;
        while ((read_len = s3is.read(read_buf)) > 0) {
            fos.write(read_buf, 0, read_len);
        }
        s3is.close();
        fos.close();
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    } catch (FileNotFoundException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    } catch (IOException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}

Example 4 with S3ObjectInputStream

use of com.amazonaws.services.s3.model.S3ObjectInputStream in project crate by crate.

the class FileReadingCollectorTest method testCollectWithOneSocketTimeout.

@Test
public void testCollectWithOneSocketTimeout() throws Throwable {
    S3ObjectInputStream inputStream = mock(S3ObjectInputStream.class);
    when(inputStream.read(new byte[anyInt()], anyInt(), anyByte())).thenAnswer(// first line: foo
    new WriteBufferAnswer(new byte[] { 102, 111, 111, 10 })).thenThrow(// exception causes retry
    new SocketTimeoutException()).thenAnswer(// first line again, because of retry
    new WriteBufferAnswer(new byte[] { 102, 111, 111, 10 })).thenAnswer(// second line: bar
    new WriteBufferAnswer(new byte[] { 98, 97, 114, 10 })).thenReturn(-1);
    TestingBatchConsumer consumer = getObjects(Collections.singletonList("s3://fakebucket/foo"), null, inputStream);
    Bucket rows = consumer.getBucket();
    assertThat(rows.size(), is(2));
    assertThat(TestingHelpers.printedTable(rows), is("foo\nbar\n"));
}

Example 5 with S3ObjectInputStream

use of com.amazonaws.services.s3.model.S3ObjectInputStream in project crate by crate.

the class FileReadingCollectorTest method getObjects.

private TestingBatchConsumer getObjects(Collection<String> fileUris, String compression) throws Throwable {
    S3ObjectInputStream inputStream = mock(S3ObjectInputStream.class);
    when(inputStream.read(new byte[anyInt()], anyInt(), anyByte())).thenReturn(-1);
    return getObjects(fileUris, compression, inputStream);
}