Examples with GetObjectRequest software.amazon.awssdk.services.s3.model.GetObjectRequest used on opensource projects

Example 36 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project YCSB by brianfrankcooper.

the class S3Client method getS3ObjectAndMetadata.

private Map.Entry<S3Object, ObjectMetadata> getS3ObjectAndMetadata(String bucket, String key, SSECustomerKey ssecLocal) {
    GetObjectRequest getObjectRequest;
    GetObjectMetadataRequest getObjectMetadataRequest;
    if (ssecLocal != null) {
        getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecLocal);
        getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecLocal);
    } else {
        getObjectRequest = new GetObjectRequest(bucket, key);
        getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key);
    }
    return new AbstractMap.SimpleEntry<>(s3Client.getObject(getObjectRequest), s3Client.getObjectMetadata(getObjectMetadataRequest));
}

Example 37 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project archaius by Netflix.

the class S3ConfigurationSource method poll.

@Override
public PollResult poll(boolean initial, Object checkPoint) throws IOException, AmazonServiceException {
    GetObjectRequest s3request = new GetObjectRequest(bucketName, key);
    InputStream is = null;
    try {
        S3Object result = client.getObject(s3request);
        is = result.getObjectContent();
        Map<String, Object> resultMap = inputStreamToMap(is);
        return PollResult.createFull(resultMap);
    } finally {
        if (is != null)
            is.close();
    }
}

Example 38 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project camel by apache.

the class S3Consumer method poll.

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }
        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }
        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}

Example 39 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project hadoop by apache.

the class S3AInputStream method reopen.

/**
   * Opens up the stream at specified target position and for given length.
   *
   * @param reason reason for reopen
   * @param targetPos target position
   * @param length length requested
   * @throws IOException on any failure to open the object
   */
private synchronized void reopen(String reason, long targetPos, long length) throws IOException {
    if (wrappedStream != null) {
        closeStream("reopen(" + reason + ")", contentRangeFinish, false);
    }
    contentRangeFinish = calculateRequestLimit(inputPolicy, targetPos, length, contentLength, readahead);
    LOG.debug("reopen({}) for {} range[{}-{}], length={}," + " streamPosition={}, nextReadPosition={}", uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos);
    streamStatistics.streamOpened();
    try {
        GetObjectRequest request = new GetObjectRequest(bucket, key).withRange(targetPos, contentRangeFinish);
        if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) && StringUtils.isNotBlank(serverSideEncryptionKey)) {
            request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
        }
        wrappedStream = client.getObject(request).getObjectContent();
        contentRangeStart = targetPos;
        if (wrappedStream == null) {
            throw new IOException("Null IO stream from reopen of (" + reason + ") " + uri);
        }
    } catch (AmazonClientException e) {
        throw translateException("Reopen at position " + targetPos, uri, e);
    }
    this.pos = targetPos;
}

Example 40 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project nifi by apache.

the class TestFetchS3Object method testGetObjectVersion.

@Test
public void testGetObjectVersion() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    attrs.put("s3.version", "request-version");
    runner.enqueue(new byte[0], attrs);
    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    Mockito.when(metadata.getVersionId()).thenReturn("response-version");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
    runner.run(1);
    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertEquals("request-version", request.getVersionId());
    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals("s3.version", "response-version");
    ff.assertContentEquals("Some Content");
}