Examples with GetObjectRequest com.amazonaws.services.s3.model.GetObjectRequest used on opensource projects

Example 11 with GetObjectRequest

use of com.amazonaws.services.s3.model.GetObjectRequest in project camel by apache.

the class S3Consumer method poll.

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }
        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }
        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}

Example 12 with GetObjectRequest

use of com.amazonaws.services.s3.model.GetObjectRequest in project hadoop by apache.

the class S3AInputStream method reopen.

/**
   * Opens up the stream at specified target position and for given length.
   *
   * @param reason reason for reopen
   * @param targetPos target position
   * @param length length requested
   * @throws IOException on any failure to open the object
   */
private synchronized void reopen(String reason, long targetPos, long length) throws IOException {
    if (wrappedStream != null) {
        closeStream("reopen(" + reason + ")", contentRangeFinish, false);
    }
    contentRangeFinish = calculateRequestLimit(inputPolicy, targetPos, length, contentLength, readahead);
    LOG.debug("reopen({}) for {} range[{}-{}], length={}," + " streamPosition={}, nextReadPosition={}", uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos);
    streamStatistics.streamOpened();
    try {
        GetObjectRequest request = new GetObjectRequest(bucket, key).withRange(targetPos, contentRangeFinish);
        if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) && StringUtils.isNotBlank(serverSideEncryptionKey)) {
            request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
        }
        wrappedStream = client.getObject(request).getObjectContent();
        contentRangeStart = targetPos;
        if (wrappedStream == null) {
            throw new IOException("Null IO stream from reopen of (" + reason + ") " + uri);
        }
    } catch (AmazonClientException e) {
        throw translateException("Reopen at position " + targetPos, uri, e);
    }
    this.pos = targetPos;
}

Example 13 with GetObjectRequest

use of com.amazonaws.services.s3.model.GetObjectRequest in project XRTB by benmfaul.

the class AwsCommander method load.

/**
	 * Load the file or s3 object.
	 * @param parts String[]. An array of tokens.
	 * @return String. The message returned from the load command.
	 * @throws Exception on I/O errirs.
	 */
String load(String[] parts) throws Exception {
    String otype = null;
    String symbolName = null;
    String name;
    // file or S3
    String type = parts[1];
    //}
    if (type.equalsIgnoreCase("S3")) {
        // bloom, cache, cuckoo.
        otype = parts[2];
        name = parts[4];
        // name of the object
        symbolName = parts[3];
        if (!symbolName.startsWith("$"))
            symbolName = "$" + symbolName;
    } else
        // file name
        name = parts[2];
    if (type.equals("file")) {
        return Configuration.getInstance().readData(parts[2]);
    }
    S3Object object = Configuration.s3.getObject(new GetObjectRequest(Configuration.s3_bucket, name));
    long size = Configuration.s3.getObjectMetadata(Configuration.s3_bucket, name).getContentLength();
    return Configuration.getInstance().readData(otype, symbolName, object, size);
}

Example 14 with GetObjectRequest

use of com.amazonaws.services.s3.model.GetObjectRequest in project gradle by gradle.

the class S3Client method doGetS3Object.

private S3Object doGetS3Object(URI uri, boolean isLightWeight) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(uri);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);
    GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, s3BucketKey);
    if (isLightWeight) {
        // Skip content download
        getObjectRequest.setRange(0, 0);
    }
    try {
        return amazonS3Client.getObject(getObjectRequest);
    } catch (AmazonServiceException e) {
        String errorCode = e.getErrorCode();
        if (null != errorCode && errorCode.equalsIgnoreCase("NoSuchKey")) {
            return null;
        }
        throw ResourceExceptions.getFailed(uri, e);
    }
}