Examples with GetObjectRequest com.amazonaws.s3.model.GetObjectRequest used on opensource projects

Example 1 with GetObjectRequest

use of com.amazonaws.s3.model.GetObjectRequest in project airavata-mft by apache.

the class S3IncomingConnector method downloadChunk.

@Override
public void downloadChunk(int chunkId, long startByte, long endByte, String downloadFile) throws Exception {
    GetObjectRequest rangeObjectRequest = new GetObjectRequest(resource.getS3Storage().getBucketName(), resource.getFile().getResourcePath());
    rangeObjectRequest.setRange(startByte, endByte - 1);
    ObjectMetadata objectMetadata = s3Client.getObject(rangeObjectRequest, new File(downloadFile));
    logger.info("Downloaded S3 chunk to path {} for resource id {}", downloadFile, resource.getResourceId());
}

Example 2 with GetObjectRequest

use of com.amazonaws.s3.model.GetObjectRequest in project amazon-athena-serverless-ehms-connector by aws-samples.

the class S3Helper method getResponseFromS3.

public String getResponseFromS3(String s3Path) throws IOException {
    if (s3Path == null) {
        throw new IOException("S3 path is null");
    }
    AmazonS3URI s3URI = new AmazonS3URI(s3Path);
    String bucket = s3URI.getBucket();
    String key = s3URI.getKey();
    BufferedReader br = new BufferedReader(new InputStreamReader(s3Client.getObject(new GetObjectRequest(bucket, key)).getObjectContent()));
    String line;
    StringBuilder sb = new StringBuilder();
    // read the files until it reaches the end of the file
    while ((line = br.readLine()) != null) {
        sb.append(line);
    }
    return sb.toString();
}

Example 3 with GetObjectRequest

use of com.amazonaws.s3.model.GetObjectRequest in project zuliasearch by zuliaio.

the class S3DocumentStorage method getAssociatedDocumentStream.

@Override
public InputStream getAssociatedDocumentStream(String uniqueId, String filename) throws Exception {
    FindIterable<Document> found = client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + FILE_UNIQUE_ID_KEY, String.join("-", uniqueId, filename)));
    Document doc = found.first();
    if (null != doc) {
        Document s3Info = doc.get("s3", Document.class);
        GetObjectRequest gor = GetObjectRequest.builder().bucket(s3Info.getString("bucket")).key(s3Info.getString("key")).build();
        ResponseInputStream<GetObjectResponse> results = s3.getObject(gor);
        return new BufferedInputStream(results);
    }
    return null;
}

Example 4 with GetObjectRequest

use of com.amazonaws.s3.model.GetObjectRequest in project zuliasearch by zuliaio.

the class S3DocumentStorage method parseTOC.

private AssociatedDocument parseTOC(Document doc) throws IOException {
    AssociatedDocument.Builder aBuilder = AssociatedDocument.newBuilder();
    aBuilder.setFilename(doc.getString(FILENAME));
    Document meta = doc.get("metadata", Document.class);
    aBuilder.setDocumentUniqueId(meta.getString(DOCUMENT_UNIQUE_ID_KEY));
    aBuilder.setTimestamp(meta.getLong(TIMESTAMP));
    aBuilder.setIndexName(indexName);
    meta.remove(TIMESTAMP);
    meta.remove(DOCUMENT_UNIQUE_ID_KEY);
    meta.remove(FILE_UNIQUE_ID_KEY);
    aBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(meta));
    Document s3Info = doc.get("s3", Document.class);
    GetObjectRequest gor = GetObjectRequest.builder().bucket(s3Info.getString("bucket")).key(s3Info.getString("key")).build();
    ResponseInputStream<GetObjectResponse> results = s3.getObject(gor);
    aBuilder.setDocument(ByteString.readFrom(results));
    return aBuilder.build();
}

Example 5 with GetObjectRequest

use of com.amazonaws.s3.model.GetObjectRequest in project proxima-platform by O2-Czech-Republic.

the class S3BlobPath method read.

@Override
public ReadableByteChannel read() {
    final long contentLength = getBlob().getSize();
    return new SeekableByteChannel() {

        private boolean open = true;

        private S3Object object;

        private ReadableByteChannel contentChannel;

        private long position = 0;

        @Override
        public int read(ByteBuffer dst) throws IOException {
            ensureOpen();
            if (position >= contentLength) {
                // End of file.
                return -1;
            }
            if (object == null) {
                final S3FileSystem fs = (S3FileSystem) getFileSystem();
                final GetObjectRequest request = new GetObjectRequest(fs.getBucket(), getBlobName());
                @Nullable final SSECustomerKey sseCustomerKey = fs.getSseCustomerKey();
                if (sseCustomerKey != null) {
                    request.setSSECustomerKey(sseCustomerKey);
                }
                if (position > 0) {
                    request.setRange(position, contentLength);
                }
                object = fs.client().getObject(request);
                contentChannel = Channels.newChannel(new BufferedInputStream(object.getObjectContent(), 1024 * 1024));
            }
            int totalBytesRead = 0;
            int bytesRead = 0;
            do {
                totalBytesRead += bytesRead;
                try {
                    bytesRead = contentChannel.read(dst);
                } catch (AmazonClientException e) {
                    throw new IOException(e);
                }
            } while (bytesRead > 0);
            position += totalBytesRead;
            return totalBytesRead;
        }

        @Override
        public long position() throws IOException {
            ensureOpen();
            return position;
        }

        @Override
        public SeekableByteChannel position(long newPosition) throws IOException {
            ensureOpen();
            Preconditions.checkArgument(newPosition >= 0, "New position is too low.");
            Preconditions.checkArgument(newPosition < contentLength, "New position is too high.");
            if (newPosition == position) {
                return this;
            }
            // next call to read()
            if (object != null) {
                object.close();
                object = null;
            }
            position = newPosition;
            return this;
        }

        @Override
        public long size() throws IOException {
            ensureOpen();
            return contentLength;
        }

        @Override
        public boolean isOpen() {
            return open;
        }

        @Override
        public void close() throws IOException {
            if (object != null) {
                object.close();
                object = null;
            }
            open = false;
        }

        @Override
        public int write(ByteBuffer src) {
            throw new NonWritableChannelException();
        }

        @Override
        public SeekableByteChannel truncate(long size) {
            throw new NonWritableChannelException();
        }

        private void ensureOpen() throws IOException {
            if (!open) {
                throw new ClosedChannelException();
            }
        }
    };
}