Examples with GetObjectRequest software.amazon.awssdk.services.s3.model.GetObjectRequest used on opensource projects

Example 1 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project zeppelin by apache.

the class S3NotebookRepo method getNote.

private Note getNote(String key) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
    S3Object s3object;
    try {
        s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }
    Note note;
    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        note = gson.fromJson(json, Note.class);
    }
    for (Paragraph p : note.getParagraphs()) {
        if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
            p.setStatus(Status.ABORT);
        }
    }
    return note;
}

Example 2 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project nifi by apache.

the class FetchS3Object method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final long startNanos = System.nanoTime();
    final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue();
    final AmazonS3 client = getClient();
    final GetObjectRequest request;
    if (versionId == null) {
        request = new GetObjectRequest(bucket, key);
    } else {
        request = new GetObjectRequest(bucket, key, versionId);
    }
    final Map<String, String> attributes = new HashMap<>();
    try (final S3Object s3Object = client.getObject(request)) {
        flowFile = session.importFrom(s3Object.getObjectContent(), flowFile);
        attributes.put("s3.bucket", s3Object.getBucketName());
        final ObjectMetadata metadata = s3Object.getObjectMetadata();
        if (metadata.getContentDisposition() != null) {
            final String fullyQualified = metadata.getContentDisposition();
            final int lastSlash = fullyQualified.lastIndexOf("/");
            if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) {
                attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash));
                attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified);
                attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1));
            } else {
                attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
            }
        }
        if (metadata.getContentMD5() != null) {
            attributes.put("hash.value", metadata.getContentMD5());
            attributes.put("hash.algorithm", "MD5");
        }
        if (metadata.getContentType() != null) {
            attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType());
        }
        if (metadata.getETag() != null) {
            attributes.put("s3.etag", metadata.getETag());
        }
        if (metadata.getExpirationTime() != null) {
            attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime()));
        }
        if (metadata.getExpirationTimeRuleId() != null) {
            attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId());
        }
        if (metadata.getUserMetadata() != null) {
            attributes.putAll(metadata.getUserMetadata());
        }
        if (metadata.getSSEAlgorithm() != null) {
            attributes.put("s3.sseAlgorithm", metadata.getSSEAlgorithm());
        }
        if (metadata.getVersionId() != null) {
            attributes.put("s3.version", metadata.getVersionId());
        }
    } catch (final IOException | AmazonClientException ioe) {
        getLogger().error("Failed to retrieve S3 Object for {}; routing to failure", new Object[] { flowFile, ioe });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    if (!attributes.isEmpty()) {
        flowFile = session.putAllAttributes(flowFile, attributes);
    }
    session.transfer(flowFile, REL_SUCCESS);
    final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success", new Object[] { flowFile, transferMillis });
    session.getProvenanceReporter().fetch(flowFile, "http://" + bucket + ".amazonaws.com/" + key, transferMillis);
}

Example 3 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project herd by FINRAOS.

the class MockS3OperationsImpl method download.

@Override
public Download download(String bucket, String key, File file, TransferManager transferManager) {
    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucket);
    MockS3Object mockS3Object = mockS3Bucket.getObjects().get(key);
    try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
        fileOutputStream.write(mockS3Object.getData());
    } catch (IOException e) {
        throw new RuntimeException("Error writing to file " + file, e);
    }
    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(mockS3Object.getData().length);
    progress.updateProgress(mockS3Object.getData().length);
    DownloadImpl download = new DownloadImpl(null, progress, null, null, null, new GetObjectRequest(bucket, key), file, mockS3Object.getObjectMetadata(), false);
    download.setState(TransferState.Completed);
    return download;
}

Example 4 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project spring-integration-aws by spring-projects.

the class S3MessageHandler method download.

private Transfer download(Message<?> requestMessage) {
    Object payload = requestMessage.getPayload();
    Assert.state(payload instanceof File, "For the 'DOWNLOAD' operation the 'payload' must be of " + "'java.io.File' type, but gotten: [" + payload.getClass() + ']');
    File targetFile = (File) payload;
    String bucket = obtainBucket(requestMessage);
    String key = null;
    if (this.keyExpression != null) {
        key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class);
    } else {
        key = targetFile.getName();
    }
    Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage);
    if (targetFile.isDirectory()) {
        return this.transferManager.downloadDirectory(bucket, key, targetFile);
    } else {
        if (this.s3ProgressListener != null) {
            return this.transferManager.download(new GetObjectRequest(bucket, key), targetFile, this.s3ProgressListener);
        } else {
            return this.transferManager.download(bucket, key, targetFile);
        }
    }
}

Example 5 with GetObjectRequest

use of software.amazon.awssdk.services.s3.model.GetObjectRequest in project onebusaway-application-modules by camsys.

the class S3FileServiceImpl method get.

@Override
public /**
 * Retrieve the specified key from S3 and store in the given directory.
 */
String get(String key, String tmpDir) {
    _log.debug("get(" + key + ", " + tmpDir + ")");
    NYCFileUtils fs = new NYCFileUtils();
    String filename = fs.parseFileName(key);
    _log.debug("filename=" + filename);
    GetObjectRequest request = new GetObjectRequest(this._bucketName, key);
    S3Object file = _s3.getObject(request);
    String pathAndFileName = tmpDir + File.separator + filename;
    fs.copy(file.getObjectContent(), pathAndFileName);
    return pathAndFileName;
}