Search in sources :

Example 26 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project esop by instaclustr.

the class BaseS3Backuper method uploadText.

@Override
public void uploadText(final String text, final RemoteObjectReference objectReference) throws Exception {
    final S3RemoteObjectReference s3RemoteObjectReference = (S3RemoteObjectReference) objectReference;
    final PutObjectRequest putObjectRequest = new PutObjectRequest(request.storageLocation.bucket, s3RemoteObjectReference.canonicalPath, new ByteArrayInputStream(text.getBytes()), new ObjectMetadata() {

        {
            setContentLength(text.getBytes().length);
        }
    });
    transferManager.upload(putObjectRequest, new UploadProgressListener(s3RemoteObjectReference)).waitForCompletion();
}
Also used : ByteArrayInputStream(java.io.ByteArrayInputStream) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest)

Example 27 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project esop by instaclustr.

the class BaseS3Backuper method uploadFile.

@Override
public void uploadFile(final long size, final InputStream localFileStream, final RemoteObjectReference objectReference) throws Exception {
    final S3RemoteObjectReference s3RemoteObjectReference = (S3RemoteObjectReference) objectReference;
    final PutObjectRequest putObjectRequest = new PutObjectRequest(request.storageLocation.bucket, s3RemoteObjectReference.canonicalPath, localFileStream, new ObjectMetadata() {

        {
            setContentLength(size);
        }
    });
    transferManager.upload(putObjectRequest, new UploadProgressListener(s3RemoteObjectReference)).waitForCompletion();
}
Also used : ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest)

Example 28 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project solarnetwork-central by SolarNetwork.

the class S3DatumExportDestinationService method export.

@Override
public void export(Configuration config, Iterable<DatumExportResource> resources, Map<String, ?> runtimeProperties, ProgressListener<DatumExportService> progressListener) throws IOException {
    if (config == null) {
        throw new IOException("No configuration provided.");
    }
    if (resources == null) {
        throw new IOException("No export resource provided.");
    }
    DestinationConfiguration destConfig = config.getDestinationConfiguration();
    if (destConfig == null) {
        throw new IOException("No destination configuration provided.");
    }
    S3DestinationProperties props = new S3DestinationProperties();
    ClassUtils.setBeanProperties(props, destConfig.getServiceProperties(), true);
    if (!props.isValid()) {
        throw new IOException("Service configuration is not valid.");
    }
    List<DatumExportResource> resourceList = StreamSupport.stream(resources.spliterator(), false).collect(Collectors.toList());
    final int resourceCount = resourceList.size();
    final com.amazonaws.event.ProgressListener s3ProgressListener = new com.amazonaws.event.ProgressListener() {

        private double overallProgress = 0;

        @Override
        public void progressChanged(ProgressEvent progressEvent) {
            ProgressEventType type = progressEvent.getEventType();
            if (type == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT) {
                double resourceProgress = (double) progressEvent.getBytesTransferred() / (double) progressEvent.getBytes();
                overallProgress += (resourceProgress / resourceCount);
                progressListener.progressChanged(S3DatumExportDestinationService.this, overallProgress);
            }
        }
    };
    AmazonS3 client = getClient(props);
    AmazonS3URI uri = props.getUri();
    for (ListIterator<DatumExportResource> itr = resourceList.listIterator(); itr.hasNext(); ) {
        // if we have >1 resource to upload, then we'll insert an index suffix in the key name, like -1, -2, etc.
        int idx = (resourceCount > 1 ? itr.nextIndex() + 1 : 0);
        DatumExportResource resource = itr.next();
        String key = getDestinationPath(props, runtimeProperties, idx);
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (resource.getContentType() != null) {
            objectMetadata.setContentType(resource.getContentType());
        }
        objectMetadata.setContentLength(resource.contentLength());
        objectMetadata.setLastModified(new Date(resource.lastModified()));
        try (InputStream in = resource.getInputStream()) {
            PutObjectRequest req = new PutObjectRequest(uri.getBucket(), key, in, objectMetadata);
            if (props.getStorageClass() != null) {
                req.setStorageClass(props.getStorageClass());
            }
            if (progressListener != null) {
                req.withGeneralProgressListener(s3ProgressListener);
            }
            client.putObject(req);
        } catch (AmazonServiceException e) {
            log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId());
            throw new RemoteServiceException("Error putting S3 object at " + key, e);
        } catch (AmazonClientException e) {
            log.debug("Error communicating with AWS: {}", e.getMessage());
            throw new IOException("Error communicating with AWS", e);
        }
    }
}
Also used : AmazonS3(com.amazonaws.services.s3.AmazonS3) ProgressEventType(com.amazonaws.event.ProgressEventType) RemoteServiceException(net.solarnetwork.central.RemoteServiceException) AmazonClientException(com.amazonaws.AmazonClientException) ProgressEvent(com.amazonaws.event.ProgressEvent) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) DestinationConfiguration(net.solarnetwork.central.datum.export.domain.DestinationConfiguration) InputStream(java.io.InputStream) IOException(java.io.IOException) AmazonS3URI(com.amazonaws.services.s3.AmazonS3URI) Date(java.util.Date) ProgressListener(net.solarnetwork.service.ProgressListener) AmazonServiceException(com.amazonaws.AmazonServiceException) DatumExportResource(net.solarnetwork.central.datum.export.domain.DatumExportResource) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata)

Example 29 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project trellis-extensions by trellis-ldp.

the class S3BinaryService method bufferUpload.

private void bufferUpload(final BinaryMetadata metadata, final InputStream stream, final Path path) throws IOException {
    // Buffer the file locally so that the PUT request can be parallelized for large objects
    try {
        try (final OutputStream output = Files.newOutputStream(path, WRITE)) {
            IOUtils.copy(stream, output);
        }
        final ObjectMetadata md = new ObjectMetadata();
        metadata.getMimeType().ifPresent(md::setContentType);
        final PutObjectRequest req = new PutObjectRequest(bucketName, getKey(metadata.getIdentifier()), path.toFile()).withMetadata(md);
        client.putObject(req);
    } finally {
        Files.delete(path);
    }
}
Also used : OutputStream(java.io.OutputStream) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest)

Example 30 with PutObjectRequest

use of com.amazonaws.s3.model.PutObjectRequest in project trellis-extensions by trellis-ldp.

the class S3MementoService method put.

@Override
public CompletionStage<Void> put(final Resource resource) {
    return runAsync(() -> {
        try {
            final File file = createTempFile("trellis-memento-", ".nq");
            file.deleteOnExit();
            final Map<String, String> metadata = new HashMap<>();
            metadata.put(S3Resource.INTERACTION_MODEL, resource.getInteractionModel().getIRIString());
            metadata.put(S3Resource.MODIFIED, resource.getModified().toString());
            resource.getContainer().map(IRI::getIRIString).ifPresent(c -> metadata.put(S3Resource.CONTAINER, c));
            resource.getBinaryMetadata().ifPresent(b -> {
                metadata.put(S3Resource.BINARY_LOCATION, b.getIdentifier().getIRIString());
                b.getMimeType().ifPresent(m -> metadata.put(S3Resource.BINARY_TYPE, m));
            });
            resource.getMembershipResource().map(IRI::getIRIString).ifPresent(m -> metadata.put(S3Resource.MEMBERSHIP_RESOURCE, m));
            resource.getMemberRelation().map(IRI::getIRIString).ifPresent(m -> metadata.put(S3Resource.MEMBER_RELATION, m));
            resource.getMemberOfRelation().map(IRI::getIRIString).ifPresent(m -> metadata.put(S3Resource.MEMBER_OF_RELATION, m));
            resource.getInsertedContentRelation().map(IRI::getIRIString).ifPresent(m -> metadata.put(S3Resource.INSERTED_CONTENT_RELATION, m));
            try (final Dataset dataset = rdf.createDataset();
                final OutputStream output = Files.newOutputStream(file.toPath());
                final Stream<? extends Quad> quads = resource.stream()) {
                quads.forEachOrdered(dataset::add);
                metadata.put(S3Resource.METADATA_GRAPHS, dataset.getGraphNames().filter(IRI.class::isInstance).map(IRI.class::cast).filter(graph -> !IGNORE.contains(graph)).map(IRI::getIRIString).collect(joining(",")));
                RDFDataMgr.write(output, toJena(dataset), NQUADS);
            }
            final ObjectMetadata md = new ObjectMetadata();
            md.setContentType("application/n-quads");
            md.setUserMetadata(metadata);
            final PutObjectRequest req = new PutObjectRequest(bucketName, getKey(resource.getIdentifier(), resource.getModified().truncatedTo(SECONDS)), file);
            client.putObject(req.withMetadata(md));
            Files.delete(file.toPath());
        } catch (final Exception ex) {
            throw new TrellisRuntimeException("Error deleting locally buffered file", ex);
        }
    });
}
Also used : TrellisRuntimeException(org.trellisldp.api.TrellisRuntimeException) SortedSet(java.util.SortedSet) File.createTempFile(java.io.File.createTempFile) Resource(org.trellisldp.api.Resource) ListObjectsV2Request(com.amazonaws.services.s3.model.ListObjectsV2Request) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) JenaCommonsRDF.toJena(org.apache.jena.commonsrdf.JenaCommonsRDF.toJena) Map(java.util.Map) Collections.unmodifiableSortedSet(java.util.Collections.unmodifiableSortedSet) AmazonS3ClientBuilder.defaultClient(com.amazonaws.services.s3.AmazonS3ClientBuilder.defaultClient) NQUADS(org.apache.jena.riot.Lang.NQUADS) Set(java.util.Set) ConfigProvider.getConfig(org.eclipse.microprofile.config.ConfigProvider.getConfig) Config(org.eclipse.microprofile.config.Config) Instant(java.time.Instant) Collectors.joining(java.util.stream.Collectors.joining) CompletionStage(java.util.concurrent.CompletionStage) Stream(java.util.stream.Stream) MementoService(org.trellisldp.api.MementoService) TRELLIS_DATA_PREFIX(org.trellisldp.api.TrellisUtils.TRELLIS_DATA_PREFIX) Alternative(javax.enterprise.inject.Alternative) HashMap(java.util.HashMap) GetObjectRequest(com.amazonaws.services.s3.model.GetObjectRequest) TreeSet(java.util.TreeSet) RDF(org.apache.commons.rdf.api.RDF) ListObjectsV2Result(com.amazonaws.services.s3.model.ListObjectsV2Result) HashSet(java.util.HashSet) Inject(javax.inject.Inject) Objects.requireNonNull(java.util.Objects.requireNonNull) CompletableFuture.supplyAsync(java.util.concurrent.CompletableFuture.supplyAsync) AmazonS3(com.amazonaws.services.s3.AmazonS3) LDP(org.trellisldp.vocabulary.LDP) S3ObjectSummary(com.amazonaws.services.s3.model.S3ObjectSummary) Dataset(org.apache.commons.rdf.api.Dataset) Trellis(org.trellisldp.vocabulary.Trellis) OutputStream(java.io.OutputStream) SECONDS(java.time.temporal.ChronoUnit.SECONDS) RDFFactory(org.trellisldp.api.RDFFactory) CompletableFuture.runAsync(java.util.concurrent.CompletableFuture.runAsync) Logger(org.slf4j.Logger) Files(java.nio.file.Files) Stream.of(java.util.stream.Stream.of) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest) File(java.io.File) IRI(org.apache.commons.rdf.api.IRI) Collections.unmodifiableSet(java.util.Collections.unmodifiableSet) Quad(org.apache.commons.rdf.api.Quad) RDFDataMgr(org.apache.jena.riot.RDFDataMgr) LoggerFactory.getLogger(org.slf4j.LoggerFactory.getLogger) MISSING_RESOURCE(org.trellisldp.api.Resource.SpecialResources.MISSING_RESOURCE) IRI(org.apache.commons.rdf.api.IRI) HashMap(java.util.HashMap) Dataset(org.apache.commons.rdf.api.Dataset) OutputStream(java.io.OutputStream) TrellisRuntimeException(org.trellisldp.api.TrellisRuntimeException) TrellisRuntimeException(org.trellisldp.api.TrellisRuntimeException) File.createTempFile(java.io.File.createTempFile) File(java.io.File) ObjectMetadata(com.amazonaws.services.s3.model.ObjectMetadata) PutObjectRequest(com.amazonaws.services.s3.model.PutObjectRequest)

Aggregations

PutObjectRequest (com.amazonaws.services.s3.model.PutObjectRequest)301 ObjectMetadata (com.amazonaws.services.s3.model.ObjectMetadata)193 ByteArrayInputStream (java.io.ByteArrayInputStream)113 Test (org.junit.Test)113 PutObjectRequest (software.amazon.awssdk.services.s3.model.PutObjectRequest)78 File (java.io.File)68 IOException (java.io.IOException)65 InputStream (java.io.InputStream)55 S3FileTransferRequestParamsDto (org.finra.herd.model.dto.S3FileTransferRequestParamsDto)42 AmazonClientException (com.amazonaws.AmazonClientException)40 PutObjectResult (com.amazonaws.services.s3.model.PutObjectResult)39 Upload (com.amazonaws.services.s3.transfer.Upload)37 AmazonServiceException (com.amazonaws.AmazonServiceException)35 Test (org.junit.jupiter.api.Test)30 AmazonS3 (com.amazonaws.services.s3.AmazonS3)28 AmazonS3Exception (com.amazonaws.services.s3.model.AmazonS3Exception)20 Date (java.util.Date)20 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)20 StorageUnitEntity (org.finra.herd.model.jpa.StorageUnitEntity)20 RequestBody (software.amazon.awssdk.core.sync.RequestBody)19