Examples with S3ObjectMetadata com.formkiq.aws.s3.S3ObjectMetadata used on opensource projects

Example 21 with S3ObjectMetadata

use of com.formkiq.aws.s3.S3ObjectMetadata in project pravega by pravega.

the class ExtendedS3Storage method doConcat.

/**
 * The concat is implemented using extended S3 implementation of multipart copy API. Please see here for
 * more detail on multipart copy:
 * http://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingLLJavaMPUapi.html
 *
 * The multipart copy is an atomic operation. We schedule two parts and commit them atomically using
 * completeMultiPartUpload call. Specifically, to concatenate, we are copying the target segment T and the
 * source segment S to T, so essentially we are doing T <- T + S.
 */
private Void doConcat(SegmentHandle targetHandle, long offset, String sourceSegment) throws StreamSegmentNotExistsException {
    Preconditions.checkArgument(!targetHandle.isReadOnly(), "target handle must not be read-only.");
    long traceId = LoggerHelpers.traceEnter(log, "concat", targetHandle.getSegmentName(), offset, sourceSegment);
    SortedSet<MultipartPartETag> partEtags = new TreeSet<>();
    String targetPath = config.getRoot() + targetHandle.getSegmentName();
    String uploadId = client.initiateMultipartUpload(config.getBucket(), targetPath);
    // check whether the target exists
    if (!doExists(targetHandle.getSegmentName())) {
        throw new StreamSegmentNotExistsException(targetHandle.getSegmentName());
    }
    // check whether the source is sealed
    SegmentProperties si = doGetStreamSegmentInfo(sourceSegment);
    Preconditions.checkState(si.isSealed(), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, targetHandle.getSegmentName());
    // Copy the first part
    CopyPartRequest copyRequest = new CopyPartRequest(config.getBucket(), targetPath, config.getBucket(), targetPath, uploadId, 1).withSourceRange(Range.fromOffsetLength(0, offset));
    CopyPartResult copyResult = client.copyPart(copyRequest);
    partEtags.add(new MultipartPartETag(copyResult.getPartNumber(), copyResult.getETag()));
    // Copy the second part
    S3ObjectMetadata metadataResult = client.getObjectMetadata(config.getBucket(), config.getRoot() + sourceSegment);
    // in bytes
    long objectSize = metadataResult.getContentLength();
    copyRequest = new CopyPartRequest(config.getBucket(), config.getRoot() + sourceSegment, config.getBucket(), targetPath, uploadId, 2).withSourceRange(Range.fromOffsetLength(0, objectSize));
    copyResult = client.copyPart(copyRequest);
    partEtags.add(new MultipartPartETag(copyResult.getPartNumber(), copyResult.getETag()));
    // Close the upload
    client.completeMultipartUpload(new CompleteMultipartUploadRequest(config.getBucket(), targetPath, uploadId).withParts(partEtags));
    client.deleteObject(config.getBucket(), config.getRoot() + sourceSegment);
    LoggerHelpers.traceLeave(log, "concat", traceId);
    return null;
}

Example 22 with S3ObjectMetadata

use of com.formkiq.aws.s3.S3ObjectMetadata in project pravega by pravega.

the class ExtendedS3Storage method doGetStreamSegmentInfo.

private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) {
    long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName);
    S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getRoot() + streamSegmentName);
    AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getRoot() + streamSegmentName);
    boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0);
    StreamSegmentInformation information = StreamSegmentInformation.builder().name(streamSegmentName).length(result.getContentLength()).sealed(!canWrite).lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())).build();
    LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName);
    return information;
}

Example 23 with S3ObjectMetadata

use of com.formkiq.aws.s3.S3ObjectMetadata in project pravega by pravega.

the class S3FileSystemImpl method getObjectMetadata.

@Override
public S3ObjectMetadata getObjectMetadata(String bucketName, String key) {
    S3ObjectMetadata metadata = new S3ObjectMetadata();
    AclSize data = aclMap.get(key);
    if (data == null) {
        throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
    }
    metadata.setContentLength(data.getSize());
    Path path = Paths.get(this.baseDir, bucketName, key);
    metadata.setLastModified(new Date(path.toFile().lastModified()));
    return metadata;
}

Example 24 with S3ObjectMetadata

use of com.formkiq.aws.s3.S3ObjectMetadata in project pravega by pravega.

the class S3ProxyImpl method putObject.

@Override
public PutObjectResult putObject(PutObjectRequest request) {
    S3ObjectMetadata metadata = request.getObjectMetadata();
    if (request.getObjectMetadata() != null) {
        request.setObjectMetadata(null);
    }
    PutObjectResult retVal = client.putObject(request);
    if (request.getAcl() != null) {
        long size = 0;
        if (request.getRange() != null) {
            size = request.getRange().getLast() - 1;
        }
        aclMap.put(request.getKey(), new AclSize(request.getAcl(), size));
    }
    return retVal;
}

Example 25 with S3ObjectMetadata

use of com.formkiq.aws.s3.S3ObjectMetadata in project hippo by NHS-digital-website.

the class ExternalFileCopyTask method processResourceNodes.

@Override
protected void processResourceNodes(final PooledS3Connector s3, final NodeIterator resourceNodes) throws RepositoryException, WorkflowException {
    final Node copiedDocumentFolderNode = copiedDocumentFolder.getNode(getWorkflowContext().getInternalWorkflowSession());
    final Node copiedDocumentHandleNode = copiedDocumentFolderNode.getNode(copiedDocumentName);
    for (final NodeIterator resourceNodeIterator = findResourceNodes(copiedDocumentHandleNode); resourceNodeIterator.hasNext(); ) {
        final Node copiedResourceNode = resourceNodeIterator.nextNode();
        final String oldReference = copiedResourceNode.getProperty(PROPERTY_EXTERNAL_STORAGE_REFERENCE).getString();
        final S3ObjectMetadata s3ObjectMetadata = s3.copyFile(oldReference, copiedResourceNode.getProperty(PROPERTY_EXTERNAL_STORAGE_FILE_NAME).getString());
        s3.unpublishResource(s3ObjectMetadata.getReference());
        copiedResourceNode.setProperty(PROPERTY_EXTERNAL_STORAGE_REFERENCE, s3ObjectMetadata.getReference());
        copiedResourceNode.setProperty(PROPERTY_EXTERNAL_STORAGE_PUBLIC_URL, s3ObjectMetadata.getUrl());
        log.debug("Copied external resource {} as private {}", oldReference, s3ObjectMetadata);
    }
}