Examples with OmMultipartInfo org.apache.hadoop.ozone.om.helpers.OmMultipartInfo used on opensource projects

Example 16 with OmMultipartInfo

use of org.apache.hadoop.ozone.om.helpers.OmMultipartInfo in project ozone by apache.

the class ObjectEndpoint method initializeMultipartUpload.

/**
 * Initialize MultiPartUpload request.
 * <p>
 * Note: the specific content type is set by the HeaderPreprocessor.
 */
@POST
@Produces(MediaType.APPLICATION_XML)
@Consumes(HeaderPreprocessor.MULTIPART_UPLOAD_MARKER)
public Response initializeMultipartUpload(@PathParam("bucket") String bucket, @PathParam("path") String key) throws IOException, OS3Exception {
    S3GAction s3GAction = S3GAction.INIT_MULTIPART_UPLOAD;
    try {
        OzoneBucket ozoneBucket = getBucket(bucket);
        String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
        ReplicationConfig replicationConfig = getReplicationConfig(ozoneBucket, storageType);
        OmMultipartInfo multipartInfo = ozoneBucket.initiateMultipartUpload(key, replicationConfig);
        MultipartUploadInitiateResponse multipartUploadInitiateResponse = new MultipartUploadInitiateResponse();
        multipartUploadInitiateResponse.setBucket(bucket);
        multipartUploadInitiateResponse.setKey(key);
        multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID());
        AUDIT.logWriteSuccess(buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
        getMetrics().incInitMultiPartUploadSuccess();
        return Response.status(Status.OK).entity(multipartUploadInitiateResponse).build();
    } catch (OMException ex) {
        AUDIT.logWriteFailure(buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex));
        getMetrics().incInitMultiPartUploadFailure();
        if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
            throw newError(S3ErrorTable.ACCESS_DENIED, key, ex);
        }
        LOG.error("Error in Initiate Multipart Upload Request for bucket: {}, " + "key: {}", bucket, key, ex);
        throw ex;
    } catch (Exception ex) {
        AUDIT.logWriteFailure(buildAuditMessageForFailure(s3GAction, getAuditParameters(), ex));
        getMetrics().incInitMultiPartUploadFailure();
        throw ex;
    }
}

Example 17 with OmMultipartInfo

use of org.apache.hadoop.ozone.om.helpers.OmMultipartInfo in project ozone by apache.

the class TestOzoneRpcClientAbstract method testInitiateMultipartUploadWithReplicationInformationSet.

@Test
public void testInitiateMultipartUploadWithReplicationInformationSet() throws IOException {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String keyName = UUID.randomUUID().toString();
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName, RATIS, ONE);
    assertNotNull(multipartInfo);
    String uploadID = multipartInfo.getUploadID();
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotNull(multipartInfo.getUploadID());
    // Call initiate multipart upload for the same key again, this should
    // generate a new uploadID.
    multipartInfo = bucket.initiateMultipartUpload(keyName, RATIS, ONE);
    assertNotNull(multipartInfo);
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotEquals(multipartInfo.getUploadID(), uploadID);
    assertNotNull(multipartInfo.getUploadID());
}

Example 18 with OmMultipartInfo

use of org.apache.hadoop.ozone.om.helpers.OmMultipartInfo in project ozone by apache.

the class TestOzoneRpcClientAbstract method testUploadPartOverrideWithRatis.

@Test
public void testUploadPartOverrideWithRatis() throws IOException {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String keyName = UUID.randomUUID().toString();
    String sampleData = "sample Value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName, ReplicationType.RATIS, THREE);
    assertNotNull(multipartInfo);
    String uploadID = multipartInfo.getUploadID();
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotNull(multipartInfo.getUploadID());
    int partNumber = 1;
    OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName, sampleData.length(), partNumber, uploadID);
    ozoneOutputStream.write(string2Bytes(sampleData), 0, sampleData.length());
    ozoneOutputStream.close();
    OmMultipartCommitUploadPartInfo commitUploadPartInfo = ozoneOutputStream.getCommitUploadPartInfo();
    assertNotNull(commitUploadPartInfo);
    String partName = commitUploadPartInfo.getPartName();
    assertNotNull(commitUploadPartInfo.getPartName());
    // Overwrite the part by creating part key with same part number
    // and same content.
    ozoneOutputStream = bucket.createMultipartKey(keyName, sampleData.length(), partNumber, uploadID);
    ozoneOutputStream.write(string2Bytes(sampleData), 0, "name".length());
    ozoneOutputStream.close();
    commitUploadPartInfo = ozoneOutputStream.getCommitUploadPartInfo();
    assertNotNull(commitUploadPartInfo);
    assertNotNull(commitUploadPartInfo.getPartName());
    // AWS S3 for same content generates same partName during upload part.
    // In AWS S3 ETag is generated from md5sum. In Ozone right now we
    // don't do this. For now to make things work for large file upload
    // through aws s3 cp, the partName are generated in a predictable fashion.
    // So, when a part is override partNames will still be same irrespective
    // of content in ozone s3. This will make S3 Mpu completeMPU pass when
    // comparing part names and large file uploads work using aws cp.
    assertEquals("Part names should be same", partName, commitUploadPartInfo.getPartName());
}

Example 19 with OmMultipartInfo

use of org.apache.hadoop.ozone.om.helpers.OmMultipartInfo in project ozone by apache.

the class TestOzoneRpcClientAbstract method testInitiateMultipartUpload.

@ParameterizedTest
@MethodSource("replicationConfigs")
void testInitiateMultipartUpload(ReplicationConfig replicationConfig) throws IOException {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String keyName = UUID.randomUUID().toString();
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName, replicationConfig);
    assertNotNull(multipartInfo);
    String uploadID = multipartInfo.getUploadID();
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotNull(multipartInfo.getUploadID());
    // Call initiate multipart upload for the same key again, this should
    // generate a new uploadID.
    multipartInfo = bucket.initiateMultipartUpload(keyName, replicationConfig);
    assertNotNull(multipartInfo);
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotEquals(multipartInfo.getUploadID(), uploadID);
    assertNotNull(multipartInfo.getUploadID());
}

Example 20 with OmMultipartInfo

use of org.apache.hadoop.ozone.om.helpers.OmMultipartInfo in project ozone by apache.

the class TestOzoneRpcClientAbstract method testUploadPartWithNoOverride.

@ParameterizedTest
@MethodSource("replicationConfigs")
void testUploadPartWithNoOverride(ReplicationConfig replication) throws IOException {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String keyName = UUID.randomUUID().toString();
    String sampleData = "sample Value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName, replication);
    assertNotNull(multipartInfo);
    String uploadID = multipartInfo.getUploadID();
    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
    Assert.assertEquals(keyName, multipartInfo.getKeyName());
    assertNotNull(multipartInfo.getUploadID());
    OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName, sampleData.length(), 1, uploadID);
    ozoneOutputStream.write(string2Bytes(sampleData), 0, sampleData.length());
    ozoneOutputStream.close();
    OmMultipartCommitUploadPartInfo commitUploadPartInfo = ozoneOutputStream.getCommitUploadPartInfo();
    assertNotNull(commitUploadPartInfo);
    assertNotNull(commitUploadPartInfo.getPartName());
}