Examples with UploadPartCopyRequest com.aliyun.oss.model.UploadPartCopyRequest used on opensource projects

Example 1 with UploadPartCopyRequest

use of com.aliyun.oss.model.UploadPartCopyRequest in project hadoop by apache.

the class AliyunOSSFileSystemStore method multipartCopy.

/**
   * Use multipart copy to copy an OSS object.
   * (The caller should make sure srcPath is a file and dstPath is valid)
   *
   * @param srcKey source key.
   * @param contentLength data size of the object to copy.
   * @param dstKey destination key.
   * @return true if success, or false if upload is aborted.
   */
private boolean multipartCopy(String srcKey, long contentLength, String dstKey) {
    long realPartSize = AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize);
    int partNum = (int) (contentLength / realPartSize);
    if (contentLength % realPartSize != 0) {
        partNum++;
    }
    InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, dstKey);
    ObjectMetadata meta = new ObjectMetadata();
    if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
        meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    initiateMultipartUploadRequest.setObjectMetadata(meta);
    InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
    String uploadId = initiateMultipartUploadResult.getUploadId();
    List<PartETag> partETags = new ArrayList<PartETag>();
    try {
        for (int i = 0; i < partNum; i++) {
            long skipBytes = realPartSize * i;
            long size = (realPartSize < contentLength - skipBytes) ? realPartSize : contentLength - skipBytes;
            UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
            partCopyRequest.setSourceBucketName(bucketName);
            partCopyRequest.setSourceKey(srcKey);
            partCopyRequest.setBucketName(bucketName);
            partCopyRequest.setKey(dstKey);
            partCopyRequest.setUploadId(uploadId);
            partCopyRequest.setPartSize(size);
            partCopyRequest.setBeginIndex(skipBytes);
            partCopyRequest.setPartNumber(i + 1);
            UploadPartCopyResult partCopyResult = ossClient.uploadPartCopy(partCopyRequest);
            statistics.incrementWriteOps(1);
            partETags.add(partCopyResult.getPartETag());
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, dstKey, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        LOG.debug(completeMultipartUploadResult.getETag());
        return true;
    } catch (OSSException | ClientException e) {
        AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
        ossClient.abortMultipartUpload(abortMultipartUploadRequest);
        return false;
    }
}

Example 2 with UploadPartCopyRequest

use of com.aliyun.oss.model.UploadPartCopyRequest in project aliyun-oss-java-sdk by aliyun.

the class UploadPartCopySample method main.

public static void main(String[] args) throws IOException {
    /*
         * Constructs a client instance with your account for accessing OSS
         */
    client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
    try {
        /*
             * Upload an object to your source bucket
             */
        System.out.println("Uploading a new object to OSS from a file\n");
        client.putObject(new PutObjectRequest(sourceBucketName, sourceKey, createSampleFile()));
        /*
             * Claim a new upload id for your target bucket
             */
        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(targetBucketName, targetKey);
        InitiateMultipartUploadResult initiateMultipartUploadResult = client.initiateMultipartUpload(initiateMultipartUploadRequest);
        String uploadId = initiateMultipartUploadResult.getUploadId();
        /*
             * Calculate how many parts to be divided
             */
        // 5MB
        final long partSize = 5 * 1024 * 1024L;
        ObjectMetadata metadata = client.getObjectMetadata(sourceBucketName, sourceKey);
        long objectSize = metadata.getContentLength();
        int partCount = (int) (objectSize / partSize);
        if (objectSize % partSize != 0) {
            partCount++;
        }
        if (partCount > 10000) {
            throw new RuntimeException("Total parts count should not exceed 10000");
        } else {
            System.out.println("Total parts count " + partCount + "\n");
        }
        /*
             * Upload multiparts by copy mode
             */
        System.out.println("Begin to upload multiparts by copy mode to OSS\n");
        List<PartETag> partETags = new ArrayList<PartETag>();
        for (int i = 0; i < partCount; i++) {
            long startPos = i * partSize;
            long curPartSize = (i + 1 == partCount) ? (objectSize - startPos) : partSize;
            ;
            UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucketName, sourceKey, targetBucketName, targetKey);
            uploadPartCopyRequest.setUploadId(uploadId);
            uploadPartCopyRequest.setPartSize(curPartSize);
            uploadPartCopyRequest.setBeginIndex(startPos);
            uploadPartCopyRequest.setPartNumber(i + 1);
            UploadPartCopyResult uploadPartCopyResult = client.uploadPartCopy(uploadPartCopyRequest);
            System.out.println("\tPart#" + uploadPartCopyResult.getPartNumber() + " done\n");
            partETags.add(uploadPartCopyResult.getPartETag());
        }
        /*
             * Complete to upload multiparts
             */
        System.out.println("Completing to upload multiparts\n");
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucketName, targetKey, uploadId, partETags);
        client.completeMultipartUpload(completeMultipartUploadRequest);
        /*
             * Fetch the object that newly created at the step below.
             */
        System.out.println("Fetching an object");
        client.getObject(new GetObjectRequest(targetBucketName, targetKey), new File(localFilePath));
    } catch (OSSException oe) {
        System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason.");
        System.out.println("Error Message: " + oe.getErrorCode());
        System.out.println("Error Code:       " + oe.getErrorCode());
        System.out.println("Request ID:      " + oe.getRequestId());
        System.out.println("Host ID:           " + oe.getHostId());
    } catch (ClientException ce) {
        System.out.println("Caught an ClientException, which means the client encountered " + "a serious internal problem while trying to communicate with OSS, " + "such as not being able to access the network.");
        System.out.println("Error Message: " + ce.getMessage());
    } finally {
        /*
             * Do not forget to shut down the client finally to release all allocated resources.
             */
        client.shutdown();
    }
}

Example 3 with UploadPartCopyRequest

use of com.aliyun.oss.model.UploadPartCopyRequest in project aliyun-oss-java-sdk by aliyun.

the class UploadPartCopyTest method testUnormalUploadPartCopyByRange.

@Test
public void testUnormalUploadPartCopyByRange() {
    final String sourceBucket = "unormal-upload-part-copy-by-range-bucket-source";
    final String targetBucket = "unormal-upload-part-copy-by-range-bucket-target";
    final String sourceKey = "unormal-upload-part-copy-by-range-object-source";
    final String targetKey = "unormal-upload-part-copy-by-range-object-target";
    // 128KB
    final long partSize = 128 * 1024;
    try {
        ossClient.createBucket(sourceBucket);
        ossClient.createBucket(targetBucket);
        // Put object into source bucket
        final long inputStreamLength = partSize * 4;
        String eTag = null;
        try {
            InputStream instream = genFixedLengthInputStream(inputStreamLength);
            PutObjectResult result = ossClient.putObject(sourceBucket, sourceKey, instream, null);
            eTag = result.getETag();
        } catch (Exception e) {
            Assert.fail(e.getMessage());
        }
        // Claim upload id for target bucket
        String uploadId = claimUploadId(ossClient, targetBucket, targetKey);
        // Upload part copy with invalid copy range
        final int partNumber = 1;
        final long beginIndex = partSize;
        List<PartETag> partETags = new ArrayList<PartETag>();
        UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucket, sourceKey, targetBucket, targetKey);
        uploadPartCopyRequest.setPartNumber(partNumber);
        uploadPartCopyRequest.setUploadId(uploadId);
        uploadPartCopyRequest.setBeginIndex(beginIndex);
        // Illegal copy range([beginIndex, begin + inputStreamLength]), just copy entire object
        uploadPartCopyRequest.setPartSize(inputStreamLength);
        UploadPartCopyResult uploadPartCopyResult = ossClient.uploadPartCopy(uploadPartCopyRequest);
        partETags.add(uploadPartCopyResult.getPartETag());
        Assert.assertEquals(eTag, uploadPartCopyResult.getETag());
        Assert.assertEquals(partNumber, uploadPartCopyResult.getPartNumber());
        Assert.assertEquals(uploadPartCopyResult.getRequestId().length(), REQUEST_ID_LEN);
        ListPartsRequest listPartsRequest = new ListPartsRequest(targetBucket, targetKey, uploadId);
        PartListing partListing = ossClient.listParts(listPartsRequest);
        Assert.assertEquals(1, partListing.getParts().size());
        Assert.assertEquals(targetBucket, partListing.getBucketName());
        Assert.assertEquals(targetKey, partListing.getKey());
        Assert.assertEquals(uploadId, partListing.getUploadId());
        Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
        Assert.assertNotNull(partListing.getNextPartNumberMarker());
        Assert.assertFalse(partListing.isTruncated());
        Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
        // Complete multipart upload
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, targetBucket, targetKey), completeMultipartUploadResult.getLocation());
        Assert.assertEquals(targetBucket, completeMultipartUploadResult.getBucketName());
        Assert.assertEquals(targetKey, completeMultipartUploadResult.getKey());
        Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
        Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
        // Get uploaded object
        OSSObject o = ossClient.getObject(targetBucket, targetKey);
        final long objectSize = inputStreamLength;
        Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
        Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
        Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        deleteBucketWithObjects(ossClient, sourceBucket);
        deleteBucketWithObjects(ossClient, targetBucket);
    }
}

Example 4 with UploadPartCopyRequest

use of com.aliyun.oss.model.UploadPartCopyRequest in project aliyun-oss-java-sdk by aliyun.

the class GetRequestIdTest method testNormalGetRequestId.

@Test
public void testNormalGetRequestId() {
    final String key = "normal-get-request-id";
    final long inputStreamLength = 1024;
    final int requestIdLength = "572BF2F2207FB3397648E9F1".length();
    try {
        // put object
        PutObjectResult putObjectResult = ossClient.putObject(bucketName, key, genFixedLengthInputStream(inputStreamLength));
        Assert.assertEquals(putObjectResult.getRequestId().length(), requestIdLength);
        // get object
        OSSObject ossObject = ossClient.getObject(bucketName, key);
        ossObject.getObjectContent().close();
        Assert.assertEquals(ossObject.getRequestId().length(), requestIdLength);
        File file = new File("tmp");
        GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, key);
        ObjectMetadata objectMeta = ossClient.getObject(getObjectRequest, file);
        Assert.assertEquals(objectMeta.getRequestId().length(), requestIdLength);
        // delete object
        ossClient.deleteObject(bucketName, key);
        // append object
        AppendObjectRequest appendObjectRequest = new AppendObjectRequest(bucketName, key, file);
        appendObjectRequest.setPosition(0L);
        AppendObjectResult appendObjectResult = ossClient.appendObject(appendObjectRequest);
        Assert.assertEquals(appendObjectResult.getRequestId().length(), requestIdLength);
        // getSimplifiedObjectMeta
        SimplifiedObjectMeta simplifiedObjectMeta = ossClient.getSimplifiedObjectMeta(bucketName, key);
        Assert.assertEquals(simplifiedObjectMeta.getRequestId().length(), requestIdLength);
        // getObjectMetadata
        ObjectMetadata objectMetadata = ossClient.getObjectMetadata(bucketName, key);
        Assert.assertEquals(objectMetadata.getRequestId().length(), requestIdLength);
        // delete objects
        DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
        ArrayList<String> keys = new ArrayList<String>();
        keys.add(key);
        deleteObjectsRequest.setKeys(keys);
        DeleteObjectsResult deleteObjectsResult = ossClient.deleteObjects(deleteObjectsRequest);
        Assert.assertEquals(deleteObjectsResult.getRequestId().length(), requestIdLength);
        // initiate multipart upload
        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
        InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
        Assert.assertEquals(initiateMultipartUploadResult.getRequestId().length(), requestIdLength);
        // upload part
        UploadPartRequest uploadPartRequest = new UploadPartRequest(bucketName, key, initiateMultipartUploadResult.getUploadId(), 1, new FileInputStream(file), inputStreamLength);
        UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
        Assert.assertEquals(uploadPartResult.getRequestId().length(), requestIdLength);
        // complete multipart upload
        List<PartETag> partETags = new ArrayList<PartETag>();
        partETags.add(new PartETag(1, uploadPartResult.getETag()));
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, initiateMultipartUploadResult.getUploadId(), partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), requestIdLength);
        // copy object
        CopyObjectResult CopyObjectResult = ossClient.copyObject(bucketName, key, bucketName, key);
        Assert.assertEquals(CopyObjectResult.getRequestId().length(), requestIdLength);
        // initiate multipart copy
        InitiateMultipartUploadRequest initiateMultipartCopyRequest = new InitiateMultipartUploadRequest(bucketName, key);
        InitiateMultipartUploadResult initiateMultipartCopyResult = ossClient.initiateMultipartUpload(initiateMultipartCopyRequest);
        Assert.assertEquals(initiateMultipartCopyResult.getRequestId().length(), requestIdLength);
        // upload part copy
        UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(bucketName, key, bucketName, key, initiateMultipartCopyResult.getUploadId(), 1, 0L, inputStreamLength);
        UploadPartCopyResult uploadPartCopyResult = ossClient.uploadPartCopy(uploadPartCopyRequest);
        Assert.assertEquals(uploadPartCopyResult.getRequestId().length(), requestIdLength);
        // abort multipart upload
        AbortMultipartUploadRequest AbortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, initiateMultipartCopyResult.getUploadId());
        ossClient.abortMultipartUpload(AbortMultipartUploadRequest);
        ossClient.deleteObject(bucketName, key);
        file.delete();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 5 with UploadPartCopyRequest

use of com.aliyun.oss.model.UploadPartCopyRequest in project aliyun-oss-java-sdk by aliyun.

the class UploadPartCopyTest method testNormalUploadPartCopy.

@Test
public void testNormalUploadPartCopy() {
    final String sourceBucket = "normal-upload-part-copy-bucket-source";
    final String targetBucket = "normal-upload-part-copy-bucket-target";
    final String sourceKey = "normal-upload-part-copy-object-source";
    final String targetKey = "normal-upload-part-copy-object-target";
    // 128KB
    final long partSize = 128 * 1024;
    try {
        ossClient.createBucket(sourceBucket);
        ossClient.createBucket(targetBucket);
        waitForCacheExpiration(5);
        // Put object into source bucket
        String eTag = null;
        try {
            InputStream instream = genFixedLengthInputStream(partSize);
            PutObjectResult result = ossClient.putObject(sourceBucket, sourceKey, instream, null);
            eTag = result.getETag();
        } catch (Exception e) {
            Assert.fail(e.getMessage());
        }
        // Claim upload id for target bucket
        String uploadId = claimUploadId(ossClient, targetBucket, targetKey);
        // Upload part copy
        final int partNumber = 1;
        List<PartETag> partETags = new ArrayList<PartETag>();
        UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucket, sourceKey, targetBucket, targetKey);
        uploadPartCopyRequest.setPartNumber(partNumber);
        uploadPartCopyRequest.setUploadId(uploadId);
        UploadPartCopyResult uploadPartCopyResult = ossClient.uploadPartCopy(uploadPartCopyRequest);
        partETags.add(uploadPartCopyResult.getPartETag());
        Assert.assertEquals(eTag, uploadPartCopyResult.getETag());
        Assert.assertEquals(partNumber, uploadPartCopyResult.getPartNumber());
        Assert.assertEquals(uploadPartCopyResult.getRequestId().length(), REQUEST_ID_LEN);
        ListPartsRequest listPartsRequest = new ListPartsRequest(targetBucket, targetKey, uploadId);
        PartListing partListing = ossClient.listParts(listPartsRequest);
        Assert.assertEquals(1, partListing.getParts().size());
        Assert.assertEquals(targetBucket, partListing.getBucketName());
        Assert.assertEquals(targetKey, partListing.getKey());
        Assert.assertEquals(uploadId, partListing.getUploadId());
        Assert.assertEquals(LIST_PART_MAX_RETURNS, partListing.getMaxParts().intValue());
        Assert.assertNotNull(partListing.getNextPartNumberMarker());
        Assert.assertFalse(partListing.isTruncated());
        Assert.assertEquals(partListing.getRequestId().length(), REQUEST_ID_LEN);
        // Complete multipart upload
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        Assert.assertEquals(composeLocation(ossClient, OSS_TEST_ENDPOINT, targetBucket, targetKey), completeMultipartUploadResult.getLocation());
        Assert.assertEquals(targetBucket, completeMultipartUploadResult.getBucketName());
        Assert.assertEquals(targetKey, completeMultipartUploadResult.getKey());
        Assert.assertEquals(calcMultipartsETag(partETags), completeMultipartUploadResult.getETag());
        Assert.assertEquals(completeMultipartUploadResult.getRequestId().length(), REQUEST_ID_LEN);
        // Get uploaded object
        OSSObject o = ossClient.getObject(targetBucket, targetKey);
        final long objectSize = 1 * partSize;
        Assert.assertEquals(objectSize, o.getObjectMetadata().getContentLength());
        Assert.assertEquals(calcMultipartsETag(partETags), o.getObjectMetadata().getETag());
        Assert.assertEquals(o.getRequestId().length(), REQUEST_ID_LEN);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        deleteBucketWithObjects(ossClient, sourceBucket);
        deleteBucketWithObjects(ossClient, targetBucket);
    }
}