Examples with InitiateMultipartUploadRequest com.aliyun.oss.model.InitiateMultipartUploadRequest used on opensource projects

Example 1 with InitiateMultipartUploadRequest

use of com.aliyun.oss.model.InitiateMultipartUploadRequest in project hadoop by apache.

the class AliyunOSSFileSystemStore method multipartUploadObject.

/**
   * Upload a file as an OSS object, using multipart upload.
   *
   * @param key object key.
   * @param file local file to upload.
   * @throws IOException if failed to upload object.
   */
public void multipartUploadObject(String key, File file) throws IOException {
    File object = file.getAbsoluteFile();
    long dataLen = object.length();
    long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
    int partNum = (int) (dataLen / realPartSize);
    if (dataLen % realPartSize != 0) {
        partNum += 1;
    }
    InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
    ObjectMetadata meta = new ObjectMetadata();
    if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
        meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    initiateMultipartUploadRequest.setObjectMetadata(meta);
    InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
    List<PartETag> partETags = new ArrayList<PartETag>();
    String uploadId = initiateMultipartUploadResult.getUploadId();
    try {
        for (int i = 0; i < partNum; i++) {
            // TODO: Optimize this, avoid opening the object multiple times
            FileInputStream fis = new FileInputStream(object);
            try {
                long skipBytes = realPartSize * i;
                AliyunOSSUtils.skipFully(fis, skipBytes);
                long size = (realPartSize < dataLen - skipBytes) ? realPartSize : dataLen - skipBytes;
                UploadPartRequest uploadPartRequest = new UploadPartRequest();
                uploadPartRequest.setBucketName(bucketName);
                uploadPartRequest.setKey(key);
                uploadPartRequest.setUploadId(uploadId);
                uploadPartRequest.setInputStream(fis);
                uploadPartRequest.setPartSize(size);
                uploadPartRequest.setPartNumber(i + 1);
                UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
                statistics.incrementWriteOps(1);
                partETags.add(uploadPartResult.getPartETag());
            } finally {
                fis.close();
            }
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        LOG.debug(completeMultipartUploadResult.getETag());
    } catch (OSSException | ClientException e) {
        AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId);
        ossClient.abortMultipartUpload(abortMultipartUploadRequest);
    }
}

Example 2 with InitiateMultipartUploadRequest

use of com.aliyun.oss.model.InitiateMultipartUploadRequest in project hadoop by apache.

the class AliyunOSSFileSystemStore method multipartCopy.

/**
   * Use multipart copy to copy an OSS object.
   * (The caller should make sure srcPath is a file and dstPath is valid)
   *
   * @param srcKey source key.
   * @param contentLength data size of the object to copy.
   * @param dstKey destination key.
   * @return true if success, or false if upload is aborted.
   */
private boolean multipartCopy(String srcKey, long contentLength, String dstKey) {
    long realPartSize = AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize);
    int partNum = (int) (contentLength / realPartSize);
    if (contentLength % realPartSize != 0) {
        partNum++;
    }
    InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, dstKey);
    ObjectMetadata meta = new ObjectMetadata();
    if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
        meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    initiateMultipartUploadRequest.setObjectMetadata(meta);
    InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
    String uploadId = initiateMultipartUploadResult.getUploadId();
    List<PartETag> partETags = new ArrayList<PartETag>();
    try {
        for (int i = 0; i < partNum; i++) {
            long skipBytes = realPartSize * i;
            long size = (realPartSize < contentLength - skipBytes) ? realPartSize : contentLength - skipBytes;
            UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
            partCopyRequest.setSourceBucketName(bucketName);
            partCopyRequest.setSourceKey(srcKey);
            partCopyRequest.setBucketName(bucketName);
            partCopyRequest.setKey(dstKey);
            partCopyRequest.setUploadId(uploadId);
            partCopyRequest.setPartSize(size);
            partCopyRequest.setBeginIndex(skipBytes);
            partCopyRequest.setPartNumber(i + 1);
            UploadPartCopyResult partCopyResult = ossClient.uploadPartCopy(partCopyRequest);
            statistics.incrementWriteOps(1);
            partETags.add(partCopyResult.getPartETag());
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, dstKey, uploadId, partETags);
        CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
        LOG.debug(completeMultipartUploadResult.getETag());
        return true;
    } catch (OSSException | ClientException e) {
        AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
        ossClient.abortMultipartUpload(abortMultipartUploadRequest);
        return false;
    }
}

Example 3 with InitiateMultipartUploadRequest

use of com.aliyun.oss.model.InitiateMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.

the class OSSUploadOperation method prepare.

private void prepare(UploadCheckPoint uploadCheckPoint, UploadFileRequest uploadFileRequest) {
    uploadCheckPoint.magic = UploadCheckPoint.UPLOAD_MAGIC;
    uploadCheckPoint.uploadFile = uploadFileRequest.getUploadFile();
    uploadCheckPoint.key = uploadFileRequest.getKey();
    uploadCheckPoint.uploadFileStat = FileStat.getFileStat(uploadCheckPoint.uploadFile);
    uploadCheckPoint.uploadParts = splitFile(uploadCheckPoint.uploadFileStat.size, uploadFileRequest.getPartSize());
    uploadCheckPoint.partETags = new ArrayList<PartETag>();
    ObjectMetadata metadata = uploadFileRequest.getObjectMetadata();
    if (metadata == null) {
        metadata = new ObjectMetadata();
    }
    if (metadata.getContentType() == null) {
        metadata.setContentType(Mimetypes.getInstance().getMimetype(uploadCheckPoint.uploadFile, uploadCheckPoint.key));
    }
    InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest(uploadFileRequest.getBucketName(), uploadFileRequest.getKey(), metadata);
    InitiateMultipartUploadResult initiateUploadResult = multipartOperation.initiateMultipartUpload(initiateUploadRequest);
    uploadCheckPoint.uploadID = initiateUploadResult.getUploadId();
}

Example 4 with InitiateMultipartUploadRequest

use of com.aliyun.oss.model.InitiateMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.

the class UploadPartCopySample method main.

public static void main(String[] args) throws IOException {
    /*
         * Constructs a client instance with your account for accessing OSS
         */
    client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
    try {
        /*
             * Upload an object to your source bucket
             */
        System.out.println("Uploading a new object to OSS from a file\n");
        client.putObject(new PutObjectRequest(sourceBucketName, sourceKey, createSampleFile()));
        /*
             * Claim a new upload id for your target bucket
             */
        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(targetBucketName, targetKey);
        InitiateMultipartUploadResult initiateMultipartUploadResult = client.initiateMultipartUpload(initiateMultipartUploadRequest);
        String uploadId = initiateMultipartUploadResult.getUploadId();
        /*
             * Calculate how many parts to be divided
             */
        // 5MB
        final long partSize = 5 * 1024 * 1024L;
        ObjectMetadata metadata = client.getObjectMetadata(sourceBucketName, sourceKey);
        long objectSize = metadata.getContentLength();
        int partCount = (int) (objectSize / partSize);
        if (objectSize % partSize != 0) {
            partCount++;
        }
        if (partCount > 10000) {
            throw new RuntimeException("Total parts count should not exceed 10000");
        } else {
            System.out.println("Total parts count " + partCount + "\n");
        }
        /*
             * Upload multiparts by copy mode
             */
        System.out.println("Begin to upload multiparts by copy mode to OSS\n");
        List<PartETag> partETags = new ArrayList<PartETag>();
        for (int i = 0; i < partCount; i++) {
            long startPos = i * partSize;
            long curPartSize = (i + 1 == partCount) ? (objectSize - startPos) : partSize;
            ;
            UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(sourceBucketName, sourceKey, targetBucketName, targetKey);
            uploadPartCopyRequest.setUploadId(uploadId);
            uploadPartCopyRequest.setPartSize(curPartSize);
            uploadPartCopyRequest.setBeginIndex(startPos);
            uploadPartCopyRequest.setPartNumber(i + 1);
            UploadPartCopyResult uploadPartCopyResult = client.uploadPartCopy(uploadPartCopyRequest);
            System.out.println("\tPart#" + uploadPartCopyResult.getPartNumber() + " done\n");
            partETags.add(uploadPartCopyResult.getPartETag());
        }
        /*
             * Complete to upload multiparts
             */
        System.out.println("Completing to upload multiparts\n");
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(targetBucketName, targetKey, uploadId, partETags);
        client.completeMultipartUpload(completeMultipartUploadRequest);
        /*
             * Fetch the object that newly created at the step below.
             */
        System.out.println("Fetching an object");
        client.getObject(new GetObjectRequest(targetBucketName, targetKey), new File(localFilePath));
    } catch (OSSException oe) {
        System.out.println("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason.");
        System.out.println("Error Message: " + oe.getErrorCode());
        System.out.println("Error Code:       " + oe.getErrorCode());
        System.out.println("Request ID:      " + oe.getRequestId());
        System.out.println("Host ID:           " + oe.getHostId());
    } catch (ClientException ce) {
        System.out.println("Caught an ClientException, which means the client encountered " + "a serious internal problem while trying to communicate with OSS, " + "such as not being able to access the network.");
        System.out.println("Error Message: " + ce.getMessage());
    } finally {
        /*
             * Do not forget to shut down the client finally to release all allocated resources.
             */
        client.shutdown();
    }
}

Example 5 with InitiateMultipartUploadRequest

use of com.aliyun.oss.model.InitiateMultipartUploadRequest in project aliyun-oss-java-sdk by aliyun.

the class OSSClientRequestTest method testInitiateMultipartUploadRequest.

@SuppressWarnings("serial")
@Test
public void testInitiateMultipartUploadRequest() {
    final InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(bucketName, objectKey);
    TestAction test1 = new TestAction() {

        public void run() throws Exception {
            multipartOp.initiateMultipartUpload(initiateMultipartUploadRequest);
        }
    };
    executeTest(test1, HttpMethod.POST, bucketName + "." + endpoint.getHost(), objectKey + "?uploads", null);
    final ObjectMetadata metadata = new ObjectMetadata();
    metadata.setCacheControl("no-cache");
    metadata.setServerSideEncryption(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    metadata.addUserMetadata("zdata", "data");
    initiateMultipartUploadRequest.setObjectMetadata(metadata);
    executeTest(test1, HttpMethod.POST, bucketName + "." + endpoint.getHost(), objectKey + "?uploads", new HashMap<String, String>() {

        {
            put("Cache-Control", metadata.getCacheControl());
            put(OSSHeaders.OSS_USER_METADATA_PREFIX + "zdata", "data");
            put("x-oss-server-side-encryption", metadata.getServerSideEncryption());
        }
    });
}