Examples with PartETag com.qcloud.cos.model.PartETag used on opensource projects

Example 6 with PartETag

use of com.qcloud.cos.model.PartETag in project cos-java-sdk-v5 by tencentyun.

the class BlindWatermarkDemo method addBlindWatermarkWithMultipart.

public static void addBlindWatermarkWithMultipart(COSClient cosClient) throws FileNotFoundException {
    // bucket名需包含appid
    // api 请参考：https://cloud.tencent.com/document/product/436/46782
    String bucketName = "examplebucket-1250000000";
    String key = "qrcode.png";
    File localFile = new File("E://qrcode.png");
    InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
    InitiateMultipartUploadResult initResult = cosClient.initiateMultipartUpload(request);
    String uploadId = initResult.getUploadId();
    // 上传分块
    List<PartETag> partETags = new LinkedList<>();
    UploadPartRequest uploadPartRequest = new UploadPartRequest();
    uploadPartRequest.setBucketName(bucketName);
    uploadPartRequest.setKey(key);
    uploadPartRequest.setUploadId(uploadId);
    // 设置分块的数据来源输入流
    uploadPartRequest.setInputStream(new FileInputStream(localFile));
    // 设置分块的长度
    // 设置数据长度
    uploadPartRequest.setPartSize(localFile.length());
    // 假设要上传的part编号是10
    uploadPartRequest.setPartNumber(1);
    UploadPartResult uploadPartResult = cosClient.uploadPart(uploadPartRequest);
    PartETag partETag = uploadPartResult.getPartETag();
    partETags.add(partETag);
    // 合并分块并带上图像处理参数
    CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
    PicOperations picOperations = new PicOperations();
    picOperations.setIsPicInfo(1);
    List<PicOperations.Rule> ruleList = new LinkedList<>();
    PicOperations.Rule rule1 = new PicOperations.Rule();
    rule1.setBucket(bucketName);
    rule1.setFileId("qrcode-watermark.png");
    rule1.setRule("watermark/3/type/3/text/dGVuY2VudCBjbG91ZA==");
    ruleList.add(rule1);
    picOperations.setRules(ruleList);
    completeMultipartUploadRequest.setPicOperations(picOperations);
    CompleteMultipartUploadResult completeMultipartUploadResult = cosClient.completeMultipartUpload(completeMultipartUploadRequest);
    CIUploadResult ciUploadResult = completeMultipartUploadResult.getCiUploadResult();
    System.out.println(completeMultipartUploadResult.getRequestId());
    System.out.println(ciUploadResult.getOriginalInfo().getEtag());
    for (CIObject ciObject : ciUploadResult.getProcessResults().getObjectList()) {
        System.out.println(ciObject.getLocation());
    }
}

Example 7 with PartETag

use of com.qcloud.cos.model.PartETag in project hadoop-cos by tencentyun.

the class CosNFSDataOutputStream method close.

@Override
public synchronized void close() throws IOException {
    if (this.closed) {
        return;
    }
    if (this.currentBlockOutputStream == null) {
        initCurrentBlock();
    }
    try {
        this.currentBlockOutputStream.flush();
        this.currentBlockOutputStream.close();
        // 加到块列表中去
        if (this.currentBlockId == 0) {
            // 单个文件就可以上传完成
            LOG.info("Single file upload...  key: {}, blockId: {}, blockWritten: {}.", this.key, this.currentBlockId, this.blockWritten);
            byte[] md5Hash = this.digest == null ? null : this.digest.digest();
            int size = this.currentBlockBuffer.getByteBuffer().remaining();
            store.storeFile(this.key, new BufferInputStream(this.currentBlockBuffer), md5Hash, this.currentBlockBuffer.getByteBuffer().remaining());
            if (null != this.writeConsistencyChecker) {
                this.writeConsistencyChecker.incrementWrittenBytes(size);
            }
            if (null != this.writeConsistencyChecker) {
                this.writeConsistencyChecker.finish();
                if (!this.writeConsistencyChecker.getCheckResult().isSucceeded()) {
                    String exceptionMsg = String.format("Failed to upload the key: %s, error message: %s.", this.key, this.writeConsistencyChecker.getCheckResult().getDescription());
                    throw new IOException(exceptionMsg);
                }
                LOG.info("Upload the key [{}] successfully. check message: {}.", this.key, this.writeConsistencyChecker.getCheckResult().getDescription());
            } else {
                LOG.info("OutputStream for key [{}] upload complete. But it is not checked.", key);
            }
        } else {
            PartETag partETag = null;
            if (this.blockWritten > 0) {
                this.currentBlockId++;
                LOG.info("Upload the last part. key: {}, blockId: [{}], blockWritten: [{}]", this.key, this.currentBlockId, this.blockWritten);
                byte[] md5Hash = this.digest == null ? null : this.digest.digest();
                int size = this.currentBlockBuffer.getByteBuffer().remaining();
                partETag = store.uploadPart(new BufferInputStream(this.currentBlockBuffer), key, uploadId, currentBlockId, currentBlockBuffer.getByteBuffer().remaining(), md5Hash);
                if (null != this.writeConsistencyChecker) {
                    this.writeConsistencyChecker.incrementWrittenBytes(size);
                }
            }
            final List<PartETag> futurePartEtagList = this.waitForFinishPartUploads();
            if (null == futurePartEtagList) {
                throw new IOException("failed to multipart upload to cos, " + "abort it.");
            }
            List<PartETag> tempPartETagList = new LinkedList<PartETag>(futurePartEtagList);
            if (null != partETag) {
                tempPartETagList.add(partETag);
            }
            store.completeMultipartUpload(this.key, this.uploadId, tempPartETagList);
            if (null != this.writeConsistencyChecker) {
                this.writeConsistencyChecker.finish();
                if (!this.writeConsistencyChecker.getCheckResult().isSucceeded()) {
                    String exceptionMsg = String.format("Failed to upload the key: %s, error message: %s.", this.key, this.writeConsistencyChecker.getCheckResult().getDescription());
                    throw new IOException(exceptionMsg);
                }
                LOG.info("Upload the key [{}] successfully. check message: {}.", this.key, this.writeConsistencyChecker.getCheckResult().getDescription());
            } else {
                LOG.info("OutputStream for key [{}] upload complete. But it is not checked.", key);
            }
        }
    } finally {
        BufferPool.getInstance().returnBuffer(this.currentBlockBuffer);
        this.blockWritten = 0;
        this.closed = true;
        this.writeConsistencyChecker = null;
        this.currentBlockBuffer = null;
        this.currentBlockOutputStream = null;
    }
}

Example 8 with PartETag

use of com.qcloud.cos.model.PartETag in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method completeMultipartUpload.

/**
 * complete cos mpu, sometimes return null complete mpu result
 * @param key cos key
 * @param uploadId upload id
 * @param partETagList each part etag list
 * @return result
 * @throws IOException when fail to complete the multipart upload.
 */
public CompleteMultipartUploadResult completeMultipartUpload(String key, String uploadId, List<PartETag> partETagList) throws IOException {
    Collections.sort(partETagList, new Comparator<PartETag>() {

        @Override
        public int compare(PartETag o1, PartETag o2) {
            return o1.getPartNumber() - o2.getPartNumber();
        }
    });
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETagList);
        completeMultipartUploadRequest.setObjectMetadata(objectMetadata);
        return (CompleteMultipartUploadResult) this.callCOSClientWithRetry(completeMultipartUploadRequest);
    } catch (CosServiceException cse) {
        // when first calling with 503 access time out, next retry will 409.
        int statusCode = cse.getStatusCode();
        if (statusCode == 409) {
            // check file whether exist
            FileMetadata fileMetadata = this.queryObjectMetadata(key);
            if (null == fileMetadata) {
                // if file not exist through exception
                handleException(cse, key);
            }
            LOG.warn("Upload the cos key [{}] complete mpu concurrently", key);
        } else {
            // other exception
            String errMsg = String.format("Complete the multipart upload failed. " + "cos service exception, cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, cse.toString());
            handleException(new Exception(errMsg), key);
        }
    } catch (Exception e) {
        String errMsg = String.format("Complete the multipart upload failed. " + "cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, e.toString());
        handleException(new Exception(errMsg), key);
    }
    return null;
}

Example 9 with PartETag

use of com.qcloud.cos.model.PartETag in project hadoop-cos by tencentyun.

the class CosNFSDataOutputStream method uploadPart.

private void uploadPart() throws IOException {
    this.currentBlockOutputStream.flush();
    this.currentBlockOutputStream.close();
    if (this.currentBlockId == 0) {
        uploadId = (store).getUploadId(key);
    }
    this.currentBlockId++;
    LOG.debug("upload part blockId: {}, uploadId: {}.", this.currentBlockId, this.uploadId);
    final byte[] md5Hash = this.digest == null ? null : this.digest.digest();
    ListenableFuture<PartETag> partETagListenableFuture = this.executorService.submit(new Callable<PartETag>() {

        private final CosNByteBuffer buffer = currentBlockBuffer;

        private final String localKey = key;

        private final String localUploadId = uploadId;

        private final int blockId = currentBlockId;

        private final byte[] blockMD5Hash = md5Hash;

        @Override
        public PartETag call() throws Exception {
            try {
                PartETag partETag = (store).uploadPart(new BufferInputStream(this.buffer), this.localKey, this.localUploadId, this.blockId, this.buffer.getByteBuffer().remaining(), this.blockMD5Hash);
                return partETag;
            } finally {
                BufferPool.getInstance().returnBuffer(this.buffer);
            }
        }
    });
    this.partEtagList.add(partETagListenableFuture);
    try {
        this.currentBlockBuffer = BufferPool.getInstance().getBuffer((int) this.blockSize);
    } catch (InterruptedException e) {
        String exceptionMsg = String.format("getting a buffer size: [%d] " + "from the buffer pool occurs an exception.", this.blockSize);
        throw new IOException(exceptionMsg, e);
    }
    if (null != this.digest) {
        this.digest.reset();
        this.currentBlockOutputStream = new DigestOutputStream(new BufferOutputStream(this.currentBlockBuffer), this.digest);
    } else {
        this.currentBlockOutputStream = new BufferOutputStream(this.currentBlockBuffer);
    }
}

Example 10 with PartETag

use of com.qcloud.cos.model.PartETag in project cos-java-sdk-v5 by tencentyun.

the class CompleteMultipartUpload method collectPartETags.

/**
 * Collects the Part ETags for initiating the complete multi-part upload
 * request. This is blocking as it waits until all the upload part threads
 * complete.
 */
private List<PartETag> collectPartETags() {
    final List<PartETag> partETags = new ArrayList<PartETag>();
    partETags.addAll(eTagsBeforeResume);
    for (Future<PartETag> future : futures) {
        try {
            partETags.add(future.get());
        } catch (Exception e) {
            throw new CosClientException("Unable to complete multi-part upload. Individual part upload failed : " + e.getCause().getMessage(), e.getCause());
        }
    }
    return partETags;
}