Examples with UploadPartRequest com.qcloud.cos.model.UploadPartRequest used on opensource projects

Example 6 with UploadPartRequest

use of com.qcloud.cos.model.UploadPartRequest in project cos-java-sdk-v5 by tencentyun.

the class BlindWatermarkDemo method addBlindWatermarkWithMultipart.

public static void addBlindWatermarkWithMultipart(COSClient cosClient) throws FileNotFoundException {
    // bucket名需包含appid
    // api 请参考：https://cloud.tencent.com/document/product/436/46782
    String bucketName = "examplebucket-1250000000";
    String key = "qrcode.png";
    File localFile = new File("E://qrcode.png");
    InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
    InitiateMultipartUploadResult initResult = cosClient.initiateMultipartUpload(request);
    String uploadId = initResult.getUploadId();
    // 上传分块
    List<PartETag> partETags = new LinkedList<>();
    UploadPartRequest uploadPartRequest = new UploadPartRequest();
    uploadPartRequest.setBucketName(bucketName);
    uploadPartRequest.setKey(key);
    uploadPartRequest.setUploadId(uploadId);
    // 设置分块的数据来源输入流
    uploadPartRequest.setInputStream(new FileInputStream(localFile));
    // 设置分块的长度
    // 设置数据长度
    uploadPartRequest.setPartSize(localFile.length());
    // 假设要上传的part编号是10
    uploadPartRequest.setPartNumber(1);
    UploadPartResult uploadPartResult = cosClient.uploadPart(uploadPartRequest);
    PartETag partETag = uploadPartResult.getPartETag();
    partETags.add(partETag);
    // 合并分块并带上图像处理参数
    CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags);
    PicOperations picOperations = new PicOperations();
    picOperations.setIsPicInfo(1);
    List<PicOperations.Rule> ruleList = new LinkedList<>();
    PicOperations.Rule rule1 = new PicOperations.Rule();
    rule1.setBucket(bucketName);
    rule1.setFileId("qrcode-watermark.png");
    rule1.setRule("watermark/3/type/3/text/dGVuY2VudCBjbG91ZA==");
    ruleList.add(rule1);
    picOperations.setRules(ruleList);
    completeMultipartUploadRequest.setPicOperations(picOperations);
    CompleteMultipartUploadResult completeMultipartUploadResult = cosClient.completeMultipartUpload(completeMultipartUploadRequest);
    CIUploadResult ciUploadResult = completeMultipartUploadResult.getCiUploadResult();
    System.out.println(completeMultipartUploadResult.getRequestId());
    System.out.println(ciUploadResult.getOriginalInfo().getEtag());
    for (CIObject ciObject : ciUploadResult.getProcessResults().getObjectList()) {
        System.out.println(ciObject.getLocation());
    }
}

Example 7 with UploadPartRequest

use of com.qcloud.cos.model.UploadPartRequest in project cos-java-sdk-v5 by tencentyun.

the class UploadPartRequestFactory method getNextUploadPartRequest.

public synchronized UploadPartRequest getNextUploadPartRequest() {
    long partSize = Math.min(optimalPartSize, remainingBytes);
    boolean isLastPart = (remainingBytes - partSize <= 0);
    UploadPartRequest req = null;
    if (wrappedStream != null) {
        req = new UploadPartRequest().withBucketName(bucketName).withKey(key).withUploadId(uploadId).withInputStream(new InputSubstream(wrappedStream, 0, partSize, isLastPart)).withPartNumber(partNumber++).withPartSize(partSize).withTrafficLimit(trafficLimit);
    } else {
        req = new UploadPartRequest().withBucketName(bucketName).withKey(key).withUploadId(uploadId).withFile(file).withFileOffset(offset).withPartNumber(partNumber++).withPartSize(partSize).withTrafficLimit(trafficLimit);
    }
    TransferManager.appendMultipartUserAgent(req);
    if (sseCustomerKey != null)
        req.setSSECustomerKey(sseCustomerKey);
    TransferManagerUtils.populateEndpointAddr(origReq, req);
    offset += partSize;
    remainingBytes -= partSize;
    req.setLastPart(isLastPart);
    req.withGeneralProgressListener(origReq.getGeneralProgressListener());
    req.getRequestClientOptions().setReadLimit(origReq.getReadLimit());
    return req;
}

Example 8 with UploadPartRequest

use of com.qcloud.cos.model.UploadPartRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method callCOSClientWithRetry.

// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
    String sdkMethod = "";
    int retryIndex = 1;
    int l5ErrorCodeRetryIndex = 1;
    while (true) {
        try {
            if (request instanceof PutObjectRequest) {
                sdkMethod = "putObject";
                if (((PutObjectRequest) request).getInputStream().markSupported()) {
                    ((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
                }
                return this.cosClient.putObject((PutObjectRequest) request);
            } else if (request instanceof UploadPartRequest) {
                sdkMethod = "uploadPart";
                if (((UploadPartRequest) request).getInputStream().markSupported()) {
                    ((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
                }
                return this.cosClient.uploadPart((UploadPartRequest) request);
            } else if (request instanceof CopyPartRequest) {
                sdkMethod = "copyPartRequest";
                return this.cosClient.copyPart((CopyPartRequest) request);
            } else if (request instanceof HeadBucketRequest) {
                // use for checking bucket type
                sdkMethod = "headBucket";
                return this.cosClient.headBucket((HeadBucketRequest) request);
            } else if (request instanceof RenameRequest) {
                sdkMethod = "rename";
                this.cosClient.rename((RenameRequest) request);
                return new Object();
            } else if (request instanceof GetObjectMetadataRequest) {
                sdkMethod = "queryObjectMeta";
                return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
            } else if (request instanceof DeleteObjectRequest) {
                sdkMethod = "deleteObject";
                this.cosClient.deleteObject((DeleteObjectRequest) request);
                return new Object();
            } else if (request instanceof CopyObjectRequest) {
                sdkMethod = "copyFile";
                return this.cosClient.copyObject((CopyObjectRequest) request);
            } else if (request instanceof GetObjectRequest) {
                sdkMethod = "getObject";
                return this.cosClient.getObject((GetObjectRequest) request);
            } else if (request instanceof ListObjectsRequest) {
                sdkMethod = "listObjects";
                return this.cosClient.listObjects((ListObjectsRequest) request);
            } else if (request instanceof InitiateMultipartUploadRequest) {
                sdkMethod = "initiateMultipartUpload";
                return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
            } else if (request instanceof CompleteMultipartUploadRequest) {
                sdkMethod = "completeMultipartUpload";
                return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
            } else if (request instanceof AbortMultipartUploadRequest) {
                sdkMethod = "abortMultipartUpload";
                this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
                return new Object();
            } else {
                throw new IOException("no such method");
            }
        } catch (ResponseNotCompleteException nce) {
            if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
                String key = ((CompleteMultipartUploadRequest) request).getKey();
                FileMetadata fileMetadata = this.queryObjectMetadata(key);
                if (null == fileMetadata) {
                    // if file not exist must throw the exception.
                    handleException(nce, key);
                }
                LOG.warn("Complete mpu resp not complete key [{}]", key);
                // todo: some other double check after cgi unified the ctime of mpu.
                return new CompleteMultipartUploadResult();
            } else {
                throw new IOException(nce);
            }
        } catch (CosServiceException cse) {
            String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
            int statusCode = cse.getStatusCode();
            String errorCode = cse.getErrorCode();
            LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
            // 对5xx错误进行重试
            if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
                if (retryIndex <= this.maxRetryTimes) {
                    LOG.info(errMsg, cse);
                    ++retryIndex;
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
                // complete mpu error code might be in body when status code is 200
                // double check to head object only works in big data job case which key is not same.
                String key = ((CompleteMultipartUploadRequest) request).getKey();
                FileMetadata fileMetadata = this.queryObjectMetadata(key);
                if (null != fileMetadata) {
                    // if file exist direct return.
                    LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
                    return new CompleteMultipartUploadResult();
                }
                // here same like the copy request not setting the interval sleep for now
                if (retryIndex <= this.maxRetryTimes) {
                    LOG.info(errMsg, cse);
                    ++retryIndex;
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else if (statusCode / 100 == 5) {
                if (retryIndex <= this.maxRetryTimes) {
                    if (statusCode == 503) {
                        if (useL5Id) {
                            if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
                                // L5上报，进行重试
                                l5EndpointResolver.updateRouteResult(-1);
                                l5ErrorCodeRetryIndex = 1;
                            } else {
                                l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
                            }
                        }
                    }
                    LOG.info(errMsg, cse);
                    long sleepLeast = retryIndex * 300L;
                    long sleepBound = retryIndex * 500L;
                    try {
                        if (request instanceof PutObjectRequest) {
                            LOG.info("Try to reset the put object request input stream.");
                            if (((PutObjectRequest) request).getInputStream().markSupported()) {
                                ((PutObjectRequest) request).getInputStream().reset();
                            } else {
                                LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
                                throw cse;
                            }
                        }
                        if (request instanceof UploadPartRequest) {
                            LOG.info("Try to reset the upload part request input stream.");
                            if (((UploadPartRequest) request).getInputStream().markSupported()) {
                                ((UploadPartRequest) request).getInputStream().reset();
                            } else {
                                LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
                                throw cse;
                            }
                        }
                        // if direct retry may occur 403 not found the upload id.
                        if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
                            String key = ((CompleteMultipartUploadRequest) request).getKey();
                            FileMetadata fileMetadata = this.queryObjectMetadata(key);
                            if (null != fileMetadata) {
                                // if file exist direct return.
                                LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
                                return new CompleteMultipartUploadResult();
                            }
                        }
                        Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
                        ++retryIndex;
                    } catch (InterruptedException e) {
                        throw new IOException(e.toString());
                    }
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else {
                throw cse;
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
}

Example 9 with UploadPartRequest

use of com.qcloud.cos.model.UploadPartRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method uploadPart.

@Override
public PartETag uploadPart(InputStream inputStream, String key, String uploadId, int partNum, long partSize, byte[] md5Hash) throws IOException {
    LOG.debug("Upload the part to the cos key [{}]. upload id: {}, part number: {}, part size: {}", key, uploadId, partNum, partSize);
    ObjectMetadata objectMetadata = new ObjectMetadata();
    if (crc32cEnabled) {
        objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
    }
    UploadPartRequest uploadPartRequest = new UploadPartRequest();
    uploadPartRequest.setBucketName(this.bucketName);
    uploadPartRequest.setUploadId(uploadId);
    uploadPartRequest.setInputStream(inputStream);
    uploadPartRequest.setPartNumber(partNum);
    uploadPartRequest.setPartSize(partSize);
    uploadPartRequest.setObjectMetadata(objectMetadata);
    if (null != md5Hash) {
        uploadPartRequest.setMd5Digest(Base64.encodeAsString(md5Hash));
    }
    uploadPartRequest.setKey(key);
    if (this.trafficLimit >= 0) {
        uploadPartRequest.setTrafficLimit(this.trafficLimit);
    }
    this.setEncryptionMetadata(uploadPartRequest, objectMetadata);
    try {
        UploadPartResult uploadPartResult = (UploadPartResult) callCOSClientWithRetry(uploadPartRequest);
        return uploadPartResult.getPartETag();
    } catch (Exception e) {
        String errMsg = String.format("The current thread:%d, " + "cos key: %s, upload id: %s, part num: %d, " + "exception: %s", Thread.currentThread().getId(), key, uploadId, partNum, e);
        handleException(new Exception(errMsg), key);
    }
    return null;
}

Example 10 with UploadPartRequest

use of com.qcloud.cos.model.UploadPartRequest in project cos-java-sdk-v5 by tencentyun.

the class MultipartUploadDemo method UploadPartDemo.

// 分块上传(上传某一个分片的数据)
public static List<PartETag> UploadPartDemo(String uploadId) {
    // 1 初始化用户身份信息(secretId, secretKey)
    COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
    // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
    ClientConfig clientConfig = new ClientConfig(new Region("ap-guangzhou"));
    // 3 生成cos客户端
    COSClient cosclient = new COSClient(cred, clientConfig);
    // bucket名需包含appid
    String bucketName = "mybucket-1251668577";
    String key = "aaa/bbb.txt";
    // uploadid(通过initiateMultipartUpload或者ListMultipartUploads获取)
    boolean userTrafficLimit = false;
    List<PartETag> partETags = new LinkedList<>();
    // 生成要上传的数据, 这里初始化一个10M的数据
    for (int i = 0; i < 10; i++) {
        byte[] data = new byte[1024 * 1024];
        UploadPartRequest uploadPartRequest = new UploadPartRequest();
        uploadPartRequest.setBucketName(bucketName);
        uploadPartRequest.setKey(key);
        uploadPartRequest.setUploadId(uploadId);
        // 设置分块的数据来源输入流
        uploadPartRequest.setInputStream(new ByteArrayInputStream(data));
        // 设置分块的长度
        // 设置数据长度
        uploadPartRequest.setPartSize(data.length);
        // 假设要上传的part编号是10
        uploadPartRequest.setPartNumber(i + 1);
        if (userTrafficLimit) {
            uploadPartRequest.setTrafficLimit(8 * 1024 * 1024);
        }
        try {
            UploadPartResult uploadPartResult = cosclient.uploadPart(uploadPartRequest);
            PartETag partETag = uploadPartResult.getPartETag();
            partETags.add(partETag);
            String crc64 = uploadPartResult.getCrc64Ecma();
        } catch (CosServiceException e) {
            throw e;
        } catch (CosClientException e) {
            throw e;
        }
    }
    cosclient.shutdown();
    return partETags;
}