Examples with CopyObjectRequest com.qcloud.cos.model.CopyObjectRequest used on opensource projects

Example 11 with CopyObjectRequest

use of com.qcloud.cos.model.CopyObjectRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method copy.

@Override
public void copy(String srcKey, String dstKey) throws IOException {
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, srcKey, bucketName, dstKey);
        FileMetadata sourceFileMetadata = this.retrieveMetadata(srcKey);
        if (null != sourceFileMetadata.getStorageClass()) {
            copyObjectRequest.setStorageClass(sourceFileMetadata.getStorageClass());
        }
        copyObjectRequest.setNewObjectMetadata(objectMetadata);
        this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
        copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
        callCOSClientWithRetry(copyObjectRequest);
    } catch (Exception e) {
        String errMsg = String.format("Copy the object failed, src cos key: %s, dst cos key: %s, " + "exception: %s", srcKey, dstKey, e);
        handleException(new Exception(errMsg), srcKey);
    }
}

Example 12 with CopyObjectRequest

use of com.qcloud.cos.model.CopyObjectRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method normalBucketRename.

public void normalBucketRename(String srcKey, String dstKey) throws IOException {
    LOG.debug("Rename normal bucket key, the source cos key [{}] to the dest cos key [{}].", srcKey, dstKey);
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, srcKey, bucketName, dstKey);
        // get the storage class of the source file
        FileMetadata sourceFileMetadata = this.retrieveMetadata(srcKey);
        if (null != sourceFileMetadata.getStorageClass()) {
            copyObjectRequest.setStorageClass(sourceFileMetadata.getStorageClass());
        }
        copyObjectRequest.setNewObjectMetadata(objectMetadata);
        this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
        copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
        callCOSClientWithRetry(copyObjectRequest);
        DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(bucketName, srcKey);
        callCOSClientWithRetry(deleteObjectRequest);
    } catch (Exception e) {
        String errMsg = String.format("Rename object failed, normal bucket, source cos key: %s, dest cos key: %s, " + "exception: %s", srcKey, dstKey, e);
        handleException(new Exception(errMsg), srcKey);
    }
}

Example 13 with CopyObjectRequest

use of com.qcloud.cos.model.CopyObjectRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method callCOSClientWithRetry.

// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
    String sdkMethod = "";
    int retryIndex = 1;
    int l5ErrorCodeRetryIndex = 1;
    while (true) {
        try {
            if (request instanceof PutObjectRequest) {
                sdkMethod = "putObject";
                if (((PutObjectRequest) request).getInputStream().markSupported()) {
                    ((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
                }
                return this.cosClient.putObject((PutObjectRequest) request);
            } else if (request instanceof UploadPartRequest) {
                sdkMethod = "uploadPart";
                if (((UploadPartRequest) request).getInputStream().markSupported()) {
                    ((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
                }
                return this.cosClient.uploadPart((UploadPartRequest) request);
            } else if (request instanceof CopyPartRequest) {
                sdkMethod = "copyPartRequest";
                return this.cosClient.copyPart((CopyPartRequest) request);
            } else if (request instanceof HeadBucketRequest) {
                // use for checking bucket type
                sdkMethod = "headBucket";
                return this.cosClient.headBucket((HeadBucketRequest) request);
            } else if (request instanceof RenameRequest) {
                sdkMethod = "rename";
                this.cosClient.rename((RenameRequest) request);
                return new Object();
            } else if (request instanceof GetObjectMetadataRequest) {
                sdkMethod = "queryObjectMeta";
                return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
            } else if (request instanceof DeleteObjectRequest) {
                sdkMethod = "deleteObject";
                this.cosClient.deleteObject((DeleteObjectRequest) request);
                return new Object();
            } else if (request instanceof CopyObjectRequest) {
                sdkMethod = "copyFile";
                return this.cosClient.copyObject((CopyObjectRequest) request);
            } else if (request instanceof GetObjectRequest) {
                sdkMethod = "getObject";
                return this.cosClient.getObject((GetObjectRequest) request);
            } else if (request instanceof ListObjectsRequest) {
                sdkMethod = "listObjects";
                return this.cosClient.listObjects((ListObjectsRequest) request);
            } else if (request instanceof InitiateMultipartUploadRequest) {
                sdkMethod = "initiateMultipartUpload";
                return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
            } else if (request instanceof CompleteMultipartUploadRequest) {
                sdkMethod = "completeMultipartUpload";
                return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
            } else if (request instanceof AbortMultipartUploadRequest) {
                sdkMethod = "abortMultipartUpload";
                this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
                return new Object();
            } else {
                throw new IOException("no such method");
            }
        } catch (ResponseNotCompleteException nce) {
            if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
                String key = ((CompleteMultipartUploadRequest) request).getKey();
                FileMetadata fileMetadata = this.queryObjectMetadata(key);
                if (null == fileMetadata) {
                    // if file not exist must throw the exception.
                    handleException(nce, key);
                }
                LOG.warn("Complete mpu resp not complete key [{}]", key);
                // todo: some other double check after cgi unified the ctime of mpu.
                return new CompleteMultipartUploadResult();
            } else {
                throw new IOException(nce);
            }
        } catch (CosServiceException cse) {
            String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
            int statusCode = cse.getStatusCode();
            String errorCode = cse.getErrorCode();
            LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
            // 对5xx错误进行重试
            if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
                if (retryIndex <= this.maxRetryTimes) {
                    LOG.info(errMsg, cse);
                    ++retryIndex;
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
                // complete mpu error code might be in body when status code is 200
                // double check to head object only works in big data job case which key is not same.
                String key = ((CompleteMultipartUploadRequest) request).getKey();
                FileMetadata fileMetadata = this.queryObjectMetadata(key);
                if (null != fileMetadata) {
                    // if file exist direct return.
                    LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
                    return new CompleteMultipartUploadResult();
                }
                // here same like the copy request not setting the interval sleep for now
                if (retryIndex <= this.maxRetryTimes) {
                    LOG.info(errMsg, cse);
                    ++retryIndex;
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else if (statusCode / 100 == 5) {
                if (retryIndex <= this.maxRetryTimes) {
                    if (statusCode == 503) {
                        if (useL5Id) {
                            if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
                                // L5上报，进行重试
                                l5EndpointResolver.updateRouteResult(-1);
                                l5ErrorCodeRetryIndex = 1;
                            } else {
                                l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
                            }
                        }
                    }
                    LOG.info(errMsg, cse);
                    long sleepLeast = retryIndex * 300L;
                    long sleepBound = retryIndex * 500L;
                    try {
                        if (request instanceof PutObjectRequest) {
                            LOG.info("Try to reset the put object request input stream.");
                            if (((PutObjectRequest) request).getInputStream().markSupported()) {
                                ((PutObjectRequest) request).getInputStream().reset();
                            } else {
                                LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
                                throw cse;
                            }
                        }
                        if (request instanceof UploadPartRequest) {
                            LOG.info("Try to reset the upload part request input stream.");
                            if (((UploadPartRequest) request).getInputStream().markSupported()) {
                                ((UploadPartRequest) request).getInputStream().reset();
                            } else {
                                LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
                                throw cse;
                            }
                        }
                        // if direct retry may occur 403 not found the upload id.
                        if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
                            String key = ((CompleteMultipartUploadRequest) request).getKey();
                            FileMetadata fileMetadata = this.queryObjectMetadata(key);
                            if (null != fileMetadata) {
                                // if file exist direct return.
                                LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
                                return new CompleteMultipartUploadResult();
                            }
                        }
                        Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
                        ++retryIndex;
                    } catch (InterruptedException e) {
                        throw new IOException(e.toString());
                    }
                } else {
                    LOG.error(errMsg, cse);
                    throw new IOException(errMsg);
                }
            } else {
                throw cse;
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
}

Example 14 with CopyObjectRequest

use of com.qcloud.cos.model.CopyObjectRequest in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method storeAttribute.

private void storeAttribute(String key, String attribute, byte[] value, boolean deleted) throws IOException {
    if (deleted) {
        LOG.debug("Delete the extended attribute. cos key: {}, attribute: {}.", key, attribute);
    }
    if (null != value && !deleted) {
        LOG.debug("Store the extended attribute. cos key: {}, attribute: {}, value: {}.", key, attribute, new String(value, StandardCharsets.UTF_8));
    }
    if (null == value && !deleted) {
        throw new IOException("The attribute value to be set can not be null.");
    }
    GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
    this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
    ObjectMetadata objectMetadata = null;
    try {
        objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
    } catch (CosServiceException e) {
        if (e.getStatusCode() != 404) {
            String errorMessage = String.format("Retrieve the file metadata failed. " + "cos key: %s, exception: %s.", key, e);
            handleException(new Exception(errorMessage), key);
        }
    }
    if (null != objectMetadata) {
        Map<String, String> userMetadata = objectMetadata.getUserMetadata();
        if (deleted) {
            if (null != userMetadata) {
                userMetadata.remove(ensureValidAttributeName(attribute));
            } else {
                return;
            }
        } else {
            if (null == userMetadata) {
                userMetadata = new HashMap<>();
            }
            CosNXAttr cosNXAttr = new CosNXAttr();
            cosNXAttr.setName(attribute);
            cosNXAttr.setValue(new String(value, CosNFileSystem.METADATA_ENCODING));
            String xAttrJsonStr = Jackson.toJsonString(cosNXAttr);
            userMetadata.put(ensureValidAttributeName(XATTR_PREFIX + attribute), Base64.encodeAsString(xAttrJsonStr.getBytes(StandardCharsets.UTF_8)));
        }
        objectMetadata.setUserMetadata(userMetadata);
        // 构造原地copy请求来设置用户自定义属性
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
        if (null != objectMetadata.getStorageClass()) {
            copyObjectRequest.setStorageClass(objectMetadata.getStorageClass());
        }
        copyObjectRequest.setNewObjectMetadata(objectMetadata);
        copyObjectRequest.setRedirectLocation("Replaced");
        this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
        copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
        try {
            callCOSClientWithRetry(copyObjectRequest);
        } catch (Exception e) {
            String errMsg = String.format("Failed to modify the user-defined attributes. " + "cos key: %s, attribute: %s, exception: %s.", key, attribute, e);
            handleException(new Exception(errMsg), key);
        }
    }
}

Example 15 with CopyObjectRequest

use of com.qcloud.cos.model.CopyObjectRequest in project cos-java-sdk-v5 by tencentyun.

the class PutObjectCopyTest method testCopySameRegionDiffSize.

private void testCopySameRegionDiffSize(long fileSize, ObjectMetadata newObjectMetaData) throws IOException {
    if (!judgeUserInfoValid()) {
        return;
    }
    File localFile = buildTestFile(fileSize);
    String srcEtag = Md5Utils.md5Hex(localFile);
    String srcKey = String.format("ut/src_len_%d.txt", fileSize);
    String destKey = String.format("ut/dest_len_%d.txt", fileSize);
    try {
        PutObjectResult putObjectResult = putObjectFromLocalFile(localFile, srcKey);
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, destKey);
        copyObjectRequest.setSourceVersionId(putObjectResult.getVersionId());
        copyObjectRequest.setStorageClass(StorageClass.Standard_IA);
        if (newObjectMetaData != null) {
            copyObjectRequest.setNewObjectMetadata(newObjectMetaData);
        }
        CopyObjectResult copyObjectResult = cosclient.copyObject(copyObjectRequest);
        assertNotNull(copyObjectResult.getRequestId());
        assertNotNull(copyObjectResult.getDateStr());
        // assertEquals(srcEtag, copyObjectResult.getETag());
        headSimpleObject(srcKey, fileSize, srcEtag);
    /*
            ObjectMetadata destObjectMetadata = headSimpleObject(destKey, fileSize, srcEtag);
            if (newObjectMetaData != null) {
                checkMetaData(newObjectMetaData, destObjectMetadata);
            }
            */
    } finally {
        // delete file on cos
        clearObject(srcKey);
        clearObject(destKey);
        // delete local file
        if (localFile.exists()) {
            assertTrue(localFile.delete());
        }
    }
}