Search in sources :

Example 46 with CosServiceException

use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method completeMultipartUpload.

/**
 * complete cos mpu, sometimes return null complete mpu result
 * @param key cos key
 * @param uploadId upload id
 * @param partETagList each part etag list
 * @return result
 * @throws IOException when fail to complete the multipart upload.
 */
public CompleteMultipartUploadResult completeMultipartUpload(String key, String uploadId, List<PartETag> partETagList) throws IOException {
    Collections.sort(partETagList, new Comparator<PartETag>() {

        @Override
        public int compare(PartETag o1, PartETag o2) {
            return o1.getPartNumber() - o2.getPartNumber();
        }
    });
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETagList);
        completeMultipartUploadRequest.setObjectMetadata(objectMetadata);
        return (CompleteMultipartUploadResult) this.callCOSClientWithRetry(completeMultipartUploadRequest);
    } catch (CosServiceException cse) {
        // when first calling with 503 access time out, next retry will 409.
        int statusCode = cse.getStatusCode();
        if (statusCode == 409) {
            // check file whether exist
            FileMetadata fileMetadata = this.queryObjectMetadata(key);
            if (null == fileMetadata) {
                // if file not exist through exception
                handleException(cse, key);
            }
            LOG.warn("Upload the cos key [{}] complete mpu concurrently", key);
        } else {
            // other exception
            String errMsg = String.format("Complete the multipart upload failed. " + "cos service exception, cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, cse.toString());
            handleException(new Exception(errMsg), key);
        }
    } catch (Exception e) {
        String errMsg = String.format("Complete the multipart upload failed. " + "cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, e.toString());
        handleException(new Exception(errMsg), key);
    }
    return null;
}
Also used : CosServiceException(com.qcloud.cos.exception.CosServiceException) CompleteMultipartUploadResult(com.qcloud.cos.model.CompleteMultipartUploadResult) ObjectMetadata(com.qcloud.cos.model.ObjectMetadata) PartETag(com.qcloud.cos.model.PartETag) ResponseNotCompleteException(com.qcloud.cos.exception.ResponseNotCompleteException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) IOException(java.io.IOException) CompleteMultipartUploadRequest(com.qcloud.cos.model.CompleteMultipartUploadRequest)

Example 47 with CosServiceException

use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method queryObjectMetadata.

private FileMetadata queryObjectMetadata(String key, CosNResultInfo info) throws IOException {
    LOG.debug("Query Object metadata. cos key: {}.", key);
    GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
    this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
    try {
        ObjectMetadata objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
        long mtime = 0;
        long fileSize;
        if (objectMetadata.getLastModified() != null) {
            mtime = objectMetadata.getLastModified().getTime();
        }
        fileSize = objectMetadata.getContentLength();
        String ETag = objectMetadata.getETag();
        String crc64ecm = objectMetadata.getCrc64Ecma();
        String crc32cm = (String) objectMetadata.getRawMetadataValue(Constants.CRC32C_RESP_HEADER);
        String versionId = objectMetadata.getVersionId();
        Map<String, byte[]> userMetadata = null;
        if (objectMetadata.getUserMetadata() != null) {
            userMetadata = new HashMap<>();
            for (Map.Entry<String, String> userMetadataEntry : objectMetadata.getUserMetadata().entrySet()) {
                if (userMetadataEntry.getKey().startsWith(ensureValidAttributeName(XATTR_PREFIX))) {
                    String xAttrJsonStr = new String(Base64.decode(userMetadataEntry.getValue()), StandardCharsets.UTF_8);
                    CosNXAttr cosNXAttr;
                    try {
                        cosNXAttr = Jackson.fromJsonString(xAttrJsonStr, CosNXAttr.class);
                    } catch (CosClientException e) {
                        LOG.warn("Parse the xAttr failed. name: {}, XAttJsonStr: {}.", userMetadataEntry.getKey(), xAttrJsonStr);
                        // Skip
                        continue;
                    }
                    if (null != cosNXAttr) {
                        userMetadata.put(cosNXAttr.getName(), cosNXAttr.getValue().getBytes(CosNFileSystem.METADATA_ENCODING));
                    }
                }
            }
        }
        boolean isFile = true;
        if (isPosixBucket) {
            if (objectMetadata.isFileModeDir() || key.equals(CosNFileSystem.PATH_DELIMITER)) {
                isFile = false;
            }
        } else {
            isFile = !key.endsWith(CosNFileSystem.PATH_DELIMITER);
        }
        FileMetadata fileMetadata = new FileMetadata(key, fileSize, mtime, isFile, ETag, crc64ecm, crc32cm, versionId, objectMetadata.getStorageClass(), userMetadata);
        // record the last request result info
        if (info != null) {
            info.setRequestID(objectMetadata.getRequestId());
        }
        LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key, objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
        return fileMetadata;
    } catch (CosServiceException e) {
        if (info != null) {
            info.setRequestID(e.getRequestId());
        }
        if (e.getStatusCode() != 404) {
            String errorMsg = String.format("Retrieve the file metadata file failure. " + "cos key: %s, exception: %s", key, e);
            handleException(new Exception(errorMsg), key);
        }
    }
    return null;
}
Also used : CosClientException(com.qcloud.cos.exception.CosClientException) ResponseNotCompleteException(com.qcloud.cos.exception.ResponseNotCompleteException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) IOException(java.io.IOException) GetObjectMetadataRequest(com.qcloud.cos.model.GetObjectMetadataRequest) CosServiceException(com.qcloud.cos.exception.CosServiceException) ObjectMetadata(com.qcloud.cos.model.ObjectMetadata) Map(java.util.Map) HashMap(java.util.HashMap)

Example 48 with CosServiceException

use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method storeAttribute.

private void storeAttribute(String key, String attribute, byte[] value, boolean deleted) throws IOException {
    if (deleted) {
        LOG.debug("Delete the extended attribute. cos key: {}, attribute: {}.", key, attribute);
    }
    if (null != value && !deleted) {
        LOG.debug("Store the extended attribute. cos key: {}, attribute: {}, value: {}.", key, attribute, new String(value, StandardCharsets.UTF_8));
    }
    if (null == value && !deleted) {
        throw new IOException("The attribute value to be set can not be null.");
    }
    GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
    this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
    ObjectMetadata objectMetadata = null;
    try {
        objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
    } catch (CosServiceException e) {
        if (e.getStatusCode() != 404) {
            String errorMessage = String.format("Retrieve the file metadata failed. " + "cos key: %s, exception: %s.", key, e);
            handleException(new Exception(errorMessage), key);
        }
    }
    if (null != objectMetadata) {
        Map<String, String> userMetadata = objectMetadata.getUserMetadata();
        if (deleted) {
            if (null != userMetadata) {
                userMetadata.remove(ensureValidAttributeName(attribute));
            } else {
                return;
            }
        } else {
            if (null == userMetadata) {
                userMetadata = new HashMap<>();
            }
            CosNXAttr cosNXAttr = new CosNXAttr();
            cosNXAttr.setName(attribute);
            cosNXAttr.setValue(new String(value, CosNFileSystem.METADATA_ENCODING));
            String xAttrJsonStr = Jackson.toJsonString(cosNXAttr);
            userMetadata.put(ensureValidAttributeName(XATTR_PREFIX + attribute), Base64.encodeAsString(xAttrJsonStr.getBytes(StandardCharsets.UTF_8)));
        }
        objectMetadata.setUserMetadata(userMetadata);
        // 构造原地copy请求来设置用户自定义属性
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
        if (null != objectMetadata.getStorageClass()) {
            copyObjectRequest.setStorageClass(objectMetadata.getStorageClass());
        }
        copyObjectRequest.setNewObjectMetadata(objectMetadata);
        copyObjectRequest.setRedirectLocation("Replaced");
        this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
        copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
        try {
            callCOSClientWithRetry(copyObjectRequest);
        } catch (Exception e) {
            String errMsg = String.format("Failed to modify the user-defined attributes. " + "cos key: %s, attribute: %s, exception: %s.", key, attribute, e);
            handleException(new Exception(errMsg), key);
        }
    }
}
Also used : CopyObjectRequest(com.qcloud.cos.model.CopyObjectRequest) GetObjectMetadataRequest(com.qcloud.cos.model.GetObjectMetadataRequest) CosServiceException(com.qcloud.cos.exception.CosServiceException) IOException(java.io.IOException) ObjectMetadata(com.qcloud.cos.model.ObjectMetadata) ResponseNotCompleteException(com.qcloud.cos.exception.ResponseNotCompleteException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) IOException(java.io.IOException)

Example 49 with CosServiceException

use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method storeFileWithRetry.

private void storeFileWithRetry(String key, InputStream inputStream, byte[] md5Hash, long length) throws IOException {
    try {
        ObjectMetadata objectMetadata = new ObjectMetadata();
        if (null != md5Hash) {
            objectMetadata.setContentMD5(Base64.encodeAsString(md5Hash));
        }
        objectMetadata.setContentLength(length);
        if (crc32cEnabled) {
            objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
        }
        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, objectMetadata);
        if (null != this.storageClass) {
            putObjectRequest.setStorageClass(this.storageClass);
        }
        if (this.trafficLimit >= 0) {
            putObjectRequest.setTrafficLimit(this.trafficLimit);
        }
        this.setEncryptionMetadata(putObjectRequest, objectMetadata);
        PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
        LOG.debug("Store the file successfully. cos key: {}, ETag: {}.", key, putObjectResult.getETag());
    } catch (CosServiceException cse) {
        // 避免并发上传的问题
        int statusCode = cse.getStatusCode();
        if (statusCode == 409) {
            // Check一下这个文件是否已经存在
            FileMetadata fileMetadata = this.queryObjectMetadata(key);
            if (null == fileMetadata) {
                // 如果文件不存在,则需要抛出异常
                handleException(cse, key);
            }
            LOG.warn("Upload the cos key [{}] concurrently", key);
        } else {
            // 其他错误都要抛出来
            handleException(cse, key);
        }
    } catch (Exception e) {
        String errMsg = String.format("Store the file failed, cos key: %s, exception: %s.", key, e);
        handleException(new Exception(errMsg), key);
    }
}
Also used : CosServiceException(com.qcloud.cos.exception.CosServiceException) PutObjectResult(com.qcloud.cos.model.PutObjectResult) ObjectMetadata(com.qcloud.cos.model.ObjectMetadata) PutObjectRequest(com.qcloud.cos.model.PutObjectRequest) ResponseNotCompleteException(com.qcloud.cos.exception.ResponseNotCompleteException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) IOException(java.io.IOException)

Example 50 with CosServiceException

use of com.qcloud.cos.exception.CosServiceException in project cos-java-sdk-v5 by tencentyun.

the class COSClient method uploadObjectInternal.

protected <UploadObjectRequest extends PutObjectRequest> ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest uploadObjectRequest) throws CosClientException, CosServiceException {
    rejectNull(uploadObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object");
    rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when uploading an object");
    final File file = uploadObjectRequest.getFile();
    final InputStream isOrig = uploadObjectRequest.getInputStream();
    final String bucketName = uploadObjectRequest.getBucketName();
    final String key = uploadObjectRequest.getKey();
    ObjectMetadata metadata = uploadObjectRequest.getMetadata();
    InputStream input = isOrig;
    if (metadata == null)
        metadata = new ObjectMetadata();
    rejectNull(bucketName, "The bucket name parameter must be specified when uploading an object");
    rejectNull(key, "The key parameter must be specified when uploading an object");
    // information from it to auto-configure a few options
    if (file == null) {
        // unlimited mark-and-reset
        if (input != null)
            input = ReleasableInputStream.wrap(input);
    } else {
        // Always set the content length, even if it's already set
        metadata.setContentLength(file.length());
        final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
        if (file.length() > maxAllowdSingleFileSize) {
            throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + file.length() + ", please use transferManager to upload big file!");
        }
        final boolean calculateMD5 = metadata.getContentMD5() == null;
        if (calculateMD5 && !skipMd5CheckStrategy.skipServerSideValidation(uploadObjectRequest)) {
            try {
                String contentMd5_b64 = Md5Utils.md5AsBase64(file);
                metadata.setContentMD5(contentMd5_b64);
            } catch (Exception e) {
                throw new CosClientException("Unable to calculate MD5 hash: " + e.getMessage(), e);
            }
        }
        input = ResettableInputStream.newResettableInputStream(file, "Unable to find file to upload");
    }
    final ObjectMetadata returnedMetadata;
    MD5DigestCalculatingInputStream md5DigestStream = null;
    try {
        CosHttpRequest<UploadObjectRequest> request = null;
        if (uploadMode.equals(UploadMode.PUT_OBJECT)) {
            request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.PUT);
        } else if (uploadMode.equals(UploadMode.APPEND_OBJECT)) {
            request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.POST);
            AppendObjectRequest appendObjectRequest = (AppendObjectRequest) uploadObjectRequest;
            String positionStr = String.valueOf(appendObjectRequest.getPosition());
            request.addParameter("append", null);
            request.addParameter("position", positionStr);
        }
        if (uploadObjectRequest.getAccessControlList() != null) {
            addAclHeaders(request, uploadObjectRequest.getAccessControlList());
        } else if (uploadObjectRequest.getCannedAcl() != null) {
            request.addHeader(Headers.COS_CANNED_ACL, uploadObjectRequest.getCannedAcl().toString());
        }
        if (uploadObjectRequest.getStorageClass() != null) {
            request.addHeader(Headers.STORAGE_CLASS, uploadObjectRequest.getStorageClass());
        }
        if (uploadObjectRequest.getRedirectLocation() != null) {
            request.addHeader(Headers.REDIRECT_LOCATION, uploadObjectRequest.getRedirectLocation());
            if (input == null) {
                input = new ByteArrayInputStream(new byte[0]);
            }
        }
        // Populate the SSE-C parameters to the request header
        populateSSE_C(request, uploadObjectRequest.getSSECustomerKey());
        // Populate the SSE KMS parameters to the request header
        populateSSE_KMS(request, uploadObjectRequest.getSSECOSKeyManagementParams());
        // Populate the traffic limit parameter to the request header
        populateTrafficLimit(request, uploadObjectRequest.getTrafficLimit());
        // Use internal interface to differentiate 0 from unset.
        final Long contentLength = (Long) metadata.getRawMetadataValue(Headers.CONTENT_LENGTH);
        if (contentLength == null) {
            /*
                 * There's nothing we can do except for let the HTTP client buffer the input stream
                 * contents if the caller doesn't tell us how much data to expect in a stream since
                 * we have to explicitly tell how much we're sending before we start sending any of
                 * it.
                 */
            log.warn("No content length specified for stream data.  " + "Stream contents will be buffered in memory and could result in " + "out of memory errors.");
        } else {
            final long expectedLength = contentLength.longValue();
            final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
            if (expectedLength > maxAllowdSingleFileSize) {
                throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + expectedLength + ", please use transferManager to upload big file!");
            }
            if (expectedLength >= 0) {
                // Performs length check on the underlying data stream.
                // For COS encryption client, the underlying data stream here
                // refers to the cipher-text data stream (ie not the underlying
                // plain-text data stream which in turn may have been wrapped
                // with it's own length check input stream.)
                LengthCheckInputStream lcis = new // expected
                LengthCheckInputStream(// expected
                input, // expected
                expectedLength, // uploaded
                EXCLUDE_SKIPPED_BYTES);
                input = lcis;
            }
        }
        if (metadata.getContentMD5() == null && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadObjectRequest)) {
            /*
                 * If the user hasn't set the content MD5, then we don't want to buffer the whole
                 * stream in memory just to calculate it. Instead, we can calculate it on the fly
                 * and validate it with the returned ETag from the object upload.
                 */
            input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
        }
        populateRequestMetadata(request, metadata);
        request.setContent(input);
        try {
            if (uploadObjectRequest.getPicOperations() != null) {
                request.addHeader(Headers.PIC_OPERATIONS, Jackson.toJsonString(uploadObjectRequest.getPicOperations()));
                returnedMetadata = invoke(request, new ResponseHeaderHandlerChain<ObjectMetadata>(new Unmarshallers.ImagePersistenceUnmarshaller(), new CosMetadataResponseHandler()));
            } else {
                returnedMetadata = invoke(request, new CosMetadataResponseHandler());
            }
        } catch (Throwable t) {
            throw Throwables.failure(t);
        }
    } finally {
        CosDataSource.Utils.cleanupDataSource(uploadObjectRequest, file, isOrig, input, log);
    }
    String contentMd5 = metadata.getContentMD5();
    if (md5DigestStream != null) {
        contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
    }
    final String etag = returnedMetadata.getETag();
    if (contentMd5 != null && uploadMode.equals(UploadMode.PUT_OBJECT) && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
        byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
        byte[] serverSideHash = null;
        try {
            serverSideHash = BinaryUtils.fromHex(etag);
        } catch (DecoderException e) {
            throw new CosClientException("Unable to verify integrity of data upload.  " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS .  " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
        }
        if (!Arrays.equals(clientSideHash, serverSideHash)) {
            throw new CosClientException("Unable to verify integrity of data upload.  " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS .  " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
        }
    }
    return returnedMetadata;
}
Also used : LengthCheckInputStream(com.qcloud.cos.internal.LengthCheckInputStream) Unmarshallers(com.qcloud.cos.internal.Unmarshallers) DigestValidationInputStream(com.qcloud.cos.internal.DigestValidationInputStream) ReleasableInputStream(com.qcloud.cos.internal.ReleasableInputStream) MD5DigestCalculatingInputStream(com.qcloud.cos.internal.MD5DigestCalculatingInputStream) LengthCheckInputStream(com.qcloud.cos.internal.LengthCheckInputStream) ServiceClientHolderInputStream(com.qcloud.cos.internal.ServiceClientHolderInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) ResettableInputStream(com.qcloud.cos.internal.ResettableInputStream) SdkFilterInputStream(com.qcloud.cos.internal.SdkFilterInputStream) InputStream(java.io.InputStream) CosClientException(com.qcloud.cos.exception.CosClientException) MD5DigestCalculatingInputStream(com.qcloud.cos.internal.MD5DigestCalculatingInputStream) CosMetadataResponseHandler(com.qcloud.cos.internal.CosMetadataResponseHandler) DecoderException(org.apache.commons.codec.DecoderException) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) MultiObjectDeleteException(com.qcloud.cos.exception.MultiObjectDeleteException) IOException(java.io.IOException) URISyntaxException(java.net.URISyntaxException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) MalformedURLException(java.net.MalformedURLException) DecoderException(org.apache.commons.codec.DecoderException) ByteArrayInputStream(java.io.ByteArrayInputStream) File(java.io.File) ResponseHeaderHandlerChain(com.qcloud.cos.internal.ResponseHeaderHandlerChain)

Aggregations

CosServiceException (com.qcloud.cos.exception.CosServiceException)82 CosClientException (com.qcloud.cos.exception.CosClientException)64 COSClient (com.qcloud.cos.COSClient)37 ClientConfig (com.qcloud.cos.ClientConfig)37 BasicCOSCredentials (com.qcloud.cos.auth.BasicCOSCredentials)37 COSCredentials (com.qcloud.cos.auth.COSCredentials)37 Region (com.qcloud.cos.region.Region)37 File (java.io.File)28 IOException (java.io.IOException)20 PutObjectRequest (com.qcloud.cos.model.PutObjectRequest)15 TransferManager (com.qcloud.cos.transfer.TransferManager)14 ExecutorService (java.util.concurrent.ExecutorService)14 GetObjectRequest (com.qcloud.cos.model.GetObjectRequest)13 ByteArrayInputStream (java.io.ByteArrayInputStream)13 PutObjectResult (com.qcloud.cos.model.PutObjectResult)12 MultiObjectDeleteException (com.qcloud.cos.exception.MultiObjectDeleteException)11 ObjectMetadata (com.qcloud.cos.model.ObjectMetadata)11 ArrayList (java.util.ArrayList)10 LinkedList (java.util.LinkedList)10 Test (org.junit.Test)10