use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method completeMultipartUpload.
/**
* complete cos mpu, sometimes return null complete mpu result
* @param key cos key
* @param uploadId upload id
* @param partETagList each part etag list
* @return result
* @throws IOException when fail to complete the multipart upload.
*/
public CompleteMultipartUploadResult completeMultipartUpload(String key, String uploadId, List<PartETag> partETagList) throws IOException {
Collections.sort(partETagList, new Comparator<PartETag>() {
@Override
public int compare(PartETag o1, PartETag o2) {
return o1.getPartNumber() - o2.getPartNumber();
}
});
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETagList);
completeMultipartUploadRequest.setObjectMetadata(objectMetadata);
return (CompleteMultipartUploadResult) this.callCOSClientWithRetry(completeMultipartUploadRequest);
} catch (CosServiceException cse) {
// when first calling with 503 access time out, next retry will 409.
int statusCode = cse.getStatusCode();
if (statusCode == 409) {
// check file whether exist
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist through exception
handleException(cse, key);
}
LOG.warn("Upload the cos key [{}] complete mpu concurrently", key);
} else {
// other exception
String errMsg = String.format("Complete the multipart upload failed. " + "cos service exception, cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, cse.toString());
handleException(new Exception(errMsg), key);
}
} catch (Exception e) {
String errMsg = String.format("Complete the multipart upload failed. " + "cos key: %s, upload id: %s, " + "exception: %s", key, uploadId, e.toString());
handleException(new Exception(errMsg), key);
}
return null;
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method queryObjectMetadata.
private FileMetadata queryObjectMetadata(String key, CosNResultInfo info) throws IOException {
LOG.debug("Query Object metadata. cos key: {}.", key);
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
try {
ObjectMetadata objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
long mtime = 0;
long fileSize;
if (objectMetadata.getLastModified() != null) {
mtime = objectMetadata.getLastModified().getTime();
}
fileSize = objectMetadata.getContentLength();
String ETag = objectMetadata.getETag();
String crc64ecm = objectMetadata.getCrc64Ecma();
String crc32cm = (String) objectMetadata.getRawMetadataValue(Constants.CRC32C_RESP_HEADER);
String versionId = objectMetadata.getVersionId();
Map<String, byte[]> userMetadata = null;
if (objectMetadata.getUserMetadata() != null) {
userMetadata = new HashMap<>();
for (Map.Entry<String, String> userMetadataEntry : objectMetadata.getUserMetadata().entrySet()) {
if (userMetadataEntry.getKey().startsWith(ensureValidAttributeName(XATTR_PREFIX))) {
String xAttrJsonStr = new String(Base64.decode(userMetadataEntry.getValue()), StandardCharsets.UTF_8);
CosNXAttr cosNXAttr;
try {
cosNXAttr = Jackson.fromJsonString(xAttrJsonStr, CosNXAttr.class);
} catch (CosClientException e) {
LOG.warn("Parse the xAttr failed. name: {}, XAttJsonStr: {}.", userMetadataEntry.getKey(), xAttrJsonStr);
// Skip
continue;
}
if (null != cosNXAttr) {
userMetadata.put(cosNXAttr.getName(), cosNXAttr.getValue().getBytes(CosNFileSystem.METADATA_ENCODING));
}
}
}
}
boolean isFile = true;
if (isPosixBucket) {
if (objectMetadata.isFileModeDir() || key.equals(CosNFileSystem.PATH_DELIMITER)) {
isFile = false;
}
} else {
isFile = !key.endsWith(CosNFileSystem.PATH_DELIMITER);
}
FileMetadata fileMetadata = new FileMetadata(key, fileSize, mtime, isFile, ETag, crc64ecm, crc32cm, versionId, objectMetadata.getStorageClass(), userMetadata);
// record the last request result info
if (info != null) {
info.setRequestID(objectMetadata.getRequestId());
}
LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key, objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
return fileMetadata;
} catch (CosServiceException e) {
if (info != null) {
info.setRequestID(e.getRequestId());
}
if (e.getStatusCode() != 404) {
String errorMsg = String.format("Retrieve the file metadata file failure. " + "cos key: %s, exception: %s", key, e);
handleException(new Exception(errorMsg), key);
}
}
return null;
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method storeAttribute.
private void storeAttribute(String key, String attribute, byte[] value, boolean deleted) throws IOException {
if (deleted) {
LOG.debug("Delete the extended attribute. cos key: {}, attribute: {}.", key, attribute);
}
if (null != value && !deleted) {
LOG.debug("Store the extended attribute. cos key: {}, attribute: {}, value: {}.", key, attribute, new String(value, StandardCharsets.UTF_8));
}
if (null == value && !deleted) {
throw new IOException("The attribute value to be set can not be null.");
}
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
ObjectMetadata objectMetadata = null;
try {
objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
} catch (CosServiceException e) {
if (e.getStatusCode() != 404) {
String errorMessage = String.format("Retrieve the file metadata failed. " + "cos key: %s, exception: %s.", key, e);
handleException(new Exception(errorMessage), key);
}
}
if (null != objectMetadata) {
Map<String, String> userMetadata = objectMetadata.getUserMetadata();
if (deleted) {
if (null != userMetadata) {
userMetadata.remove(ensureValidAttributeName(attribute));
} else {
return;
}
} else {
if (null == userMetadata) {
userMetadata = new HashMap<>();
}
CosNXAttr cosNXAttr = new CosNXAttr();
cosNXAttr.setName(attribute);
cosNXAttr.setValue(new String(value, CosNFileSystem.METADATA_ENCODING));
String xAttrJsonStr = Jackson.toJsonString(cosNXAttr);
userMetadata.put(ensureValidAttributeName(XATTR_PREFIX + attribute), Base64.encodeAsString(xAttrJsonStr.getBytes(StandardCharsets.UTF_8)));
}
objectMetadata.setUserMetadata(userMetadata);
// 构造原地copy请求来设置用户自定义属性
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
if (null != objectMetadata.getStorageClass()) {
copyObjectRequest.setStorageClass(objectMetadata.getStorageClass());
}
copyObjectRequest.setNewObjectMetadata(objectMetadata);
copyObjectRequest.setRedirectLocation("Replaced");
this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
try {
callCOSClientWithRetry(copyObjectRequest);
} catch (Exception e) {
String errMsg = String.format("Failed to modify the user-defined attributes. " + "cos key: %s, attribute: %s, exception: %s.", key, attribute, e);
handleException(new Exception(errMsg), key);
}
}
}
use of com.qcloud.cos.exception.CosServiceException in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method storeFileWithRetry.
private void storeFileWithRetry(String key, InputStream inputStream, byte[] md5Hash, long length) throws IOException {
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
if (null != md5Hash) {
objectMetadata.setContentMD5(Base64.encodeAsString(md5Hash));
}
objectMetadata.setContentLength(length);
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, objectMetadata);
if (null != this.storageClass) {
putObjectRequest.setStorageClass(this.storageClass);
}
if (this.trafficLimit >= 0) {
putObjectRequest.setTrafficLimit(this.trafficLimit);
}
this.setEncryptionMetadata(putObjectRequest, objectMetadata);
PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
LOG.debug("Store the file successfully. cos key: {}, ETag: {}.", key, putObjectResult.getETag());
} catch (CosServiceException cse) {
// 避免并发上传的问题
int statusCode = cse.getStatusCode();
if (statusCode == 409) {
// Check一下这个文件是否已经存在
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// 如果文件不存在,则需要抛出异常
handleException(cse, key);
}
LOG.warn("Upload the cos key [{}] concurrently", key);
} else {
// 其他错误都要抛出来
handleException(cse, key);
}
} catch (Exception e) {
String errMsg = String.format("Store the file failed, cos key: %s, exception: %s.", key, e);
handleException(new Exception(errMsg), key);
}
}
use of com.qcloud.cos.exception.CosServiceException in project cos-java-sdk-v5 by tencentyun.
the class COSClient method uploadObjectInternal.
protected <UploadObjectRequest extends PutObjectRequest> ObjectMetadata uploadObjectInternal(UploadMode uploadMode, UploadObjectRequest uploadObjectRequest) throws CosClientException, CosServiceException {
rejectNull(uploadObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object");
rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when uploading an object");
final File file = uploadObjectRequest.getFile();
final InputStream isOrig = uploadObjectRequest.getInputStream();
final String bucketName = uploadObjectRequest.getBucketName();
final String key = uploadObjectRequest.getKey();
ObjectMetadata metadata = uploadObjectRequest.getMetadata();
InputStream input = isOrig;
if (metadata == null)
metadata = new ObjectMetadata();
rejectNull(bucketName, "The bucket name parameter must be specified when uploading an object");
rejectNull(key, "The key parameter must be specified when uploading an object");
// information from it to auto-configure a few options
if (file == null) {
// unlimited mark-and-reset
if (input != null)
input = ReleasableInputStream.wrap(input);
} else {
// Always set the content length, even if it's already set
metadata.setContentLength(file.length());
final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
if (file.length() > maxAllowdSingleFileSize) {
throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + file.length() + ", please use transferManager to upload big file!");
}
final boolean calculateMD5 = metadata.getContentMD5() == null;
if (calculateMD5 && !skipMd5CheckStrategy.skipServerSideValidation(uploadObjectRequest)) {
try {
String contentMd5_b64 = Md5Utils.md5AsBase64(file);
metadata.setContentMD5(contentMd5_b64);
} catch (Exception e) {
throw new CosClientException("Unable to calculate MD5 hash: " + e.getMessage(), e);
}
}
input = ResettableInputStream.newResettableInputStream(file, "Unable to find file to upload");
}
final ObjectMetadata returnedMetadata;
MD5DigestCalculatingInputStream md5DigestStream = null;
try {
CosHttpRequest<UploadObjectRequest> request = null;
if (uploadMode.equals(UploadMode.PUT_OBJECT)) {
request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.PUT);
} else if (uploadMode.equals(UploadMode.APPEND_OBJECT)) {
request = createRequest(bucketName, key, uploadObjectRequest, HttpMethodName.POST);
AppendObjectRequest appendObjectRequest = (AppendObjectRequest) uploadObjectRequest;
String positionStr = String.valueOf(appendObjectRequest.getPosition());
request.addParameter("append", null);
request.addParameter("position", positionStr);
}
if (uploadObjectRequest.getAccessControlList() != null) {
addAclHeaders(request, uploadObjectRequest.getAccessControlList());
} else if (uploadObjectRequest.getCannedAcl() != null) {
request.addHeader(Headers.COS_CANNED_ACL, uploadObjectRequest.getCannedAcl().toString());
}
if (uploadObjectRequest.getStorageClass() != null) {
request.addHeader(Headers.STORAGE_CLASS, uploadObjectRequest.getStorageClass());
}
if (uploadObjectRequest.getRedirectLocation() != null) {
request.addHeader(Headers.REDIRECT_LOCATION, uploadObjectRequest.getRedirectLocation());
if (input == null) {
input = new ByteArrayInputStream(new byte[0]);
}
}
// Populate the SSE-C parameters to the request header
populateSSE_C(request, uploadObjectRequest.getSSECustomerKey());
// Populate the SSE KMS parameters to the request header
populateSSE_KMS(request, uploadObjectRequest.getSSECOSKeyManagementParams());
// Populate the traffic limit parameter to the request header
populateTrafficLimit(request, uploadObjectRequest.getTrafficLimit());
// Use internal interface to differentiate 0 from unset.
final Long contentLength = (Long) metadata.getRawMetadataValue(Headers.CONTENT_LENGTH);
if (contentLength == null) {
/*
* There's nothing we can do except for let the HTTP client buffer the input stream
* contents if the caller doesn't tell us how much data to expect in a stream since
* we have to explicitly tell how much we're sending before we start sending any of
* it.
*/
log.warn("No content length specified for stream data. " + "Stream contents will be buffered in memory and could result in " + "out of memory errors.");
} else {
final long expectedLength = contentLength.longValue();
final long maxAllowdSingleFileSize = 5 * 1024L * 1024L * 1024L;
if (expectedLength > maxAllowdSingleFileSize) {
throw new CosClientException("max size 5GB is allowed by putObject Method, your filesize is " + expectedLength + ", please use transferManager to upload big file!");
}
if (expectedLength >= 0) {
// Performs length check on the underlying data stream.
// For COS encryption client, the underlying data stream here
// refers to the cipher-text data stream (ie not the underlying
// plain-text data stream which in turn may have been wrapped
// with it's own length check input stream.)
LengthCheckInputStream lcis = new // expected
LengthCheckInputStream(// expected
input, // expected
expectedLength, // uploaded
EXCLUDE_SKIPPED_BYTES);
input = lcis;
}
}
if (metadata.getContentMD5() == null && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadObjectRequest)) {
/*
* If the user hasn't set the content MD5, then we don't want to buffer the whole
* stream in memory just to calculate it. Instead, we can calculate it on the fly
* and validate it with the returned ETag from the object upload.
*/
input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
}
populateRequestMetadata(request, metadata);
request.setContent(input);
try {
if (uploadObjectRequest.getPicOperations() != null) {
request.addHeader(Headers.PIC_OPERATIONS, Jackson.toJsonString(uploadObjectRequest.getPicOperations()));
returnedMetadata = invoke(request, new ResponseHeaderHandlerChain<ObjectMetadata>(new Unmarshallers.ImagePersistenceUnmarshaller(), new CosMetadataResponseHandler()));
} else {
returnedMetadata = invoke(request, new CosMetadataResponseHandler());
}
} catch (Throwable t) {
throw Throwables.failure(t);
}
} finally {
CosDataSource.Utils.cleanupDataSource(uploadObjectRequest, file, isOrig, input, log);
}
String contentMd5 = metadata.getContentMD5();
if (md5DigestStream != null) {
contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
}
final String etag = returnedMetadata.getETag();
if (contentMd5 != null && uploadMode.equals(UploadMode.PUT_OBJECT) && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
byte[] serverSideHash = null;
try {
serverSideHash = BinaryUtils.fromHex(etag);
} catch (DecoderException e) {
throw new CosClientException("Unable to verify integrity of data upload. " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS . " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
}
if (!Arrays.equals(clientSideHash, serverSideHash)) {
throw new CosClientException("Unable to verify integrity of data upload. " + "Client calculated content hash (contentMD5: " + contentMd5 + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by COS . " + "You may need to delete the data stored in COS . (metadata.contentMD5: " + metadata.getContentMD5() + ", bucketName: " + bucketName + ", key: " + key + ")");
}
}
return returnedMetadata;
}
Aggregations