use of com.qcloud.cos.model.GetObjectMetadataRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method callCOSClientWithRetry.
// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
String sdkMethod = "";
int retryIndex = 1;
int l5ErrorCodeRetryIndex = 1;
while (true) {
try {
if (request instanceof PutObjectRequest) {
sdkMethod = "putObject";
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
}
return this.cosClient.putObject((PutObjectRequest) request);
} else if (request instanceof UploadPartRequest) {
sdkMethod = "uploadPart";
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
}
return this.cosClient.uploadPart((UploadPartRequest) request);
} else if (request instanceof CopyPartRequest) {
sdkMethod = "copyPartRequest";
return this.cosClient.copyPart((CopyPartRequest) request);
} else if (request instanceof HeadBucketRequest) {
// use for checking bucket type
sdkMethod = "headBucket";
return this.cosClient.headBucket((HeadBucketRequest) request);
} else if (request instanceof RenameRequest) {
sdkMethod = "rename";
this.cosClient.rename((RenameRequest) request);
return new Object();
} else if (request instanceof GetObjectMetadataRequest) {
sdkMethod = "queryObjectMeta";
return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
} else if (request instanceof DeleteObjectRequest) {
sdkMethod = "deleteObject";
this.cosClient.deleteObject((DeleteObjectRequest) request);
return new Object();
} else if (request instanceof CopyObjectRequest) {
sdkMethod = "copyFile";
return this.cosClient.copyObject((CopyObjectRequest) request);
} else if (request instanceof GetObjectRequest) {
sdkMethod = "getObject";
return this.cosClient.getObject((GetObjectRequest) request);
} else if (request instanceof ListObjectsRequest) {
sdkMethod = "listObjects";
return this.cosClient.listObjects((ListObjectsRequest) request);
} else if (request instanceof InitiateMultipartUploadRequest) {
sdkMethod = "initiateMultipartUpload";
return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
} else if (request instanceof CompleteMultipartUploadRequest) {
sdkMethod = "completeMultipartUpload";
return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
} else if (request instanceof AbortMultipartUploadRequest) {
sdkMethod = "abortMultipartUpload";
this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
return new Object();
} else {
throw new IOException("no such method");
}
} catch (ResponseNotCompleteException nce) {
if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist must throw the exception.
handleException(nce, key);
}
LOG.warn("Complete mpu resp not complete key [{}]", key);
// todo: some other double check after cgi unified the ctime of mpu.
return new CompleteMultipartUploadResult();
} else {
throw new IOException(nce);
}
} catch (CosServiceException cse) {
String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
int statusCode = cse.getStatusCode();
String errorCode = cse.getErrorCode();
LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
// 对5xx错误进行重试
if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
// complete mpu error code might be in body when status code is 200
// double check to head object only works in big data job case which key is not same.
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
// here same like the copy request not setting the interval sleep for now
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (statusCode / 100 == 5) {
if (retryIndex <= this.maxRetryTimes) {
if (statusCode == 503) {
if (useL5Id) {
if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
// L5上报,进行重试
l5EndpointResolver.updateRouteResult(-1);
l5ErrorCodeRetryIndex = 1;
} else {
l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
}
}
}
LOG.info(errMsg, cse);
long sleepLeast = retryIndex * 300L;
long sleepBound = retryIndex * 500L;
try {
if (request instanceof PutObjectRequest) {
LOG.info("Try to reset the put object request input stream.");
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().reset();
} else {
LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
throw cse;
}
}
if (request instanceof UploadPartRequest) {
LOG.info("Try to reset the upload part request input stream.");
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().reset();
} else {
LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
throw cse;
}
}
// if direct retry may occur 403 not found the upload id.
if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
}
Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
++retryIndex;
} catch (InterruptedException e) {
throw new IOException(e.toString());
}
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else {
throw cse;
}
} catch (Exception e) {
throw new IOException(e);
}
}
}
use of com.qcloud.cos.model.GetObjectMetadataRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method queryObjectMetadata.
private FileMetadata queryObjectMetadata(String key, CosNResultInfo info) throws IOException {
LOG.debug("Query Object metadata. cos key: {}.", key);
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
try {
ObjectMetadata objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
long mtime = 0;
long fileSize;
if (objectMetadata.getLastModified() != null) {
mtime = objectMetadata.getLastModified().getTime();
}
fileSize = objectMetadata.getContentLength();
String ETag = objectMetadata.getETag();
String crc64ecm = objectMetadata.getCrc64Ecma();
String crc32cm = (String) objectMetadata.getRawMetadataValue(Constants.CRC32C_RESP_HEADER);
String versionId = objectMetadata.getVersionId();
Map<String, byte[]> userMetadata = null;
if (objectMetadata.getUserMetadata() != null) {
userMetadata = new HashMap<>();
for (Map.Entry<String, String> userMetadataEntry : objectMetadata.getUserMetadata().entrySet()) {
if (userMetadataEntry.getKey().startsWith(ensureValidAttributeName(XATTR_PREFIX))) {
String xAttrJsonStr = new String(Base64.decode(userMetadataEntry.getValue()), StandardCharsets.UTF_8);
CosNXAttr cosNXAttr;
try {
cosNXAttr = Jackson.fromJsonString(xAttrJsonStr, CosNXAttr.class);
} catch (CosClientException e) {
LOG.warn("Parse the xAttr failed. name: {}, XAttJsonStr: {}.", userMetadataEntry.getKey(), xAttrJsonStr);
// Skip
continue;
}
if (null != cosNXAttr) {
userMetadata.put(cosNXAttr.getName(), cosNXAttr.getValue().getBytes(CosNFileSystem.METADATA_ENCODING));
}
}
}
}
boolean isFile = true;
if (isPosixBucket) {
if (objectMetadata.isFileModeDir() || key.equals(CosNFileSystem.PATH_DELIMITER)) {
isFile = false;
}
} else {
isFile = !key.endsWith(CosNFileSystem.PATH_DELIMITER);
}
FileMetadata fileMetadata = new FileMetadata(key, fileSize, mtime, isFile, ETag, crc64ecm, crc32cm, versionId, objectMetadata.getStorageClass(), userMetadata);
// record the last request result info
if (info != null) {
info.setRequestID(objectMetadata.getRequestId());
}
LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key, objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
return fileMetadata;
} catch (CosServiceException e) {
if (info != null) {
info.setRequestID(e.getRequestId());
}
if (e.getStatusCode() != 404) {
String errorMsg = String.format("Retrieve the file metadata file failure. " + "cos key: %s, exception: %s", key, e);
handleException(new Exception(errorMsg), key);
}
}
return null;
}
use of com.qcloud.cos.model.GetObjectMetadataRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method storeAttribute.
private void storeAttribute(String key, String attribute, byte[] value, boolean deleted) throws IOException {
if (deleted) {
LOG.debug("Delete the extended attribute. cos key: {}, attribute: {}.", key, attribute);
}
if (null != value && !deleted) {
LOG.debug("Store the extended attribute. cos key: {}, attribute: {}, value: {}.", key, attribute, new String(value, StandardCharsets.UTF_8));
}
if (null == value && !deleted) {
throw new IOException("The attribute value to be set can not be null.");
}
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key);
this.setEncryptionMetadata(getObjectMetadataRequest, new ObjectMetadata());
ObjectMetadata objectMetadata = null;
try {
objectMetadata = (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
} catch (CosServiceException e) {
if (e.getStatusCode() != 404) {
String errorMessage = String.format("Retrieve the file metadata failed. " + "cos key: %s, exception: %s.", key, e);
handleException(new Exception(errorMessage), key);
}
}
if (null != objectMetadata) {
Map<String, String> userMetadata = objectMetadata.getUserMetadata();
if (deleted) {
if (null != userMetadata) {
userMetadata.remove(ensureValidAttributeName(attribute));
} else {
return;
}
} else {
if (null == userMetadata) {
userMetadata = new HashMap<>();
}
CosNXAttr cosNXAttr = new CosNXAttr();
cosNXAttr.setName(attribute);
cosNXAttr.setValue(new String(value, CosNFileSystem.METADATA_ENCODING));
String xAttrJsonStr = Jackson.toJsonString(cosNXAttr);
userMetadata.put(ensureValidAttributeName(XATTR_PREFIX + attribute), Base64.encodeAsString(xAttrJsonStr.getBytes(StandardCharsets.UTF_8)));
}
objectMetadata.setUserMetadata(userMetadata);
// 构造原地copy请求来设置用户自定义属性
if (crc32cEnabled) {
objectMetadata.setHeader(Constants.CRC32C_REQ_HEADER, Constants.CRC32C_REQ_HEADER_VAL);
}
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName, key);
if (null != objectMetadata.getStorageClass()) {
copyObjectRequest.setStorageClass(objectMetadata.getStorageClass());
}
copyObjectRequest.setNewObjectMetadata(objectMetadata);
copyObjectRequest.setRedirectLocation("Replaced");
this.setEncryptionMetadata(copyObjectRequest, objectMetadata);
copyObjectRequest.setSourceEndpointBuilder(this.cosClient.getClientConfig().getEndpointBuilder());
try {
callCOSClientWithRetry(copyObjectRequest);
} catch (Exception e) {
String errMsg = String.format("Failed to modify the user-defined attributes. " + "cos key: %s, attribute: %s, exception: %s.", key, attribute, e);
handleException(new Exception(errMsg), key);
}
}
}
use of com.qcloud.cos.model.GetObjectMetadataRequest in project cos-java-sdk-v5 by tencentyun.
the class TransferManager method getPersistableResumeRecord.
private PersistableResumeDownload getPersistableResumeRecord(GetObjectRequest getObjectRequest, File destFile, String resumableTaskFilePath) {
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey());
if (getObjectRequest.getSSECustomerKey() != null)
getObjectMetadataRequest.setSSECustomerKey(getObjectRequest.getSSECustomerKey());
if (getObjectRequest.getVersionId() != null)
getObjectMetadataRequest.setVersionId(getObjectRequest.getVersionId());
final ObjectMetadata objectMetadata = cos.getObjectMetadata(getObjectMetadataRequest);
long cosContentLength = objectMetadata.getContentLength();
long cosLastModified = objectMetadata.getLastModified().getTime();
String cosEtag = objectMetadata.getETag();
String cosCrc64 = objectMetadata.getCrc64Ecma();
File resumableTaskFile;
if (resumableTaskFilePath == null || resumableTaskFilePath == "") {
resumableTaskFile = new File(destFile.getAbsolutePath() + ".cosresumabletask");
} else {
resumableTaskFile = new File(resumableTaskFilePath);
}
PersistableResumeDownload downloadRecord = null;
FileInputStream is = null;
try {
// attempt to create the parent if it doesn't exist
File parentDirectory = resumableTaskFile.getParentFile();
if (parentDirectory != null && !parentDirectory.exists()) {
if (!(parentDirectory.mkdirs())) {
throw new CosClientException("Unable to create directory in the path" + parentDirectory.getAbsolutePath());
}
}
if (!resumableTaskFile.exists()) {
resumableTaskFile.createNewFile();
}
is = new FileInputStream(resumableTaskFile);
downloadRecord = PersistableResumeDownload.deserializeFrom(is);
log.info("deserialize download record from " + resumableTaskFile.getAbsolutePath() + "record: " + downloadRecord.serialize());
} catch (IOException e) {
throw new CosClientException("can not create file" + resumableTaskFile.getAbsolutePath() + e);
} catch (IllegalArgumentException e) {
log.warn("resumedownload task file cannot deserialize" + e);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
throw new CosClientException("can not close input stream " + resumableTaskFile.getAbsolutePath() + e);
}
}
}
if (downloadRecord == null || downloadRecord.getLastModified() != cosLastModified || !downloadRecord.getContentLength().equals(Long.toString(cosContentLength)) || !downloadRecord.getEtag().equals(cosEtag) || !downloadRecord.getCrc64ecma().equals(cosCrc64)) {
HashMap<String, Integer> downloadedBlocks = new HashMap<String, Integer>();
downloadRecord = new PersistableResumeDownload(cosLastModified, Long.toString(cosContentLength), cosEtag, cosCrc64, downloadedBlocks);
}
downloadRecord.setDumpFile(resumableTaskFile);
return downloadRecord;
}
use of com.qcloud.cos.model.GetObjectMetadataRequest in project cos-java-sdk-v5 by tencentyun.
the class AbstractCOSClientTest method headMultiPartObject.
protected static ObjectMetadata headMultiPartObject(String key, long expectedLength, int expectedPartNum) {
ObjectMetadata objectMetadata = cosclient.getObjectMetadata(new GetObjectMetadataRequest(bucket, key));
if (!useClientEncryption) {
assertEquals(expectedLength, objectMetadata.getContentLength());
}
String etag = objectMetadata.getETag();
assertTrue(etag.contains("-"));
try {
int etagPartNum = Integer.valueOf(etag.substring(etag.indexOf("-") + 1));
assertEquals(expectedPartNum, etagPartNum);
} catch (NumberFormatException e) {
fail("part number in etag is invalid. etag: " + etag);
}
assertNotNull(objectMetadata.getLastModified());
return objectMetadata;
}
Aggregations