use of com.qcloud.cos.model.ListObjectsRequest in project cos-java-sdk-v5 by tencentyun.
the class ListObjectsDemo method listAllObjects.
// 如果要获取超过maxkey数量的object或者获取所有的object, 则需要循环调用listobject, 用上一次返回的next marker作为下一次调用的marker,
// 直到返回的truncated为false
public static void listAllObjects() {
// 1 初始化用户身份信息(secretId, secretKey)
COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
// 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
// 3 生成cos客户端
COSClient cosclient = new COSClient(cred, clientConfig);
// bucket名需包含appid
String bucketName = "mybucket-1251668577";
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
// 设置bucket名称
listObjectsRequest.setBucketName(bucketName);
// prefix表示列出的object的key以prefix开始
listObjectsRequest.setPrefix("");
// deliter表示分隔符, 设置为/表示列出当前目录下的object, 设置为空表示列出所有的object
listObjectsRequest.setDelimiter("");
// 设置最大遍历出多少个对象, 一次listobject最大支持1000
listObjectsRequest.setMaxKeys(1000);
ObjectListing objectListing = null;
do {
try {
objectListing = cosclient.listObjects(listObjectsRequest);
} catch (CosServiceException e) {
e.printStackTrace();
return;
} catch (CosClientException e) {
e.printStackTrace();
return;
}
// common prefix表示表示被delimiter截断的路径, 如delimter设置为/, common prefix则表示所有子目录的路径
List<String> commonPrefixs = objectListing.getCommonPrefixes();
// object summary表示所有列出的object列表
List<COSObjectSummary> cosObjectSummaries = objectListing.getObjectSummaries();
for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
// 文件的路径key
String key = cosObjectSummary.getKey();
// 文件的etag
String etag = cosObjectSummary.getETag();
// 文件的长度
long fileSize = cosObjectSummary.getSize();
// 文件的存储类型
String storageClasses = cosObjectSummary.getStorageClass();
}
String nextMarker = objectListing.getNextMarker();
listObjectsRequest.setMarker(nextMarker);
} while (objectListing.isTruncated());
cosclient.shutdown();
}
use of com.qcloud.cos.model.ListObjectsRequest in project cos-java-sdk-v5 by tencentyun.
the class ListObjectTest method ListObjectWithDelimiterTest.
@Test
public void ListObjectWithDelimiterTest() {
if (!judgeUserInfoValid()) {
return;
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket, keyPrefix, null, "/", 100);
ObjectListing objectListing = cosclient.listObjects(listObjectsRequest);
assertEquals(1L, objectListing.getCommonPrefixes().size());
assertEquals(0L, objectListing.getObjectSummaries().size());
}
use of com.qcloud.cos.model.ListObjectsRequest in project cos-java-sdk-v5 by tencentyun.
the class AbstractCOSClientTest method clearBucket.
private static void clearBucket() throws Exception {
abortAllNotFinishedMultipartUpload();
// 先判断bucket是否开启了版本控制
GetBucketVersioningConfigurationRequest getBucketVersioningConfigurationRequest = new GetBucketVersioningConfigurationRequest(bucket);
BucketVersioningConfiguration bucketVersioningConfiguration = cosclient.getBucketVersioningConfiguration(getBucketVersioningConfigurationRequest);
if (bucketVersioningConfiguration.getStatus().compareToIgnoreCase(BucketVersioningConfiguration.ENABLED) == 0) {
clearObjectVersions();
}
String nextMarker = "";
boolean isTruncated = false;
do {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucket);
listObjectsRequest.setMaxKeys(1000);
listObjectsRequest.setPrefix("");
listObjectsRequest.setDelimiter("");
listObjectsRequest.setMarker(nextMarker);
ObjectListing objectListing = cosclient.listObjects(listObjectsRequest);
for (COSObjectSummary cosObjectSummary : objectListing.getObjectSummaries()) {
String key = cosObjectSummary.getKey();
// 删除这个key
System.out.println(key);
cosclient.deleteObject(bucket, key);
}
nextMarker = objectListing.getNextMarker();
isTruncated = objectListing.isTruncated();
} while (isTruncated);
}
use of com.qcloud.cos.model.ListObjectsRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method callCOSClientWithRetry.
// posix bucket mkdir if the middle part exist will return the 500 error,
// and the rename if the dst exist will return the 500 status too,
// which make the related 5** retry useless. mo improve the resp info to filter.
private <X> Object callCOSClientWithRetry(X request) throws CosServiceException, IOException {
String sdkMethod = "";
int retryIndex = 1;
int l5ErrorCodeRetryIndex = 1;
while (true) {
try {
if (request instanceof PutObjectRequest) {
sdkMethod = "putObject";
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().mark((int) ((PutObjectRequest) request).getMetadata().getContentLength());
}
return this.cosClient.putObject((PutObjectRequest) request);
} else if (request instanceof UploadPartRequest) {
sdkMethod = "uploadPart";
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().mark((int) ((UploadPartRequest) request).getPartSize());
}
return this.cosClient.uploadPart((UploadPartRequest) request);
} else if (request instanceof CopyPartRequest) {
sdkMethod = "copyPartRequest";
return this.cosClient.copyPart((CopyPartRequest) request);
} else if (request instanceof HeadBucketRequest) {
// use for checking bucket type
sdkMethod = "headBucket";
return this.cosClient.headBucket((HeadBucketRequest) request);
} else if (request instanceof RenameRequest) {
sdkMethod = "rename";
this.cosClient.rename((RenameRequest) request);
return new Object();
} else if (request instanceof GetObjectMetadataRequest) {
sdkMethod = "queryObjectMeta";
return this.cosClient.getObjectMetadata((GetObjectMetadataRequest) request);
} else if (request instanceof DeleteObjectRequest) {
sdkMethod = "deleteObject";
this.cosClient.deleteObject((DeleteObjectRequest) request);
return new Object();
} else if (request instanceof CopyObjectRequest) {
sdkMethod = "copyFile";
return this.cosClient.copyObject((CopyObjectRequest) request);
} else if (request instanceof GetObjectRequest) {
sdkMethod = "getObject";
return this.cosClient.getObject((GetObjectRequest) request);
} else if (request instanceof ListObjectsRequest) {
sdkMethod = "listObjects";
return this.cosClient.listObjects((ListObjectsRequest) request);
} else if (request instanceof InitiateMultipartUploadRequest) {
sdkMethod = "initiateMultipartUpload";
return this.cosClient.initiateMultipartUpload((InitiateMultipartUploadRequest) request);
} else if (request instanceof CompleteMultipartUploadRequest) {
sdkMethod = "completeMultipartUpload";
return this.cosClient.completeMultipartUpload((CompleteMultipartUploadRequest) request);
} else if (request instanceof AbortMultipartUploadRequest) {
sdkMethod = "abortMultipartUpload";
this.cosClient.abortMultipartUpload((AbortMultipartUploadRequest) request);
return new Object();
} else {
throw new IOException("no such method");
}
} catch (ResponseNotCompleteException nce) {
if (this.completeMPUCheckEnabled && request instanceof CompleteMultipartUploadRequest) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null == fileMetadata) {
// if file not exist must throw the exception.
handleException(nce, key);
}
LOG.warn("Complete mpu resp not complete key [{}]", key);
// todo: some other double check after cgi unified the ctime of mpu.
return new CompleteMultipartUploadResult();
} else {
throw new IOException(nce);
}
} catch (CosServiceException cse) {
String errMsg = String.format("all cos sdk failed, retryIndex: [%d / %d], " + "call method: %s, exception: %s", retryIndex, this.maxRetryTimes, sdkMethod, cse);
int statusCode = cse.getStatusCode();
String errorCode = cse.getErrorCode();
LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
// 对5xx错误进行重试
if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
// complete mpu error code might be in body when status code is 200
// double check to head object only works in big data job case which key is not same.
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}", errorCode, key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
// here same like the copy request not setting the interval sleep for now
if (retryIndex <= this.maxRetryTimes) {
LOG.info(errMsg, cse);
++retryIndex;
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else if (statusCode / 100 == 5) {
if (retryIndex <= this.maxRetryTimes) {
if (statusCode == 503) {
if (useL5Id) {
if (l5ErrorCodeRetryIndex >= this.l5UpdateMaxRetryTimes) {
// L5上报,进行重试
l5EndpointResolver.updateRouteResult(-1);
l5ErrorCodeRetryIndex = 1;
} else {
l5ErrorCodeRetryIndex = l5ErrorCodeRetryIndex + 1;
}
}
}
LOG.info(errMsg, cse);
long sleepLeast = retryIndex * 300L;
long sleepBound = retryIndex * 500L;
try {
if (request instanceof PutObjectRequest) {
LOG.info("Try to reset the put object request input stream.");
if (((PutObjectRequest) request).getInputStream().markSupported()) {
((PutObjectRequest) request).getInputStream().reset();
} else {
LOG.error("The put object request input stream can not be reset, so it can not be" + " retried.");
throw cse;
}
}
if (request instanceof UploadPartRequest) {
LOG.info("Try to reset the upload part request input stream.");
if (((UploadPartRequest) request).getInputStream().markSupported()) {
((UploadPartRequest) request).getInputStream().reset();
} else {
LOG.error("The upload part request input stream can not be reset, so it can not " + "be retried" + ".");
throw cse;
}
}
// if direct retry may occur 403 not found the upload id.
if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
String key = ((CompleteMultipartUploadRequest) request).getKey();
FileMetadata fileMetadata = this.queryObjectMetadata(key);
if (null != fileMetadata) {
// if file exist direct return.
LOG.info("complete mpu error might access time out, " + "but key {} already exist, length {}", key, fileMetadata.getLength());
return new CompleteMultipartUploadResult();
}
}
Thread.sleep(ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
++retryIndex;
} catch (InterruptedException e) {
throw new IOException(e.toString());
}
} else {
LOG.error(errMsg, cse);
throw new IOException(errMsg);
}
} else {
throw cse;
}
} catch (Exception e) {
throw new IOException(e);
}
}
}
use of com.qcloud.cos.model.ListObjectsRequest in project hadoop-cos by tencentyun.
the class CosNativeFileSystemStore method list.
/**
* list objects
*
* @param prefix prefix
* @param delimiter delimiter
* @param maxListingLength max no. of entries
* @param priorLastKey last key in any previous search
* @return a list of matches
* @throws IOException on any reported failure
*/
private CosNPartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey, CosNResultInfo info) throws IOException {
LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.", prefix, delimiter, maxListingLength, priorLastKey);
if (!prefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
prefix += CosNFileSystem.PATH_DELIMITER;
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucketName);
listObjectsRequest.setPrefix(prefix);
listObjectsRequest.setDelimiter(delimiter);
listObjectsRequest.setMarker(priorLastKey);
listObjectsRequest.setMaxKeys(maxListingLength);
ObjectListing objectListing = null;
try {
objectListing = (ObjectListing) callCOSClientWithRetry(listObjectsRequest);
} catch (Exception e) {
String errMsg = String.format("prefix: %s, delimiter: %s, maxListingLength: %d, " + "priorLastKey: %s. list occur a exception: %s", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey, e);
LOG.error(errMsg);
handleException(new Exception(errMsg), prefix);
}
if (null == objectListing) {
String errMessage = String.format("List objects failed for the prefix: %s, delimiter: %s, " + "maxListingLength:%d, priorLastKey: %s.", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey);
handleException(new Exception(errMessage), prefix);
}
ArrayList<FileMetadata> fileMetadataArray = new ArrayList<>();
ArrayList<FileMetadata> commonPrefixArray = new ArrayList<>();
List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
boolean isKeySamePrefix = false;
for (COSObjectSummary cosObjectSummary : summaries) {
String filePath = cosObjectSummary.getKey();
if (!filePath.startsWith(CosNFileSystem.PATH_DELIMITER)) {
filePath = CosNFileSystem.PATH_DELIMITER + filePath;
}
if (filePath.equals(prefix)) {
isKeySamePrefix = true;
continue;
}
long mtime = 0;
if (cosObjectSummary.getLastModified() != null) {
mtime = cosObjectSummary.getLastModified().getTime();
}
long fileLen = cosObjectSummary.getSize();
String fileEtag = cosObjectSummary.getETag();
if (cosObjectSummary.getKey().endsWith(CosNFileSystem.PATH_DELIMITER) && cosObjectSummary.getSize() == 0) {
fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, false, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
} else {
fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, true, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
}
}
List<String> commonPrefixes = objectListing.getCommonPrefixes();
for (String commonPrefix : commonPrefixes) {
if (!commonPrefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
commonPrefix = CosNFileSystem.PATH_DELIMITER + commonPrefix;
}
commonPrefixArray.add(new FileMetadata(commonPrefix, 0, 0, false));
}
FileMetadata[] fileMetadata = new FileMetadata[fileMetadataArray.size()];
for (int i = 0; i < fileMetadataArray.size(); ++i) {
fileMetadata[i] = fileMetadataArray.get(i);
}
FileMetadata[] commonPrefixMetaData = new FileMetadata[commonPrefixArray.size()];
for (int i = 0; i < commonPrefixArray.size(); ++i) {
commonPrefixMetaData[i] = commonPrefixArray.get(i);
}
// 如果truncated为false, 则表明已经遍历完
if (!objectListing.isTruncated()) {
CosNPartialListing ret = new CosNPartialListing(null, fileMetadata, commonPrefixMetaData);
if (info != null) {
info.setRequestID(objectListing.getRequestId());
info.setKeySameToPrefix(isKeySamePrefix);
}
return ret;
} else {
CosNPartialListing ret = new CosNPartialListing(objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData);
if (info != null) {
info.setRequestID(objectListing.getRequestId());
info.setKeySameToPrefix(isKeySamePrefix);
}
return ret;
}
}
Aggregations