Search in sources :

Example 1 with COSObjectSummary

use of com.qcloud.cos.model.COSObjectSummary in project cos-java-sdk-v5 by tencentyun.

the class ListObjectsDemo method listAllObjects.

// 如果要获取超过maxkey数量的object或者获取所有的object, 则需要循环调用listobject, 用上一次返回的next marker作为下一次调用的marker,
// 直到返回的truncated为false
public static void listAllObjects() {
    // 1 初始化用户身份信息(secretId, secretKey)
    COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
    // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
    ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
    // 3 生成cos客户端
    COSClient cosclient = new COSClient(cred, clientConfig);
    // bucket名需包含appid
    String bucketName = "mybucket-1251668577";
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    // 设置bucket名称
    listObjectsRequest.setBucketName(bucketName);
    // prefix表示列出的object的key以prefix开始
    listObjectsRequest.setPrefix("");
    // deliter表示分隔符, 设置为/表示列出当前目录下的object, 设置为空表示列出所有的object
    listObjectsRequest.setDelimiter("");
    // 设置最大遍历出多少个对象, 一次listobject最大支持1000
    listObjectsRequest.setMaxKeys(1000);
    ObjectListing objectListing = null;
    do {
        try {
            objectListing = cosclient.listObjects(listObjectsRequest);
        } catch (CosServiceException e) {
            e.printStackTrace();
            return;
        } catch (CosClientException e) {
            e.printStackTrace();
            return;
        }
        // common prefix表示表示被delimiter截断的路径, 如delimter设置为/, common prefix则表示所有子目录的路径
        List<String> commonPrefixs = objectListing.getCommonPrefixes();
        // object summary表示所有列出的object列表
        List<COSObjectSummary> cosObjectSummaries = objectListing.getObjectSummaries();
        for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
            // 文件的路径key
            String key = cosObjectSummary.getKey();
            // 文件的etag
            String etag = cosObjectSummary.getETag();
            // 文件的长度
            long fileSize = cosObjectSummary.getSize();
            // 文件的存储类型
            String storageClasses = cosObjectSummary.getStorageClass();
        }
        String nextMarker = objectListing.getNextMarker();
        listObjectsRequest.setMarker(nextMarker);
    } while (objectListing.isTruncated());
    cosclient.shutdown();
}
Also used : COSCredentials(com.qcloud.cos.auth.COSCredentials) BasicCOSCredentials(com.qcloud.cos.auth.BasicCOSCredentials) BasicCOSCredentials(com.qcloud.cos.auth.BasicCOSCredentials) COSObjectSummary(com.qcloud.cos.model.COSObjectSummary) CosClientException(com.qcloud.cos.exception.CosClientException) ObjectListing(com.qcloud.cos.model.ObjectListing) COSClient(com.qcloud.cos.COSClient) ListObjectsRequest(com.qcloud.cos.model.ListObjectsRequest) CosServiceException(com.qcloud.cos.exception.CosServiceException) Region(com.qcloud.cos.region.Region) ClientConfig(com.qcloud.cos.ClientConfig)

Example 2 with COSObjectSummary

use of com.qcloud.cos.model.COSObjectSummary in project cos-java-sdk-v5 by tencentyun.

the class ListObjectTest method ListObjectNoDelimiterTest.

@Test
public void ListObjectNoDelimiterTest() {
    if (!judgeUserInfoValid()) {
        return;
    }
    ObjectListing objectListing = cosclient.listObjects(bucket, keyPrefix);
    assertEquals(0L, objectListing.getCommonPrefixes().size());
    assertEquals(arrayNum, objectListing.getObjectSummaries().size());
    List<COSObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    for (int i = 0; i < arrayNum; ++i) {
        COSObjectSummary cosObjectSummary = objectSummaries.get(i);
        String expectedKey = String.format("%s/%dk.txt", keyPrefix, i);
        assertEquals(expectedKey, cosObjectSummary.getKey());
        assertEquals(i * 1024L, cosObjectSummary.getSize());
    }
}
Also used : COSObjectSummary(com.qcloud.cos.model.COSObjectSummary) ObjectListing(com.qcloud.cos.model.ObjectListing) Test(org.junit.Test)

Example 3 with COSObjectSummary

use of com.qcloud.cos.model.COSObjectSummary in project cos-java-sdk-v5 by tencentyun.

the class ListObjectTest method ListNextBatchObjectWithNoTrunCated.

@Test
public void ListNextBatchObjectWithNoTrunCated() {
    if (!judgeUserInfoValid()) {
        return;
    }
    ObjectListing objectListingPrev = new ObjectListing();
    objectListingPrev.setBucketName(bucket);
    objectListingPrev.setPrefix(keyPrefix);
    objectListingPrev.setNextMarker("");
    objectListingPrev.setMaxKeys(100);
    objectListingPrev.setDelimiter("");
    objectListingPrev.setTruncated(true);
    ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest = new ListNextBatchOfObjectsRequest(objectListingPrev);
    ObjectListing objectListingNext = cosclient.listNextBatchOfObjects(listNextBatchOfObjectsRequest);
    assertEquals(0L, objectListingNext.getCommonPrefixes().size());
    assertEquals(arrayNum, objectListingNext.getObjectSummaries().size());
    List<COSObjectSummary> objectSummaries = objectListingNext.getObjectSummaries();
    for (int i = 0; i < arrayNum; ++i) {
        COSObjectSummary cosObjectSummary = objectSummaries.get(i);
        String expectedKey = String.format("%s/%dk.txt", keyPrefix, i);
        assertEquals(expectedKey, cosObjectSummary.getKey());
        assertEquals(i * 1024L, cosObjectSummary.getSize());
    }
}
Also used : COSObjectSummary(com.qcloud.cos.model.COSObjectSummary) ObjectListing(com.qcloud.cos.model.ObjectListing) ListNextBatchOfObjectsRequest(com.qcloud.cos.model.ListNextBatchOfObjectsRequest) Test(org.junit.Test)

Example 4 with COSObjectSummary

use of com.qcloud.cos.model.COSObjectSummary in project cos-java-sdk-v5 by tencentyun.

the class AbstractCOSClientTest method clearBucket.

private static void clearBucket() throws Exception {
    abortAllNotFinishedMultipartUpload();
    // 先判断bucket是否开启了版本控制
    GetBucketVersioningConfigurationRequest getBucketVersioningConfigurationRequest = new GetBucketVersioningConfigurationRequest(bucket);
    BucketVersioningConfiguration bucketVersioningConfiguration = cosclient.getBucketVersioningConfiguration(getBucketVersioningConfigurationRequest);
    if (bucketVersioningConfiguration.getStatus().compareToIgnoreCase(BucketVersioningConfiguration.ENABLED) == 0) {
        clearObjectVersions();
    }
    String nextMarker = "";
    boolean isTruncated = false;
    do {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucket);
        listObjectsRequest.setMaxKeys(1000);
        listObjectsRequest.setPrefix("");
        listObjectsRequest.setDelimiter("");
        listObjectsRequest.setMarker(nextMarker);
        ObjectListing objectListing = cosclient.listObjects(listObjectsRequest);
        for (COSObjectSummary cosObjectSummary : objectListing.getObjectSummaries()) {
            String key = cosObjectSummary.getKey();
            // 删除这个key
            System.out.println(key);
            cosclient.deleteObject(bucket, key);
        }
        nextMarker = objectListing.getNextMarker();
        isTruncated = objectListing.isTruncated();
    } while (isTruncated);
}
Also used : ListObjectsRequest(com.qcloud.cos.model.ListObjectsRequest) COSObjectSummary(com.qcloud.cos.model.COSObjectSummary) GetBucketVersioningConfigurationRequest(com.qcloud.cos.model.GetBucketVersioningConfigurationRequest) BucketVersioningConfiguration(com.qcloud.cos.model.BucketVersioningConfiguration) ObjectListing(com.qcloud.cos.model.ObjectListing)

Example 5 with COSObjectSummary

use of com.qcloud.cos.model.COSObjectSummary in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method list.

/**
 * list objects
 *
 * @param prefix           prefix
 * @param delimiter        delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey     last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */
private CosNPartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey, CosNResultInfo info) throws IOException {
    LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.", prefix, delimiter, maxListingLength, priorLastKey);
    if (!prefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
        prefix += CosNFileSystem.PATH_DELIMITER;
    }
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter(delimiter);
    listObjectsRequest.setMarker(priorLastKey);
    listObjectsRequest.setMaxKeys(maxListingLength);
    ObjectListing objectListing = null;
    try {
        objectListing = (ObjectListing) callCOSClientWithRetry(listObjectsRequest);
    } catch (Exception e) {
        String errMsg = String.format("prefix: %s, delimiter: %s, maxListingLength: %d, " + "priorLastKey: %s. list occur a exception: %s", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey, e);
        LOG.error(errMsg);
        handleException(new Exception(errMsg), prefix);
    }
    if (null == objectListing) {
        String errMessage = String.format("List objects failed for the prefix: %s, delimiter: %s, " + "maxListingLength:%d, priorLastKey: %s.", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey);
        handleException(new Exception(errMessage), prefix);
    }
    ArrayList<FileMetadata> fileMetadataArray = new ArrayList<>();
    ArrayList<FileMetadata> commonPrefixArray = new ArrayList<>();
    List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
    boolean isKeySamePrefix = false;
    for (COSObjectSummary cosObjectSummary : summaries) {
        String filePath = cosObjectSummary.getKey();
        if (!filePath.startsWith(CosNFileSystem.PATH_DELIMITER)) {
            filePath = CosNFileSystem.PATH_DELIMITER + filePath;
        }
        if (filePath.equals(prefix)) {
            isKeySamePrefix = true;
            continue;
        }
        long mtime = 0;
        if (cosObjectSummary.getLastModified() != null) {
            mtime = cosObjectSummary.getLastModified().getTime();
        }
        long fileLen = cosObjectSummary.getSize();
        String fileEtag = cosObjectSummary.getETag();
        if (cosObjectSummary.getKey().endsWith(CosNFileSystem.PATH_DELIMITER) && cosObjectSummary.getSize() == 0) {
            fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, false, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
        } else {
            fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, true, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
        }
    }
    List<String> commonPrefixes = objectListing.getCommonPrefixes();
    for (String commonPrefix : commonPrefixes) {
        if (!commonPrefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
            commonPrefix = CosNFileSystem.PATH_DELIMITER + commonPrefix;
        }
        commonPrefixArray.add(new FileMetadata(commonPrefix, 0, 0, false));
    }
    FileMetadata[] fileMetadata = new FileMetadata[fileMetadataArray.size()];
    for (int i = 0; i < fileMetadataArray.size(); ++i) {
        fileMetadata[i] = fileMetadataArray.get(i);
    }
    FileMetadata[] commonPrefixMetaData = new FileMetadata[commonPrefixArray.size()];
    for (int i = 0; i < commonPrefixArray.size(); ++i) {
        commonPrefixMetaData[i] = commonPrefixArray.get(i);
    }
    // 如果truncated为false, 则表明已经遍历完
    if (!objectListing.isTruncated()) {
        CosNPartialListing ret = new CosNPartialListing(null, fileMetadata, commonPrefixMetaData);
        if (info != null) {
            info.setRequestID(objectListing.getRequestId());
            info.setKeySameToPrefix(isKeySamePrefix);
        }
        return ret;
    } else {
        CosNPartialListing ret = new CosNPartialListing(objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData);
        if (info != null) {
            info.setRequestID(objectListing.getRequestId());
            info.setKeySameToPrefix(isKeySamePrefix);
        }
        return ret;
    }
}
Also used : COSObjectSummary(com.qcloud.cos.model.COSObjectSummary) ArrayList(java.util.ArrayList) ObjectListing(com.qcloud.cos.model.ObjectListing) ResponseNotCompleteException(com.qcloud.cos.exception.ResponseNotCompleteException) CosServiceException(com.qcloud.cos.exception.CosServiceException) CosClientException(com.qcloud.cos.exception.CosClientException) IOException(java.io.IOException) ListObjectsRequest(com.qcloud.cos.model.ListObjectsRequest)

Aggregations

COSObjectSummary (com.qcloud.cos.model.COSObjectSummary)7 ObjectListing (com.qcloud.cos.model.ObjectListing)7 ListObjectsRequest (com.qcloud.cos.model.ListObjectsRequest)5 CosClientException (com.qcloud.cos.exception.CosClientException)3 CosServiceException (com.qcloud.cos.exception.CosServiceException)3 COSClient (com.qcloud.cos.COSClient)2 ClientConfig (com.qcloud.cos.ClientConfig)2 BasicCOSCredentials (com.qcloud.cos.auth.BasicCOSCredentials)2 COSCredentials (com.qcloud.cos.auth.COSCredentials)2 Region (com.qcloud.cos.region.Region)2 ArrayList (java.util.ArrayList)2 Test (org.junit.Test)2 COSProgressListener (com.qcloud.cos.event.COSProgressListener)1 COSProgressListenerChain (com.qcloud.cos.event.COSProgressListenerChain)1 MultipleFileTransferProgressUpdatingListener (com.qcloud.cos.event.MultipleFileTransferProgressUpdatingListener)1 MultipleFileTransferStateChangeListener (com.qcloud.cos.event.MultipleFileTransferStateChangeListener)1 ProgressListener (com.qcloud.cos.event.ProgressListener)1 ProgressListenerChain (com.qcloud.cos.event.ProgressListenerChain)1 ResponseNotCompleteException (com.qcloud.cos.exception.ResponseNotCompleteException)1 BucketVersioningConfiguration (com.qcloud.cos.model.BucketVersioningConfiguration)1