Examples with ObjectListing com.qcloud.cos.model.ObjectListing used on opensource projects

Example 6 with ObjectListing

use of com.qcloud.cos.model.ObjectListing in project cos-java-sdk-v5 by tencentyun.

the class AbstractCOSClientTest method clearBucket.

private static void clearBucket() throws Exception {
    abortAllNotFinishedMultipartUpload();
    // 先判断bucket是否开启了版本控制
    GetBucketVersioningConfigurationRequest getBucketVersioningConfigurationRequest = new GetBucketVersioningConfigurationRequest(bucket);
    BucketVersioningConfiguration bucketVersioningConfiguration = cosclient.getBucketVersioningConfiguration(getBucketVersioningConfigurationRequest);
    if (bucketVersioningConfiguration.getStatus().compareToIgnoreCase(BucketVersioningConfiguration.ENABLED) == 0) {
        clearObjectVersions();
    }
    String nextMarker = "";
    boolean isTruncated = false;
    do {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucket);
        listObjectsRequest.setMaxKeys(1000);
        listObjectsRequest.setPrefix("");
        listObjectsRequest.setDelimiter("");
        listObjectsRequest.setMarker(nextMarker);
        ObjectListing objectListing = cosclient.listObjects(listObjectsRequest);
        for (COSObjectSummary cosObjectSummary : objectListing.getObjectSummaries()) {
            String key = cosObjectSummary.getKey();
            // 删除这个key
            System.out.println(key);
            cosclient.deleteObject(bucket, key);
        }
        nextMarker = objectListing.getNextMarker();
        isTruncated = objectListing.isTruncated();
    } while (isTruncated);
}

Example 7 with ObjectListing

use of com.qcloud.cos.model.ObjectListing in project hadoop-cos by tencentyun.

the class CosNativeFileSystemStore method list.

/**
 * list objects
 *
 * @param prefix           prefix
 * @param delimiter        delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey     last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */
private CosNPartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey, CosNResultInfo info) throws IOException {
    LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.", prefix, delimiter, maxListingLength, priorLastKey);
    if (!prefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
        prefix += CosNFileSystem.PATH_DELIMITER;
    }
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(bucketName);
    listObjectsRequest.setPrefix(prefix);
    listObjectsRequest.setDelimiter(delimiter);
    listObjectsRequest.setMarker(priorLastKey);
    listObjectsRequest.setMaxKeys(maxListingLength);
    ObjectListing objectListing = null;
    try {
        objectListing = (ObjectListing) callCOSClientWithRetry(listObjectsRequest);
    } catch (Exception e) {
        String errMsg = String.format("prefix: %s, delimiter: %s, maxListingLength: %d, " + "priorLastKey: %s. list occur a exception: %s", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey, e);
        LOG.error(errMsg);
        handleException(new Exception(errMsg), prefix);
    }
    if (null == objectListing) {
        String errMessage = String.format("List objects failed for the prefix: %s, delimiter: %s, " + "maxListingLength:%d, priorLastKey: %s.", prefix, (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey);
        handleException(new Exception(errMessage), prefix);
    }
    ArrayList<FileMetadata> fileMetadataArray = new ArrayList<>();
    ArrayList<FileMetadata> commonPrefixArray = new ArrayList<>();
    List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
    boolean isKeySamePrefix = false;
    for (COSObjectSummary cosObjectSummary : summaries) {
        String filePath = cosObjectSummary.getKey();
        if (!filePath.startsWith(CosNFileSystem.PATH_DELIMITER)) {
            filePath = CosNFileSystem.PATH_DELIMITER + filePath;
        }
        if (filePath.equals(prefix)) {
            isKeySamePrefix = true;
            continue;
        }
        long mtime = 0;
        if (cosObjectSummary.getLastModified() != null) {
            mtime = cosObjectSummary.getLastModified().getTime();
        }
        long fileLen = cosObjectSummary.getSize();
        String fileEtag = cosObjectSummary.getETag();
        if (cosObjectSummary.getKey().endsWith(CosNFileSystem.PATH_DELIMITER) && cosObjectSummary.getSize() == 0) {
            fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, false, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
        } else {
            fileMetadataArray.add(new FileMetadata(filePath, fileLen, mtime, true, fileEtag, null, null, null, cosObjectSummary.getStorageClass()));
        }
    }
    List<String> commonPrefixes = objectListing.getCommonPrefixes();
    for (String commonPrefix : commonPrefixes) {
        if (!commonPrefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
            commonPrefix = CosNFileSystem.PATH_DELIMITER + commonPrefix;
        }
        commonPrefixArray.add(new FileMetadata(commonPrefix, 0, 0, false));
    }
    FileMetadata[] fileMetadata = new FileMetadata[fileMetadataArray.size()];
    for (int i = 0; i < fileMetadataArray.size(); ++i) {
        fileMetadata[i] = fileMetadataArray.get(i);
    }
    FileMetadata[] commonPrefixMetaData = new FileMetadata[commonPrefixArray.size()];
    for (int i = 0; i < commonPrefixArray.size(); ++i) {
        commonPrefixMetaData[i] = commonPrefixArray.get(i);
    }
    // 如果truncated为false, 则表明已经遍历完
    if (!objectListing.isTruncated()) {
        CosNPartialListing ret = new CosNPartialListing(null, fileMetadata, commonPrefixMetaData);
        if (info != null) {
            info.setRequestID(objectListing.getRequestId());
            info.setKeySameToPrefix(isKeySamePrefix);
        }
        return ret;
    } else {
        CosNPartialListing ret = new CosNPartialListing(objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData);
        if (info != null) {
            info.setRequestID(objectListing.getRequestId());
            info.setKeySameToPrefix(isKeySamePrefix);
        }
        return ret;
    }
}

Example 8 with ObjectListing

use of com.qcloud.cos.model.ObjectListing in project cos-java-sdk-v5 by tencentyun.

the class ListObjectsDemo method listObjectsDemo.

public static void listObjectsDemo() {
    // 1 初始化用户身份信息(appid, secretId, secretKey)
    COSCredentials cred = new BasicCOSCredentials("AKIDXXXXXXXX", "1A2Z3YYYYYYYYYY");
    // 2 设置bucket的区域, COS地域的简称请参照 https://www.qcloud.com/document/product/436/6224
    ClientConfig clientConfig = new ClientConfig(new Region("ap-beijing-1"));
    // 3 生成cos客户端
    COSClient cosclient = new COSClient(cred, clientConfig);
    // bucket名称, 需包含appid
    String bucketName = "mybucket-1251668577";
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    // 设置bucket名称
    listObjectsRequest.setBucketName(bucketName);
    // prefix表示列出的object的key以prefix开始
    listObjectsRequest.setPrefix("");
    // 设置最大遍历出多少个对象, 一次listobject最大支持1000
    listObjectsRequest.setMaxKeys(1000);
    // listObjectsRequest.setDelimiter("/");
    ObjectListing objectListing = null;
    try {
        objectListing = cosclient.listObjects(listObjectsRequest);
    } catch (CosServiceException e) {
        e.printStackTrace();
    } catch (CosClientException e) {
        e.printStackTrace();
    }
    // common prefix表示表示被delimiter截断的路径, 如delimter设置为/, common prefix则表示所有子目录的路径
    List<String> commonPrefixs = objectListing.getCommonPrefixes();
    // object summary表示所有列出的object列表
    List<COSObjectSummary> cosObjectSummaries = objectListing.getObjectSummaries();
    for (COSObjectSummary cosObjectSummary : cosObjectSummaries) {
        // 文件的路径key
        String key = cosObjectSummary.getKey();
        // 文件的etag
        String etag = cosObjectSummary.getETag();
        // 文件的长度
        long fileSize = cosObjectSummary.getSize();
        // 文件的存储类型
        String storageClasses = cosObjectSummary.getStorageClass();
        System.out.println("key: " + key);
    }
    cosclient.shutdown();
}

Example 9 with ObjectListing

use of com.qcloud.cos.model.ObjectListing in project cos-java-sdk-v5 by tencentyun.

the class TransferManager method downloadDirectory.

/**
 * Downloads all objects in the virtual directory designated by the keyPrefix given to the
 * destination directory given. All virtual subdirectories will be downloaded recursively.
 *
 * @param bucketName The bucket containing the virtual directory
 * @param keyPrefix The key prefix for the virtual directory, or null for the entire bucket. All
 *        subdirectories will be downloaded recursively.
 * @param destinationDirectory The directory to place downloaded files. Subdirectories will be
 *        created as necessary.
 */
public MultipleFileDownload downloadDirectory(String bucketName, String keyPrefix, File destinationDirectory) {
    if (keyPrefix == null)
        keyPrefix = "";
    List<COSObjectSummary> objectSummaries = new LinkedList<COSObjectSummary>();
    Stack<String> commonPrefixes = new Stack<String>();
    commonPrefixes.add(keyPrefix);
    long totalSize = 0;
    // This is a depth-first search.
    do {
        String prefix = commonPrefixes.pop();
        ObjectListing listObjectsResponse = null;
        do {
            if (listObjectsResponse == null) {
                ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix);
                listObjectsResponse = cos.listObjects(listObjectsRequest);
            } else {
                listObjectsResponse = cos.listNextBatchOfObjects(listObjectsResponse);
            }
            for (COSObjectSummary s : listObjectsResponse.getObjectSummaries()) {
                // name.
                if (!s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER)) {
                    objectSummaries.add(s);
                    totalSize += s.getSize();
                } else {
                    log.debug("Skipping download for object " + s.getKey() + " since it is also a virtual directory");
                }
            }
            commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes());
        } while (listObjectsResponse.isTruncated());
    } while (!commonPrefixes.isEmpty());
    /* This is the hook for adding additional progress listeners */
    ProgressListenerChain additionalListeners = new ProgressListenerChain();
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(totalSize);
    /*
         * Bind additional progress listeners to this MultipleFileTransferProgressUpdatingListener
         * to receive ByteTransferred events from each single-file download implementation.
         */
    ProgressListener listener = new MultipleFileTransferProgressUpdatingListener(transferProgress, additionalListeners);
    List<DownloadImpl> downloads = new ArrayList<DownloadImpl>();
    String description = "Downloading from " + bucketName + "/" + keyPrefix;
    final MultipleFileDownloadImpl multipleFileDownload = new MultipleFileDownloadImpl(description, transferProgress, additionalListeners, keyPrefix, bucketName, downloads);
    multipleFileDownload.setMonitor(new MultipleFileTransferMonitor(multipleFileDownload, downloads));
    final CountDownLatch latch = new CountDownLatch(1);
    MultipleFileTransferStateChangeListener transferListener = new MultipleFileTransferStateChangeListener(latch, multipleFileDownload);
    for (COSObjectSummary summary : objectSummaries) {
        // TODO: non-standard delimiters
        File f = new File(destinationDirectory, summary.getKey());
        File parentFile = f.getParentFile();
        if (parentFile == null || !parentFile.exists() && !parentFile.mkdirs()) {
            throw new RuntimeException("Couldn't create parent directories for " + f.getAbsolutePath());
        }
        // All the single-file downloads share the same
        // MultipleFileTransferProgressUpdatingListener and
        // MultipleFileTransferStateChangeListener
        downloads.add((DownloadImpl) doDownload(new GetObjectRequest(summary.getBucketName(), summary.getKey()).<GetObjectRequest>withGeneralProgressListener(listener), f, transferListener, null, false));
    }
    if (downloads.isEmpty()) {
        multipleFileDownload.setState(TransferState.Completed);
        return multipleFileDownload;
    }
    // Notify all state changes waiting for the downloads to all be queued
    // to wake up and continue.
    latch.countDown();
    return multipleFileDownload;
}

Example 10 with ObjectListing

use of com.qcloud.cos.model.ObjectListing in project cos-java-sdk-v5 by tencentyun.

the class ListObjectTest method ListObjectStartWithDelimiterTest.

@Test
public void ListObjectStartWithDelimiterTest() {
    if (!judgeUserInfoValid()) {
        return;
    }
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucket, keyPrefix, null, "/", 100);
    ObjectListing objectListing = cosclient.listObjects(listObjectsRequest);
    assertEquals(1L, objectListing.getCommonPrefixes().size());
    assertEquals(0L, objectListing.getObjectSummaries().size());
}