Examples with ObjectListing com.aliyun.oss.model.ObjectListing used on opensource projects

Example 1 with ObjectListing

use of com.aliyun.oss.model.ObjectListing in project hadoop by apache.

the class AliyunOSSFileSystemStore method purge.

/**
   * Clean up all objects matching the prefix.
   *
   * @param prefix Aliyun OSS object prefix.
   * @throws IOException if failed to clean up objects.
   */
public void purge(String prefix) throws IOException {
    String key;
    try {
        ObjectListing objects = listObjects(prefix, maxKeys, null, true);
        for (OSSObjectSummary object : objects.getObjectSummaries()) {
            key = object.getKey();
            ossClient.deleteObject(bucketName, key);
        }
        for (String dir : objects.getCommonPrefixes()) {
            deleteDirs(dir);
        }
    } catch (OSSException | ClientException e) {
        LOG.error("Failed to purge " + prefix);
    }
}

Example 2 with ObjectListing

use of com.aliyun.oss.model.ObjectListing in project hadoop by apache.

the class AliyunOSSFileSystem method listStatus.

@Override
public FileStatus[] listStatus(Path path) throws IOException {
    String key = pathToKey(path);
    if (LOG.isDebugEnabled()) {
        LOG.debug("List status for path: " + path);
    }
    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(path);
    if (fileStatus.isDirectory()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }
        ObjectListing objects = store.listObjects(key, maxKeys, null, false);
        while (true) {
            statistics.incrementReadOps(1);
            for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
                String objKey = objectSummary.getKey();
                if (objKey.equals(key + "/")) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + objKey);
                    }
                    continue;
                } else {
                    Path keyPath = keyToPath(objectSummary.getKey()).makeQualified(uri, workingDir);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                    result.add(new FileStatus(objectSummary.getSize(), false, 1, getDefaultBlockSize(keyPath), objectSummary.getLastModified().getTime(), keyPath));
                }
            }
            for (String prefix : objects.getCommonPrefixes()) {
                if (prefix.equals(key + "/")) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + prefix);
                    }
                    continue;
                } else {
                    Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: rd: " + keyPath);
                    }
                    result.add(getFileStatus(keyPath));
                }
            }
            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }
                String nextMarker = objects.getNextMarker();
                objects = store.listObjects(key, maxKeys, nextMarker, false);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + path);
        }
        result.add(fileStatus);
    }
    return result.toArray(new FileStatus[result.size()]);
}

Example 3 with ObjectListing

use of com.aliyun.oss.model.ObjectListing in project hadoop by apache.

the class AliyunOSSFileSystem method getFileStatus.

@Override
public FileStatus getFileStatus(Path path) throws IOException {
    Path qualifiedPath = path.makeQualified(uri, workingDir);
    String key = pathToKey(qualifiedPath);
    // Root always exists
    if (key.length() == 0) {
        return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
    }
    ObjectMetadata meta = store.getObjectMetadata(key);
    // If key not found and key does not end with "/"
    if (meta == null && !key.endsWith("/")) {
        // In case of 'dir + "/"'
        key += "/";
        meta = store.getObjectMetadata(key);
    }
    if (meta == null) {
        ObjectListing listing = store.listObjects(key, 1, null, false);
        if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) || CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) {
            return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
        } else {
            throw new FileNotFoundException(path + ": No such file or directory!");
        }
    } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
        return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(), qualifiedPath);
    } else {
        return new FileStatus(meta.getContentLength(), false, 1, getDefaultBlockSize(path), meta.getLastModified().getTime(), qualifiedPath);
    }
}

Example 4 with ObjectListing

use of com.aliyun.oss.model.ObjectListing in project alluxio by Alluxio.

the class OSSUnderFileSystem method getObjectListingChunk.

@Override
protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException {
    String delimiter = recursive ? "" : PATH_SEPARATOR;
    key = PathUtils.normalizePath(key, PATH_SEPARATOR);
    // In case key is root (empty string) do not normalize prefix
    key = key.equals(PATH_SEPARATOR) ? "" : key;
    ListObjectsRequest request = new ListObjectsRequest(mBucketName);
    request.setPrefix(key);
    request.setMaxKeys(getListingChunkLength());
    request.setDelimiter(delimiter);
    ObjectListing result = getObjectListingChunk(request);
    if (result != null) {
        return new OSSObjectListingChunk(request, result);
    }
    return null;
}

Example 5 with ObjectListing

use of com.aliyun.oss.model.ObjectListing in project hadoop by apache.

the class AliyunOSSFileSystem method copyDirectory.

/**
   * Copy a directory from source path to destination path.
   * (the caller should make sure srcPath is a directory, and dstPath is valid)
   *
   * @param srcPath source path.
   * @param dstPath destination path.
   * @return true if directory is successfully copied.
   */
private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
    String srcKey = AliyunOSSUtils.maybeAddTrailingSlash(pathToKey(srcPath));
    String dstKey = AliyunOSSUtils.maybeAddTrailingSlash(pathToKey(dstPath));
    if (dstKey.startsWith(srcKey)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Cannot rename a directory to a subdirectory of self");
        }
        return false;
    }
    store.storeEmptyFile(dstKey);
    ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true);
    statistics.incrementReadOps(1);
    // Copy files from src folder to dst
    while (true) {
        for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
            String newKey = dstKey.concat(objectSummary.getKey().substring(srcKey.length()));
            store.copyFile(objectSummary.getKey(), newKey);
        }
        if (objects.isTruncated()) {
            String nextMarker = objects.getNextMarker();
            objects = store.listObjects(srcKey, maxKeys, nextMarker, true);
            statistics.incrementReadOps(1);
        } else {
            break;
        }
    }
    return true;
}