Examples with OSSObjectSummary com.aliyun.oss.model.OSSObjectSummary used on opensource projects

Example 1 with OSSObjectSummary

use of com.aliyun.oss.model.OSSObjectSummary in project hadoop by apache.

the class AliyunOSSFileSystemStore method purge.

/**
   * Clean up all objects matching the prefix.
   *
   * @param prefix Aliyun OSS object prefix.
   * @throws IOException if failed to clean up objects.
   */
public void purge(String prefix) throws IOException {
    String key;
    try {
        ObjectListing objects = listObjects(prefix, maxKeys, null, true);
        for (OSSObjectSummary object : objects.getObjectSummaries()) {
            key = object.getKey();
            ossClient.deleteObject(bucketName, key);
        }
        for (String dir : objects.getCommonPrefixes()) {
            deleteDirs(dir);
        }
    } catch (OSSException | ClientException e) {
        LOG.error("Failed to purge " + prefix);
    }
}

Example 2 with OSSObjectSummary

use of com.aliyun.oss.model.OSSObjectSummary in project hadoop by apache.

the class AliyunOSSFileSystem method listStatus.

@Override
public FileStatus[] listStatus(Path path) throws IOException {
    String key = pathToKey(path);
    if (LOG.isDebugEnabled()) {
        LOG.debug("List status for path: " + path);
    }
    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(path);
    if (fileStatus.isDirectory()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }
        ObjectListing objects = store.listObjects(key, maxKeys, null, false);
        while (true) {
            statistics.incrementReadOps(1);
            for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
                String objKey = objectSummary.getKey();
                if (objKey.equals(key + "/")) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + objKey);
                    }
                    continue;
                } else {
                    Path keyPath = keyToPath(objectSummary.getKey()).makeQualified(uri, workingDir);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                    result.add(new FileStatus(objectSummary.getSize(), false, 1, getDefaultBlockSize(keyPath), objectSummary.getLastModified().getTime(), keyPath));
                }
            }
            for (String prefix : objects.getCommonPrefixes()) {
                if (prefix.equals(key + "/")) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + prefix);
                    }
                    continue;
                } else {
                    Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: rd: " + keyPath);
                    }
                    result.add(getFileStatus(keyPath));
                }
            }
            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }
                String nextMarker = objects.getNextMarker();
                objects = store.listObjects(key, maxKeys, nextMarker, false);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + path);
        }
        result.add(fileStatus);
    }
    return result.toArray(new FileStatus[result.size()]);
}

Example 3 with OSSObjectSummary

use of com.aliyun.oss.model.OSSObjectSummary in project hadoop by apache.

the class AliyunOSSFileSystem method copyDirectory.

/**
   * Copy a directory from source path to destination path.
   * (the caller should make sure srcPath is a directory, and dstPath is valid)
   *
   * @param srcPath source path.
   * @param dstPath destination path.
   * @return true if directory is successfully copied.
   */
private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
    String srcKey = AliyunOSSUtils.maybeAddTrailingSlash(pathToKey(srcPath));
    String dstKey = AliyunOSSUtils.maybeAddTrailingSlash(pathToKey(dstPath));
    if (dstKey.startsWith(srcKey)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Cannot rename a directory to a subdirectory of self");
        }
        return false;
    }
    store.storeEmptyFile(dstKey);
    ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true);
    statistics.incrementReadOps(1);
    // Copy files from src folder to dst
    while (true) {
        for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
            String newKey = dstKey.concat(objectSummary.getKey().substring(srcKey.length()));
            store.copyFile(objectSummary.getKey(), newKey);
        }
        if (objects.isTruncated()) {
            String nextMarker = objects.getNextMarker();
            objects = store.listObjects(srcKey, maxKeys, nextMarker, true);
            statistics.incrementReadOps(1);
        } else {
            break;
        }
    }
    return true;
}

Example 4 with OSSObjectSummary

use of com.aliyun.oss.model.OSSObjectSummary in project hadoop by apache.

the class AliyunOSSFileSystemStore method deleteDirs.

/**
   * Delete a directory from Aliyun OSS.
   *
   * @param key directory key to delete.
   * @throws IOException if failed to delete directory.
   */
public void deleteDirs(String key) throws IOException {
    key = AliyunOSSUtils.maybeAddTrailingSlash(key);
    ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
    listRequest.setPrefix(key);
    listRequest.setDelimiter(null);
    listRequest.setMaxKeys(maxKeys);
    while (true) {
        ObjectListing objects = ossClient.listObjects(listRequest);
        statistics.incrementReadOps(1);
        List<String> keysToDelete = new ArrayList<String>();
        for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
            keysToDelete.add(objectSummary.getKey());
        }
        deleteObjects(keysToDelete);
        if (objects.isTruncated()) {
            listRequest.setMarker(objects.getNextMarker());
        } else {
            break;
        }
    }
}