Examples with ListObjectsRequest com.amazonaws.services.s3.model.ListObjectsRequest used on opensource projects

Example 26 with ListObjectsRequest

use of com.amazonaws.services.s3.model.ListObjectsRequest in project stocator by SparkTC.

the class COSAPIClient method getFileStatus.

@Override
public FileStatus getFileStatus(String hostName, Path path, String msg) throws IOException, FileNotFoundException {
    FileStatus res = null;
    FileStatus cached = memoryCache.getFileStatus(path.toString());
    if (cached != null) {
        return cached;
    }
    LOG.trace("getFileStatus(start) for {}, hostname: {}", path, hostName);
    /*
     * The requested path is equal to hostName. HostName is equal to
     * hostNameScheme, thus the container. Therefore we have no object to look
     * for and we return the FileStatus as a directory. Containers have to
     * lastModified.
     */
    if (path.toString().equals(hostName) || (path.toString().length() + 1 == hostName.length())) {
        LOG.trace("getFileStatus(completed) {}", path);
        res = new FileStatus(0L, true, 1, mBlockSize, 0L, path);
        memoryCache.putFileStatus(path.toString(), res);
        return res;
    }
    if (path.toString().contains(HADOOP_TEMPORARY)) {
        LOG.debug("getFileStatus on temp object {}. Return not found", path.toString());
        throw new FileNotFoundException("Not found " + path.toString());
    }
    String key = pathToKey(path);
    LOG.debug("getFileStatus: on original key {}", key);
    FileStatus fileStatus = null;
    try {
        fileStatus = getFileStatusKeyBased(key, path);
    } catch (AmazonS3Exception e) {
        LOG.warn("file status {} returned {}", key, e.getStatusCode());
        if (e.getStatusCode() != 404) {
            LOG.warn("Throw IOException for {}. Most likely authentication failed", key);
            throw new IOException(e);
        }
    }
    if (fileStatus != null) {
        LOG.trace("getFileStatus(completed) {}", path);
        memoryCache.putFileStatus(path.toString(), fileStatus);
        return fileStatus;
    }
    // probably not needed this call
    if (!key.endsWith("/")) {
        String newKey = key + "/";
        try {
            LOG.debug("getFileStatus: original key not found. Alternative key {}", key);
            fileStatus = getFileStatusKeyBased(newKey, path);
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() != 404) {
                throw new IOException(e);
            }
        }
        if (fileStatus != null) {
            LOG.trace("getFileStatus(completed) {}", path);
            memoryCache.putFileStatus(path.toString(), fileStatus);
            return fileStatus;
        } else {
            // if here: both key and key/ returned not found.
            // trying to see if pseudo directory of the form
            // a/b/key/d/e (a/b/key/ doesn't exists by itself)
            // perform listing on the key
            LOG.debug("getFileStatus: Modifined key {} not found. Trying to list", key);
            key = maybeAddTrailingSlash(key);
            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(mBucket);
            request.setPrefix(key);
            request.setDelimiter("/");
            request.setMaxKeys(1);
            ObjectListing objects = mClient.listObjects(request);
            if (!objects.getCommonPrefixes().isEmpty() || !objects.getObjectSummaries().isEmpty()) {
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            } else if (key.isEmpty()) {
                LOG.trace("Found root directory");
                LOG.debug("getFileStatus(completed) {}", path);
                res = new FileStatus(0, true, 1, 0, 0, path);
                memoryCache.putFileStatus(path.toString(), res);
                return res;
            }
        }
    }
    LOG.debug("Not found {}. Throw FNF exception", path.toString());
    throw new FileNotFoundException("Not found " + path.toString());
}