Examples with ContinueToken org.apache.hadoop.ozone.s3.util.ContinueToken used on opensource projects

Example 1 with ContinueToken

use of org.apache.hadoop.ozone.s3.util.ContinueToken in project ozone by apache.

the class BucketEndpoint method get.

/**
 * Rest endpoint to list objects in a specific bucket.
 * <p>
 * See: https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
 * for more details.
 */
@GET
@SuppressFBWarnings
@SuppressWarnings("parameternumber")
public Response get(@PathParam("bucket") String bucketName, @QueryParam("delimiter") String delimiter, @QueryParam("encoding-type") String encodingType, @QueryParam("marker") String marker, @DefaultValue("1000") @QueryParam("max-keys") int maxKeys, @QueryParam("prefix") String prefix, @QueryParam("browser") String browser, @QueryParam("continuation-token") String continueToken, @QueryParam("start-after") String startAfter, @QueryParam("uploads") String uploads, @QueryParam("acl") String aclMarker, @Context HttpHeaders hh) throws OS3Exception, IOException {
    if (aclMarker != null) {
        S3BucketAcl result = getAcl(bucketName);
        return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build();
    }
    if (browser != null) {
        InputStream browserPage = getClass().getResourceAsStream("/browser.html");
        return Response.ok(browserPage, MediaType.TEXT_HTML_TYPE).build();
    }
    if (uploads != null) {
        return listMultipartUploads(bucketName, prefix);
    }
    if (prefix == null) {
        prefix = "";
    }
    OzoneBucket bucket = getBucket(bucketName);
    Iterator<? extends OzoneKey> ozoneKeyIterator;
    ContinueToken decodedToken = ContinueToken.decodeFromString(continueToken);
    // Assign marker to startAfter. for the compatibility of aws api v1
    if (startAfter == null && marker != null) {
        startAfter = marker;
    }
    try {
        if (startAfter != null && continueToken != null) {
            // If continuation token and start after both are provided, then we
            // ignore start After
            ozoneKeyIterator = bucket.listKeys(prefix, decodedToken.getLastKey());
        } else if (startAfter != null && continueToken == null) {
            ozoneKeyIterator = bucket.listKeys(prefix, startAfter);
        } else if (startAfter == null && continueToken != null) {
            ozoneKeyIterator = bucket.listKeys(prefix, decodedToken.getLastKey());
        } else {
            ozoneKeyIterator = bucket.listKeys(prefix);
        }
    } catch (OMException ex) {
        if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
            throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
        } else {
            throw ex;
        }
    }
    ListObjectResponse response = new ListObjectResponse();
    response.setDelimiter(delimiter);
    response.setName(bucketName);
    response.setPrefix(prefix);
    response.setMarker(marker == null ? "" : marker);
    response.setMaxKeys(maxKeys);
    response.setEncodingType(ENCODING_TYPE);
    response.setTruncated(false);
    response.setContinueToken(continueToken);
    String prevDir = null;
    if (continueToken != null) {
        prevDir = decodedToken.getLastDir();
    }
    String lastKey = null;
    int count = 0;
    while (ozoneKeyIterator.hasNext()) {
        OzoneKey next = ozoneKeyIterator.next();
        String relativeKeyName = next.getName().substring(prefix.length());
        int depth = StringUtils.countMatches(relativeKeyName, delimiter);
        if (delimiter != null) {
            if (depth > 0) {
                // means key has multiple delimiters in its value.
                // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
                String dirName = relativeKeyName.substring(0, relativeKeyName.indexOf(delimiter));
                if (!dirName.equals(prevDir)) {
                    response.addPrefix(prefix + dirName + delimiter);
                    prevDir = dirName;
                    count++;
                }
            } else if (relativeKeyName.endsWith(delimiter)) {
                // means or key is same as prefix with delimiter at end and ends with
                // delimiter. ex: dir/, where prefix is dir and delimiter is /
                response.addPrefix(relativeKeyName);
                count++;
            } else {
                // means our key is matched with prefix if prefix is given and it
                // does not have any common prefix.
                addKey(response, next);
                count++;
            }
        } else {
            addKey(response, next);
            count++;
        }
        if (count == maxKeys) {
            lastKey = next.getName();
            break;
        }
    }
    response.setKeyCount(count);
    if (count < maxKeys) {
        response.setTruncated(false);
    } else if (ozoneKeyIterator.hasNext()) {
        response.setTruncated(true);
        ContinueToken nextToken = new ContinueToken(lastKey, prevDir);
        response.setNextToken(nextToken.encodeToString());
        // Set nextMarker to be lastKey. for the compatibility of aws api v1
        response.setNextMarker(lastKey);
    } else {
        response.setTruncated(false);
    }
    response.setKeyCount(response.getCommonPrefixes().size() + response.getContents().size());
    return Response.ok(response).build();
}

Example 2 with ContinueToken

use of org.apache.hadoop.ozone.s3.util.ContinueToken in project ozone by apache.

the class ContinueToken method decodeFromString.

/**
 * Decode a continuation token which is used in get Bucket.
 *
 * @param key
 * @return if key is not null return decoded token, otherwise returns null.
 * @throws OS3Exception
 */
public static ContinueToken decodeFromString(String key) throws OS3Exception {
    if (key != null) {
        int indexSeparator = key.indexOf(CONTINUE_TOKEN_SEPARATOR);
        if (indexSeparator == -1) {
            throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, key);
        }
        String hex = key.substring(0, indexSeparator);
        String digest = key.substring(indexSeparator + 1);
        try {
            checkHash(key, hex, digest);
            ByteBuffer buffer = ByteBuffer.wrap(Hex.decodeHex(hex));
            int keySize = buffer.getInt();
            byte[] actualKeyBytes = new byte[keySize];
            buffer.get(actualKeyBytes);
            byte[] actualDirBytes = new byte[buffer.remaining()];
            buffer.get(actualDirBytes);
            return new ContinueToken(new String(actualKeyBytes, StandardCharsets.UTF_8), new String(actualDirBytes, StandardCharsets.UTF_8));
        } catch (DecoderException ex) {
            OS3Exception os3Exception = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, key, ex);
            os3Exception.setErrorMessage("The continuation token provided is " + "incorrect");
            throw os3Exception;
        }
    } else {
        return null;
    }
}