Examples with OzoneKey org.apache.hadoop.ozone.client.OzoneKey used on opensource projects

Example 1 with OzoneKey

use of org.apache.hadoop.ozone.client.OzoneKey in project ozone by apache.

the class RpcClient method headObject.

@Override
public OzoneKey headObject(String volumeName, String bucketName, String keyName) throws IOException {
    Preconditions.checkNotNull(volumeName);
    Preconditions.checkNotNull(bucketName);
    Preconditions.checkNotNull(keyName);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setLatestVersionLocation(true).setHeadOp(true).build();
    OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
    return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), keyInfo.getReplicationConfig());
}

Example 2 with OzoneKey

use of org.apache.hadoop.ozone.client.OzoneKey in project ozone by apache.

the class TestStorageContainerManagerHA method testPutKey.

public void testPutKey() throws Exception {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    Instant testStartTime = Instant.now();
    ObjectStore store = OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
    String value = "sample value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    String keyName = UUID.randomUUID().toString();
    OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, RATIS, ONE, new HashMap<>());
    out.write(value.getBytes(UTF_8));
    out.close();
    OzoneKey key = bucket.getKey(keyName);
    Assert.assertEquals(keyName, key.getName());
    OzoneInputStream is = bucket.readKey(keyName);
    byte[] fileContent = new byte[value.getBytes(UTF_8).length];
    is.read(fileContent);
    Assert.assertEquals(value, new String(fileContent, UTF_8));
    Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
    Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
    is.close();
    final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setKeyName(keyName).setRefreshPipeline(true).build();
    final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
    final List<OmKeyLocationInfo> keyLocationInfos = keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
    long index = -1;
    for (StorageContainerManager scm : cluster.getStorageContainerManagers()) {
        if (scm.checkLeader()) {
            index = getLastAppliedIndex(scm);
        }
    }
    Assert.assertFalse(index == -1);
    long finalIndex = index;
    // Ensure all follower scms have caught up with the leader
    GenericTestUtils.waitFor(() -> areAllScmInSync(finalIndex), 100, 10000);
    final long containerID = keyLocationInfos.get(0).getContainerID();
    for (int k = 0; k < numOfSCMs; k++) {
        StorageContainerManager scm = cluster.getStorageContainerManagers().get(k);
        // flush to DB on each SCM
        ((SCMRatisServerImpl) scm.getScmHAManager().getRatisServer()).getStateMachine().takeSnapshot();
        Assert.assertTrue(scm.getContainerManager().containerExist(ContainerID.valueOf(containerID)));
        Assert.assertNotNull(scm.getScmMetadataStore().getContainerTable().get(ContainerID.valueOf(containerID)));
    }
}

Example 3 with OzoneKey

use of org.apache.hadoop.ozone.client.OzoneKey in project ozone by apache.

the class ObjectEndpoint method head.

/**
 * Rest endpoint to check existence of an object in a bucket.
 * <p>
 * See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
 * for more details.
 */
@HEAD
public Response head(@PathParam("bucket") String bucketName, @PathParam("path") String keyPath) throws IOException, OS3Exception {
    OzoneKey key;
    try {
        key = getBucket(bucketName).headObject(keyPath);
    // TODO: return the specified range bytes of this object.
    } catch (OMException ex) {
        if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
            // Just return 404 with no content
            return Response.status(Status.NOT_FOUND).build();
        } else if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
            throw newError(S3ErrorTable.ACCESS_DENIED, keyPath, ex);
        } else {
            throw ex;
        }
    }
    ResponseBuilder response = Response.ok().status(HttpStatus.SC_OK).header("ETag", "" + key.getModificationTime()).header("Content-Length", key.getDataSize()).header("Content-Type", "binary/octet-stream");
    addLastModifiedDate(response, key);
    return response.build();
}

Example 4 with OzoneKey

use of org.apache.hadoop.ozone.client.OzoneKey in project ozone by apache.

the class BucketEndpoint method get.

/**
 * Rest endpoint to list objects in a specific bucket.
 * <p>
 * See: https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
 * for more details.
 */
@GET
@SuppressFBWarnings
@SuppressWarnings("parameternumber")
public Response get(@PathParam("bucket") String bucketName, @QueryParam("delimiter") String delimiter, @QueryParam("encoding-type") String encodingType, @QueryParam("marker") String marker, @DefaultValue("1000") @QueryParam("max-keys") int maxKeys, @QueryParam("prefix") String prefix, @QueryParam("browser") String browser, @QueryParam("continuation-token") String continueToken, @QueryParam("start-after") String startAfter, @QueryParam("uploads") String uploads, @QueryParam("acl") String aclMarker, @Context HttpHeaders hh) throws OS3Exception, IOException {
    if (aclMarker != null) {
        S3BucketAcl result = getAcl(bucketName);
        return Response.ok(result, MediaType.APPLICATION_XML_TYPE).build();
    }
    if (browser != null) {
        InputStream browserPage = getClass().getResourceAsStream("/browser.html");
        return Response.ok(browserPage, MediaType.TEXT_HTML_TYPE).build();
    }
    if (uploads != null) {
        return listMultipartUploads(bucketName, prefix);
    }
    if (prefix == null) {
        prefix = "";
    }
    OzoneBucket bucket = getBucket(bucketName);
    Iterator<? extends OzoneKey> ozoneKeyIterator;
    ContinueToken decodedToken = ContinueToken.decodeFromString(continueToken);
    // Assign marker to startAfter. for the compatibility of aws api v1
    if (startAfter == null && marker != null) {
        startAfter = marker;
    }
    try {
        if (startAfter != null && continueToken != null) {
            // If continuation token and start after both are provided, then we
            // ignore start After
            ozoneKeyIterator = bucket.listKeys(prefix, decodedToken.getLastKey());
        } else if (startAfter != null && continueToken == null) {
            ozoneKeyIterator = bucket.listKeys(prefix, startAfter);
        } else if (startAfter == null && continueToken != null) {
            ozoneKeyIterator = bucket.listKeys(prefix, decodedToken.getLastKey());
        } else {
            ozoneKeyIterator = bucket.listKeys(prefix);
        }
    } catch (OMException ex) {
        if (ex.getResult() == ResultCodes.PERMISSION_DENIED) {
            throw newError(S3ErrorTable.ACCESS_DENIED, bucketName, ex);
        } else {
            throw ex;
        }
    }
    ListObjectResponse response = new ListObjectResponse();
    response.setDelimiter(delimiter);
    response.setName(bucketName);
    response.setPrefix(prefix);
    response.setMarker(marker == null ? "" : marker);
    response.setMaxKeys(maxKeys);
    response.setEncodingType(ENCODING_TYPE);
    response.setTruncated(false);
    response.setContinueToken(continueToken);
    String prevDir = null;
    if (continueToken != null) {
        prevDir = decodedToken.getLastDir();
    }
    String lastKey = null;
    int count = 0;
    while (ozoneKeyIterator.hasNext()) {
        OzoneKey next = ozoneKeyIterator.next();
        String relativeKeyName = next.getName().substring(prefix.length());
        int depth = StringUtils.countMatches(relativeKeyName, delimiter);
        if (delimiter != null) {
            if (depth > 0) {
                // means key has multiple delimiters in its value.
                // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
                String dirName = relativeKeyName.substring(0, relativeKeyName.indexOf(delimiter));
                if (!dirName.equals(prevDir)) {
                    response.addPrefix(prefix + dirName + delimiter);
                    prevDir = dirName;
                    count++;
                }
            } else if (relativeKeyName.endsWith(delimiter)) {
                // means or key is same as prefix with delimiter at end and ends with
                // delimiter. ex: dir/, where prefix is dir and delimiter is /
                response.addPrefix(relativeKeyName);
                count++;
            } else {
                // means our key is matched with prefix if prefix is given and it
                // does not have any common prefix.
                addKey(response, next);
                count++;
            }
        } else {
            addKey(response, next);
            count++;
        }
        if (count == maxKeys) {
            lastKey = next.getName();
            break;
        }
    }
    response.setKeyCount(count);
    if (count < maxKeys) {
        response.setTruncated(false);
    } else if (ozoneKeyIterator.hasNext()) {
        response.setTruncated(true);
        ContinueToken nextToken = new ContinueToken(lastKey, prevDir);
        response.setNextToken(nextToken.encodeToString());
        // Set nextMarker to be lastKey. for the compatibility of aws api v1
        response.setNextMarker(lastKey);
    } else {
        response.setTruncated(false);
    }
    response.setKeyCount(response.getCommonPrefixes().size() + response.getContents().size());
    return Response.ok(response).build();
}

Example 5 with OzoneKey

use of org.apache.hadoop.ozone.client.OzoneKey in project ozone by apache.

the class TestOzoneRpcClientAbstract method testPutKeyRatisThreeNodes.

@Test
public void testPutKeyRatisThreeNodes() throws IOException {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    Instant testStartTime = Instant.now();
    String value = "sample value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    for (int i = 0; i < 10; i++) {
        String keyName = UUID.randomUUID().toString();
        OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS, THREE, new HashMap<>());
        out.write(value.getBytes(UTF_8));
        out.close();
        OzoneKey key = bucket.getKey(keyName);
        Assert.assertEquals(keyName, key.getName());
        OzoneInputStream is = bucket.readKey(keyName);
        byte[] fileContent = new byte[value.getBytes(UTF_8).length];
        is.read(fileContent);
        is.close();
        Assert.assertTrue(verifyRatisReplication(volumeName, bucketName, keyName, ReplicationType.RATIS, THREE));
        Assert.assertEquals(value, new String(fileContent, UTF_8));
        Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
        Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
    }
}