Examples with OpenKeyBucket org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket used on opensource projects

Example 1 with OpenKeyBucket

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket in project ozone by apache.

the class TestOMOpenKeysDeleteRequest method testDeleteSubsetOfOpenKeys.

/**
 * Tests adding multiple keys to the open key table, and updating the table
 * cache to only remove some of them.
 * Keys not removed should still be present in the open key table.
 * Mixes which keys will be kept and deleted among different volumes and
 * buckets.
 * @throws Exception
 */
@Test
public void testDeleteSubsetOfOpenKeys() throws Exception {
    final String volume1 = "volume1";
    final String volume2 = "bucket1";
    final String bucket1 = "volume2";
    final String bucket2 = "bucket2";
    OpenKeyBucket v1b1KeysToDelete = makeOpenKeys(volume1, bucket1, 3);
    OpenKeyBucket v1b1KeysToKeep = makeOpenKeys(volume1, bucket1, 3);
    OpenKeyBucket v1b2KeysToDelete = makeOpenKeys(volume1, bucket2, 3);
    OpenKeyBucket v1b2KeysToKeep = makeOpenKeys(volume1, bucket2, 3);
    OpenKeyBucket v2b2KeysToDelete = makeOpenKeys(volume2, bucket2, 3);
    OpenKeyBucket v2b2KeysToKeep = makeOpenKeys(volume2, bucket2, 3);
    addToOpenKeyTableDB(v1b1KeysToKeep, v1b2KeysToKeep, v2b2KeysToKeep, v1b1KeysToDelete, v1b2KeysToDelete, v2b2KeysToDelete);
    deleteOpenKeysFromCache(v1b1KeysToDelete, v1b2KeysToDelete, v2b2KeysToDelete);
    assertNotInOpenKeyTable(v1b1KeysToDelete, v1b2KeysToDelete, v2b2KeysToDelete);
    assertInOpenKeyTable(v1b1KeysToKeep, v1b2KeysToKeep, v2b2KeysToKeep);
}

Example 2 with OpenKeyBucket

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket in project ozone by apache.

the class TestOMOpenKeysDeleteRequest method makeOpenKeys.

/**
 * Constructs a list of {@link OpenKeyBucket} objects of size {@code numKeys}.
 * The keys created will all have the same volume and bucket, but
 * randomized key names and client IDs. These keys are not added to the
 * open key table.
 *
 * @param volume The volume all open keys created will have.
 * @param bucket The bucket all open keys created will have.
 * @param numKeys The number of keys with randomized key names and client
 * IDs to create.
 * @return A list of new open keys with size {@code numKeys}.
 */
private OpenKeyBucket makeOpenKeys(String volume, String bucket, int numKeys) {
    OpenKeyBucket.Builder keysPerBucketBuilder = OpenKeyBucket.newBuilder().setVolumeName(volume).setBucketName(bucket);
    for (int i = 0; i < numKeys; i++) {
        String keyName = UUID.randomUUID().toString();
        long clientID = random.nextLong();
        OpenKey openKey = OpenKey.newBuilder().setName(keyName).setClientID(clientID).build();
        keysPerBucketBuilder.addKeys(openKey);
    }
    return keysPerBucketBuilder.build();
}

Example 3 with OpenKeyBucket

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket in project ozone by apache.

the class OMOpenKeysDeleteRequest method validateAndUpdateCache.

@Override
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) {
    OMMetrics omMetrics = ozoneManager.getMetrics();
    omMetrics.incNumOpenKeyDeleteRequests();
    OzoneManagerProtocolProtos.DeleteOpenKeysRequest deleteOpenKeysRequest = getOmRequest().getDeleteOpenKeysRequest();
    List<OpenKeyBucket> submittedOpenKeyBuckets = deleteOpenKeysRequest.getOpenKeysPerBucketList();
    long numSubmittedOpenKeys = 0;
    for (OpenKeyBucket keyBucket : submittedOpenKeyBuckets) {
        numSubmittedOpenKeys += keyBucket.getKeysCount();
    }
    LOG.debug("{} open keys submitted for deletion.", numSubmittedOpenKeys);
    omMetrics.incNumOpenKeysSubmittedForDeletion(numSubmittedOpenKeys);
    OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest());
    IOException exception = null;
    OMClientResponse omClientResponse = null;
    Result result = null;
    Map<String, OmKeyInfo> deletedOpenKeys = new HashMap<>();
    try {
        for (OpenKeyBucket openKeyBucket : submittedOpenKeyBuckets) {
            // For each bucket where keys will be deleted from,
            // get its bucket lock and update the cache accordingly.
            Map<String, OmKeyInfo> deleted = updateOpenKeyTableCache(ozoneManager, trxnLogIndex, openKeyBucket);
            deletedOpenKeys.putAll(deleted);
        }
        omClientResponse = new OMOpenKeysDeleteResponse(omResponse.build(), deletedOpenKeys, ozoneManager.isRatisEnabled());
        result = Result.SUCCESS;
    } catch (IOException ex) {
        result = Result.FAILURE;
        exception = ex;
        omClientResponse = new OMOpenKeysDeleteResponse(createErrorOMResponse(omResponse, exception), getBucketLayout());
    } finally {
        addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, omDoubleBufferHelper);
    }
    processResults(omMetrics, numSubmittedOpenKeys, deletedOpenKeys.size(), deleteOpenKeysRequest, result);
    return omClientResponse;
}

Example 4 with OpenKeyBucket

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket in project ozone by apache.

the class TestOMOpenKeysDeleteRequest method testMetrics.

/**
 * Tests metrics set by {@link OMOpenKeysDeleteRequest}.
 * Submits a set of keys for deletion where only some of the keys actually
 * exist in the open key table, and asserts that the metrics count keys
 * that were submitted for deletion versus those that were actually deleted.
 * @throws Exception
 */
@Test
public void testMetrics() throws Exception {
    final int numExistentKeys = 3;
    final int numNonExistentKeys = 5;
    OMMetrics metrics = ozoneManager.getMetrics();
    Assert.assertEquals(metrics.getNumOpenKeyDeleteRequests(), 0);
    Assert.assertEquals(metrics.getNumOpenKeyDeleteRequestFails(), 0);
    Assert.assertEquals(metrics.getNumOpenKeysSubmittedForDeletion(), 0);
    Assert.assertEquals(metrics.getNumOpenKeysDeleted(), 0);
    OpenKeyBucket existentKeys = makeOpenKeys(volumeName, bucketName, keyName, numExistentKeys);
    OpenKeyBucket nonExistentKeys = makeOpenKeys(volumeName, bucketName, keyName, numNonExistentKeys);
    addToOpenKeyTableDB(existentKeys);
    deleteOpenKeysFromCache(existentKeys, nonExistentKeys);
    assertNotInOpenKeyTable(existentKeys);
    assertNotInOpenKeyTable(nonExistentKeys);
    Assert.assertEquals(1, metrics.getNumOpenKeyDeleteRequests());
    Assert.assertEquals(0, metrics.getNumOpenKeyDeleteRequestFails());
    Assert.assertEquals(numExistentKeys + numNonExistentKeys, metrics.getNumOpenKeysSubmittedForDeletion());
    Assert.assertEquals(numExistentKeys, metrics.getNumOpenKeysDeleted());
}

Example 5 with OpenKeyBucket

use of org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket in project ozone by apache.

the class TestOMOpenKeysDeleteRequest method getFullOpenKeyNames.

/**
 * Expands all the open keys represented by {@code openKeyBuckets} to their
 * full
 * key names as strings.
 * @param openKeyBuckets
 * @return
 */
private List<String> getFullOpenKeyNames(OpenKeyBucket... openKeyBuckets) {
    List<String> fullKeyNames = new ArrayList<>();
    for (OpenKeyBucket keysPerBucket : openKeyBuckets) {
        String volume = keysPerBucket.getVolumeName();
        String bucket = keysPerBucket.getBucketName();
        for (OpenKey openKey : keysPerBucket.getKeysList()) {
            String fullName = omMetadataManager.getOpenKey(volume, bucket, openKey.getName(), openKey.getClientID());
            fullKeyNames.add(fullName);
        }
    }
    return fullKeyNames;
}