Examples with RepeatedOmKeyInfo org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo used on opensource projects

Example 1 with RepeatedOmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo in project ozone by apache.

the class OmUtils method prepareKeyForDelete.

/**
 * Prepares key info to be moved to deletedTable.
 * 1. It strips GDPR metadata from key info
 * 2. For given object key, if the repeatedOmKeyInfo instance is null, it
 * implies that no entry for the object key exists in deletedTable so we
 * create a new instance to include this key, else we update the existing
 * repeatedOmKeyInfo instance.
 * 3. Set the updateID to the transactionLogIndex.
 * @param keyInfo args supplied by client
 * @param repeatedOmKeyInfo key details from deletedTable
 * @param trxnLogIndex For Multipart keys, this is the transactionLogIndex
 *                     of the MultipartUploadAbort request which needs to
 *                     be set as the updateID of the partKeyInfos.
 *                     For regular Key deletes, this value should be set to
 *                     the same updateID as is in keyInfo.
 * @return {@link RepeatedOmKeyInfo}
 */
public static RepeatedOmKeyInfo prepareKeyForDelete(OmKeyInfo keyInfo, RepeatedOmKeyInfo repeatedOmKeyInfo, long trxnLogIndex, boolean isRatisEnabled) {
    // FileEncryptionInfo from KeyInfo.
    if (Boolean.valueOf(keyInfo.getMetadata().get(OzoneConsts.GDPR_FLAG))) {
        keyInfo.getMetadata().remove(OzoneConsts.GDPR_FLAG);
        keyInfo.getMetadata().remove(OzoneConsts.GDPR_ALGORITHM);
        keyInfo.getMetadata().remove(OzoneConsts.GDPR_SECRET);
        keyInfo.clearFileEncryptionInfo();
    }
    // Set the updateID
    keyInfo.setUpdateID(trxnLogIndex, isRatisEnabled);
    if (repeatedOmKeyInfo == null) {
        // The key doesn't exist in deletedTable, so create a new instance.
        repeatedOmKeyInfo = new RepeatedOmKeyInfo(keyInfo);
    } else {
        // The key exists in deletedTable, so update existing instance.
        repeatedOmKeyInfo.addOmKeyInfo(keyInfo);
    }
    return repeatedOmKeyInfo;
}

Example 2 with RepeatedOmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo in project ozone by apache.

the class TestS3MultipartUploadCompleteRequest method checkDeleteTableCount.

public void checkDeleteTableCount(String volumeName, String bucketName, String keyName, int count) throws Exception {
    String dbOzoneKey = getOzoneDBKey(volumeName, bucketName, keyName);
    RepeatedOmKeyInfo keysToDelete = omMetadataManager.getDeletedTable().get(dbOzoneKey);
    // deleted key entries count is expected to be 0
    if (count == 0) {
        Assert.assertNull(keysToDelete);
        return;
    }
    Assert.assertNotNull(keysToDelete);
    // Count must consider unused parts on commit
    Assert.assertEquals(count, keysToDelete.getOmKeyInfoList().size());
}

Example 3 with RepeatedOmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo in project ozone by apache.

the class TestRepeatedOmKeyInfoCodec method testWithoutPipeline.

public void testWithoutPipeline(int chunkNum) {
    RepeatedOmKeyInfoCodec codec = new RepeatedOmKeyInfoCodec(true);
    OmKeyInfo originKey = getKeyInfo(chunkNum);
    RepeatedOmKeyInfo repeatedOmKeyInfo = new RepeatedOmKeyInfo(originKey);
    try {
        byte[] rawData = codec.toPersistedFormat(repeatedOmKeyInfo);
        RepeatedOmKeyInfo key = codec.fromPersistedFormat(rawData);
        System.out.println("Chunk number = " + chunkNum + ", Serialized key size without pipeline = " + rawData.length);
        assertNull(key.getOmKeyInfoList().get(0).getLatestVersionLocations().getLocationList().get(0).getPipeline());
    } catch (IOException e) {
        fail("Should success");
    }
}

Example 4 with RepeatedOmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo in project ozone by apache.

the class TestRepeatedOmKeyInfoCodec method threadSafety.

public void threadSafety() throws InterruptedException {
    final OmKeyInfo key = getKeyInfo(1);
    final RepeatedOmKeyInfo subject = new RepeatedOmKeyInfo(key);
    final RepeatedOmKeyInfoCodec codec = new RepeatedOmKeyInfoCodec(true);
    final AtomicBoolean failed = new AtomicBoolean();
    ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true).build();
    threadFactory.newThread(() -> {
        for (int i = 0; i < 1000000; i++) {
            try {
                codec.toPersistedFormat(subject.copyObject());
            } catch (Exception e) {
                e.printStackTrace();
                failed.set(true);
            }
        }
    }).start();
    threadFactory.newThread(() -> {
        for (int i = 0; i < 10000; i++) {
            subject.addOmKeyInfo(key);
        }
    }).start();
    final long start = Time.monotonicNow();
    while (!failed.get() && (Time.monotonicNow() - start < 5000)) {
        Thread.sleep(100);
    }
    assertFalse(failed.get());
}

Example 5 with RepeatedOmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo in project ozone by apache.

the class TestOzoneRpcClientAbstract method testDeletedKeyForGDPR.

/**
 * Tests deletedKey for GDPR.
 * 1. Create GDPR Enabled bucket.
 * 2. Create a Key in this bucket so it gets encrypted via GDPRSymmetricKey.
 * 3. Read key and validate the content/metadata is as expected because the
 * readKey will decrypt using the GDPR Symmetric Key with details from KeyInfo
 * Metadata.
 * 4. Delete this key in GDPR enabled bucket
 * 5. Confirm the deleted key metadata in deletedTable does not contain the
 * GDPR encryption details (flag, secret, algorithm).
 * @throws Exception
 */
@Test
public void testDeletedKeyForGDPR() throws Exception {
    // Step 1
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String keyName = UUID.randomUUID().toString();
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    BucketArgs args = BucketArgs.newBuilder().addMetadata(OzoneConsts.GDPR_FLAG, "true").build();
    volume.createBucket(bucketName, args);
    OzoneBucket bucket = volume.getBucket(bucketName);
    Assert.assertEquals(bucketName, bucket.getName());
    Assert.assertNotNull(bucket.getMetadata());
    Assert.assertEquals("true", bucket.getMetadata().get(OzoneConsts.GDPR_FLAG));
    // Step 2
    String text = "hello world";
    Map<String, String> keyMetadata = new HashMap<>();
    keyMetadata.put(OzoneConsts.GDPR_FLAG, "true");
    OzoneOutputStream out = bucket.createKey(keyName, text.getBytes(UTF_8).length, RATIS, ONE, keyMetadata);
    out.write(text.getBytes(UTF_8));
    out.close();
    // Step 3
    OzoneKeyDetails key = bucket.getKey(keyName);
    Assert.assertEquals(keyName, key.getName());
    Assert.assertEquals("true", key.getMetadata().get(OzoneConsts.GDPR_FLAG));
    Assert.assertEquals("AES", key.getMetadata().get(OzoneConsts.GDPR_ALGORITHM));
    Assert.assertTrue(key.getMetadata().get(OzoneConsts.GDPR_SECRET) != null);
    OzoneInputStream is = bucket.readKey(keyName);
    byte[] fileContent = new byte[text.getBytes(UTF_8).length];
    is.read(fileContent);
    Assert.assertTrue(verifyRatisReplication(volumeName, bucketName, keyName, RATIS, ONE));
    Assert.assertEquals(text, new String(fileContent, UTF_8));
    // Step 4
    bucket.deleteKey(keyName);
    // Step 5
    OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
    String objectKey = omMetadataManager.getOzoneKey(volumeName, bucketName, keyName);
    RepeatedOmKeyInfo deletedKeys = omMetadataManager.getDeletedTable().get(objectKey);
    if (deletedKeys != null) {
        Map<String, String> deletedKeyMetadata = deletedKeys.getOmKeyInfoList().get(0).getMetadata();
        Assert.assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_FLAG));
        Assert.assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_SECRET));
        Assert.assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_ALGORITHM));
    }
}