Examples with OmKeyInfo org.apache.hadoop.ozone.om.helpers.OmKeyInfo used on opensource projects

Example 16 with OmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.

the class TestOmBlockVersioning method testAllocateCommit.

@Test
public void testAllocateCommit() throws Exception {
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
    String keyName = "key" + RandomStringUtils.randomNumeric(5);
    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
    // Versioning isn't supported currently, but just preserving old behaviour
    bucket.setVersioning(true);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setDataSize(1000).setRefreshPipeline(true).setAcls(new ArrayList<>()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)).build();
    // 1st update, version 0
    OpenKeySession openKey = writeClient.openKey(keyArgs);
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
    OmKeyLocationInfoGroup highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(0, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 2nd update, version 1
    openKey = writeClient.openKey(keyArgs);
    // OmKeyLocationInfo locationInfo =
    // writeClient.allocateBlock(keyArgs, openKey.getId());
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(1, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 3rd update, version 2
    openKey = writeClient.openKey(keyArgs);
    // this block will be appended to the latest version of version 2.
    OmKeyLocationInfo locationInfo = writeClient.allocateBlock(keyArgs, openKey.getId(), new ExcludeList());
    List<OmKeyLocationInfo> locationInfoList = openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly();
    Assert.assertTrue(locationInfoList.size() == 1);
    locationInfoList.add(locationInfo);
    keyArgs.setLocationInfoList(locationInfoList);
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(2, highestVersion.getVersion());
    assertEquals(2, highestVersion.getLocationList().size());
}

Example 17 with OmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.

the class TestKeyManagerImpl method testLatestLocationVersion.

@Test
public void testLatestLocationVersion() throws IOException {
    String keyName = RandomStringUtils.randomAlphabetic(5);
    OmKeyArgs keyArgs = createBuilder(VERSIONED_BUCKET_NAME).setKeyName(keyName).setLatestVersionLocation(true).build();
    // lookup for a non-existent key
    try {
        keyManager.lookupKey(keyArgs, null);
        Assert.fail("Lookup key should fail for non existent key");
    } catch (OMException ex) {
        if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
            throw ex;
        }
    }
    // create a key
    OpenKeySession keySession = writeClient.createFile(keyArgs, false, false);
    // randomly select 3 datanodes
    List<DatanodeDetails> nodeList = new ArrayList<>();
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(0, null, null, null, null, 0));
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(1, null, null, null, null, 0));
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(2, null, null, null, null, 0));
    Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(1)));
    Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(2)));
    // create a pipeline using 3 datanodes
    Pipeline pipeline = scm.getPipelineManager().createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), nodeList);
    List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
    List<OmKeyLocationInfo> locationList = keySession.getKeyInfo().getLatestVersionLocations().getLocationList();
    Assert.assertEquals(1, locationList.size());
    locationInfoList.add(new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(new BlockID(locationList.get(0).getContainerID(), locationList.get(0).getLocalID())).build());
    keyArgs.setLocationInfoList(locationInfoList);
    writeClient.commitKey(keyArgs, keySession.getId());
    // Mock out the pipelines from the SCM
    ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(1L).setPipelineID(pipeline.getId()).build();
    List<ContainerWithPipeline> containerWithPipelines = Arrays.asList(new ContainerWithPipeline(containerInfo, pipeline));
    when(mockScmContainerClient.getContainerWithPipelineBatch(Arrays.asList(1L))).thenReturn(containerWithPipelines);
    OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    keySession = writeClient.createFile(keyArgs, true, true);
    writeClient.commitKey(keyArgs, keySession.getId());
    // Test lookupKey (latestLocationVersion == true)
    key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    // Test ListStatus (latestLocationVersion == true)
    List<OzoneFileStatus> fileStatuses = keyManager.listStatus(keyArgs, false, "", 1);
    Assert.assertEquals(fileStatuses.size(), 1);
    Assert.assertEquals(fileStatuses.get(0).getKeyInfo().getKeyLocationVersions().size(), 1);
    // Test GetFileStatus (latestLocationVersion == true)
    OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
    Assert.assertEquals(ozoneFileStatus.getKeyInfo().getKeyLocationVersions().size(), 1);
    // Test LookupFile (latestLocationVersion == true)
    key = keyManager.lookupFile(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    keyArgs = createBuilder(VERSIONED_BUCKET_NAME).setKeyName(keyName).setLatestVersionLocation(false).build();
    // Test lookupKey (latestLocationVersion == false)
    key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
    // Test ListStatus (latestLocationVersion == false)
    fileStatuses = keyManager.listStatus(keyArgs, false, "", 100);
    Assert.assertEquals(fileStatuses.size(), 1);
    Assert.assertEquals(fileStatuses.get(0).getKeyInfo().getKeyLocationVersions().size(), 2);
    // Test GetFileStatus (latestLocationVersion == false)
    ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
    Assert.assertEquals(ozoneFileStatus.getKeyInfo().getKeyLocationVersions().size(), 2);
    // Test LookupFile (latestLocationVersion == false)
    key = keyManager.lookupFile(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
    // Test ListKeys (latestLocationVersion is always true for ListKeys)
    List<OmKeyInfo> keyInfos = keyManager.listKeys(keyArgs.getVolumeName(), keyArgs.getBucketName(), "", keyArgs.getKeyName(), 100);
    Assert.assertEquals(keyInfos.size(), 1);
    Assert.assertEquals(keyInfos.get(0).getKeyLocationVersions().size(), 1);
}

Example 18 with OmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.

the class TestDataScrubber method verifyRatisReplication.

private boolean verifyRatisReplication(String volumeName, String bucketName, String keyName, ReplicationType type, ReplicationFactor factor) throws IOException {
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).build();
    HddsProtos.ReplicationType replicationType = HddsProtos.ReplicationType.valueOf(type.toString());
    HddsProtos.ReplicationFactor replicationFactor = HddsProtos.ReplicationFactor.valueOf(factor.getValue());
    OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
    for (OmKeyLocationInfo info : keyInfo.getLatestVersionLocations().getLocationList()) {
        ContainerInfo container = storageContainerLocationClient.getContainer(info.getContainerID());
        if (!ReplicationConfig.getLegacyFactor(container.getReplicationConfig()).equals(replicationFactor) || (container.getReplicationType() != replicationType)) {
            return false;
        }
    }
    return true;
}

Example 19 with OmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.

the class TestObjectStoreWithFSO method testLookupKey.

@Test
public void testLookupKey() throws Exception {
    String parent = "a/b/c/";
    String fileName = "key" + RandomStringUtils.randomNumeric(5);
    String key = parent + fileName;
    OzoneClient client = cluster.getClient();
    ObjectStore objectStore = client.getObjectStore();
    OzoneVolume ozoneVolume = objectStore.getVolume(volumeName);
    Assert.assertTrue(ozoneVolume.getName().equals(volumeName));
    OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
    Assert.assertTrue(ozoneBucket.getName().equals(bucketName));
    Table<String, OmKeyInfo> openFileTable = cluster.getOzoneManager().getMetadataManager().getOpenKeyTable(getBucketLayout());
    String data = "random data";
    OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, data.length(), ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    KeyOutputStream keyOutputStream = (KeyOutputStream) ozoneOutputStream.getOutputStream();
    long clientID = keyOutputStream.getClientID();
    OmDirectoryInfo dirPathC = getDirInfo(parent);
    Assert.assertNotNull("Failed to find dir path: a/b/c", dirPathC);
    // after file creation
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), false);
    ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8), 0, data.length());
    // open key
    try {
        ozoneBucket.getKey(key);
        fail("Should throw exception as fileName is not visible and its still " + "open for writing!");
    } catch (OMException ome) {
        // expected
        assertEquals(ome.getResult(), OMException.ResultCodes.KEY_NOT_FOUND);
    }
    ozoneOutputStream.close();
    OzoneKeyDetails keyDetails = ozoneBucket.getKey(key);
    Assert.assertEquals(key, keyDetails.getName());
    Table<String, OmKeyInfo> fileTable = cluster.getOzoneManager().getMetadataManager().getKeyTable(getBucketLayout());
    // When closing the key, entry should be removed from openFileTable
    // and it should be added to fileTable.
    verifyKeyInFileTable(fileTable, fileName, dirPathC.getObjectID(), false);
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), true);
    ozoneBucket.deleteKey(key);
    // get deleted key
    try {
        ozoneBucket.getKey(key);
        fail("Should throw exception as fileName not exists!");
    } catch (OMException ome) {
        // expected
        assertEquals(ome.getResult(), OMException.ResultCodes.KEY_NOT_FOUND);
    }
    // after key delete
    verifyKeyInFileTable(fileTable, fileName, dirPathC.getObjectID(), true);
    verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), true);
}

Example 20 with OmKeyInfo

use of org.apache.hadoop.ozone.om.helpers.OmKeyInfo in project ozone by apache.

the class TestOMKeyRequest method verifyPathInOpenKeyTable.

/**
 * Verify path in open key table. Also, it returns OMKeyInfo for the given
 * key path.
 *
 * @param key      key name
 * @param id       client id
 * @param doAssert if true then do assertion, otherwise it just skip.
 * @return om key info for the given key path.
 * @throws Exception DB failure
 */
protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, boolean doAssert) throws Exception {
    String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, key, id);
    OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout()).get(openKey);
    if (doAssert) {
        Assert.assertNotNull("Failed to find key in OpenKeyTable", omKeyInfo);
    }
    return omKeyInfo;
}