Examples with OmKeyArgs org.apache.hadoop.ozone.om.helpers.OmKeyArgs used on opensource projects

Example 21 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestOmBlockVersioning method testAllocateCommit.

@Test
public void testAllocateCommit() throws Exception {
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
    String keyName = "key" + RandomStringUtils.randomNumeric(5);
    OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster, volumeName, bucketName);
    // Versioning isn't supported currently, but just preserving old behaviour
    bucket.setVersioning(true);
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setDataSize(1000).setRefreshPipeline(true).setAcls(new ArrayList<>()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)).build();
    // 1st update, version 0
    OpenKeySession openKey = writeClient.openKey(keyArgs);
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
    OmKeyLocationInfoGroup highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(0, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 2nd update, version 1
    openKey = writeClient.openKey(keyArgs);
    // OmKeyLocationInfo locationInfo =
    // writeClient.allocateBlock(keyArgs, openKey.getId());
    // explicitly set the keyLocation list before committing the key.
    keyArgs.setLocationInfoList(openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly());
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(1, highestVersion.getVersion());
    assertEquals(1, highestVersion.getLocationList().size());
    // 3rd update, version 2
    openKey = writeClient.openKey(keyArgs);
    // this block will be appended to the latest version of version 2.
    OmKeyLocationInfo locationInfo = writeClient.allocateBlock(keyArgs, openKey.getId(), new ExcludeList());
    List<OmKeyLocationInfo> locationInfoList = openKey.getKeyInfo().getLatestVersionLocations().getBlocksLatestVersionOnly();
    Assert.assertTrue(locationInfoList.size() == 1);
    locationInfoList.add(locationInfo);
    keyArgs.setLocationInfoList(locationInfoList);
    writeClient.commitKey(keyArgs, openKey.getId());
    keyInfo = ozoneManager.lookupKey(keyArgs);
    highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
    assertEquals(2, highestVersion.getVersion());
    assertEquals(2, highestVersion.getLocationList().size());
}

Example 22 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestContainerReportWithKeys method testContainerReportKeyWrite.

@Test
public void testContainerReportKeyWrite() throws Exception {
    final String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    final String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
    final String keyName = "key" + RandomStringUtils.randomNumeric(5);
    final int keySize = 100;
    OzoneClient client = OzoneClientFactory.getRpcClient(conf);
    ObjectStore objectStore = client.getObjectStore();
    objectStore.createVolume(volumeName);
    objectStore.getVolume(volumeName).createBucket(bucketName);
    OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName).createKey(keyName, keySize, ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
    String dataString = RandomStringUtils.randomAlphabetic(keySize);
    key.write(dataString.getBytes(UTF_8));
    key.close();
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setDataSize(keySize).setRefreshPipeline(true).build();
    OmKeyLocationInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions().get(0).getBlocksLatestVersionOnly().get(0);
    ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID());
    Set<ContainerReplica> replicas = scm.getContainerManager().getContainerReplicas(ContainerID.valueOf(keyInfo.getContainerID()));
    Assert.assertTrue(replicas.size() == 1);
    replicas.stream().forEach(rp -> Assert.assertTrue(rp.getDatanodeDetails().getParent() != null));
    LOG.info("SCM Container Info keyCount: {} usedBytes: {}", cinfo.getNumberOfKeys(), cinfo.getUsedBytes());
}

Example 23 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestKeyManagerImpl method testLookupFile.

@Test
public void testLookupFile() throws IOException {
    String keyName = RandomStringUtils.randomAlphabetic(5);
    OmKeyArgs keyArgs = createBuilder().setKeyName(keyName).build();
    // lookup for a non-existent file
    try {
        keyManager.lookupFile(keyArgs, null);
        Assert.fail("Lookup file should fail for non existent file");
    } catch (OMException ex) {
        if (ex.getResult() != OMException.ResultCodes.FILE_NOT_FOUND) {
            throw ex;
        }
    }
    // create a file
    OpenKeySession keySession = writeClient.createFile(keyArgs, false, false);
    keyArgs.setLocationInfoList(keySession.getKeyInfo().getLatestVersionLocations().getLocationList());
    writeClient.commitKey(keyArgs, keySession.getId());
    Assert.assertEquals(keyManager.lookupFile(keyArgs, null).getKeyName(), keyName);
    // lookup for created file
    keyArgs = createBuilder().setKeyName("").build();
    try {
        keyManager.lookupFile(keyArgs, null);
        Assert.fail("Lookup file should fail for a directory");
    } catch (OMException ex) {
        if (ex.getResult() != OMException.ResultCodes.NOT_A_FILE) {
            throw ex;
        }
    }
}

Example 24 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestKeyManagerImpl method testLatestLocationVersion.

@Test
public void testLatestLocationVersion() throws IOException {
    String keyName = RandomStringUtils.randomAlphabetic(5);
    OmKeyArgs keyArgs = createBuilder(VERSIONED_BUCKET_NAME).setKeyName(keyName).setLatestVersionLocation(true).build();
    // lookup for a non-existent key
    try {
        keyManager.lookupKey(keyArgs, null);
        Assert.fail("Lookup key should fail for non existent key");
    } catch (OMException ex) {
        if (ex.getResult() != OMException.ResultCodes.KEY_NOT_FOUND) {
            throw ex;
        }
    }
    // create a key
    OpenKeySession keySession = writeClient.createFile(keyArgs, false, false);
    // randomly select 3 datanodes
    List<DatanodeDetails> nodeList = new ArrayList<>();
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(0, null, null, null, null, 0));
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(1, null, null, null, null, 0));
    nodeList.add((DatanodeDetails) scm.getClusterMap().getNode(2, null, null, null, null, 0));
    Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(1)));
    Assume.assumeFalse(nodeList.get(0).equals(nodeList.get(2)));
    // create a pipeline using 3 datanodes
    Pipeline pipeline = scm.getPipelineManager().createPipeline(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), nodeList);
    List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();
    List<OmKeyLocationInfo> locationList = keySession.getKeyInfo().getLatestVersionLocations().getLocationList();
    Assert.assertEquals(1, locationList.size());
    locationInfoList.add(new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(new BlockID(locationList.get(0).getContainerID(), locationList.get(0).getLocalID())).build());
    keyArgs.setLocationInfoList(locationInfoList);
    writeClient.commitKey(keyArgs, keySession.getId());
    // Mock out the pipelines from the SCM
    ContainerInfo containerInfo = new ContainerInfo.Builder().setContainerID(1L).setPipelineID(pipeline.getId()).build();
    List<ContainerWithPipeline> containerWithPipelines = Arrays.asList(new ContainerWithPipeline(containerInfo, pipeline));
    when(mockScmContainerClient.getContainerWithPipelineBatch(Arrays.asList(1L))).thenReturn(containerWithPipelines);
    OmKeyInfo key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    keySession = writeClient.createFile(keyArgs, true, true);
    writeClient.commitKey(keyArgs, keySession.getId());
    // Test lookupKey (latestLocationVersion == true)
    key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    // Test ListStatus (latestLocationVersion == true)
    List<OzoneFileStatus> fileStatuses = keyManager.listStatus(keyArgs, false, "", 1);
    Assert.assertEquals(fileStatuses.size(), 1);
    Assert.assertEquals(fileStatuses.get(0).getKeyInfo().getKeyLocationVersions().size(), 1);
    // Test GetFileStatus (latestLocationVersion == true)
    OzoneFileStatus ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
    Assert.assertEquals(ozoneFileStatus.getKeyInfo().getKeyLocationVersions().size(), 1);
    // Test LookupFile (latestLocationVersion == true)
    key = keyManager.lookupFile(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 1);
    keyArgs = createBuilder(VERSIONED_BUCKET_NAME).setKeyName(keyName).setLatestVersionLocation(false).build();
    // Test lookupKey (latestLocationVersion == false)
    key = keyManager.lookupKey(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
    // Test ListStatus (latestLocationVersion == false)
    fileStatuses = keyManager.listStatus(keyArgs, false, "", 100);
    Assert.assertEquals(fileStatuses.size(), 1);
    Assert.assertEquals(fileStatuses.get(0).getKeyInfo().getKeyLocationVersions().size(), 2);
    // Test GetFileStatus (latestLocationVersion == false)
    ozoneFileStatus = keyManager.getFileStatus(keyArgs, null);
    Assert.assertEquals(ozoneFileStatus.getKeyInfo().getKeyLocationVersions().size(), 2);
    // Test LookupFile (latestLocationVersion == false)
    key = keyManager.lookupFile(keyArgs, null);
    Assert.assertEquals(key.getKeyLocationVersions().size(), 2);
    // Test ListKeys (latestLocationVersion is always true for ListKeys)
    List<OmKeyInfo> keyInfos = keyManager.listKeys(keyArgs.getVolumeName(), keyArgs.getBucketName(), "", keyArgs.getKeyName(), 100);
    Assert.assertEquals(keyInfos.size(), 1);
    Assert.assertEquals(keyInfos.get(0).getKeyLocationVersions().size(), 1);
}

Example 25 with OmKeyArgs

use of org.apache.hadoop.ozone.om.helpers.OmKeyArgs in project ozone by apache.

the class TestDataScrubber method verifyRatisReplication.

private boolean verifyRatisReplication(String volumeName, String bucketName, String keyName, ReplicationType type, ReplicationFactor factor) throws IOException {
    OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true).build();
    HddsProtos.ReplicationType replicationType = HddsProtos.ReplicationType.valueOf(type.toString());
    HddsProtos.ReplicationFactor replicationFactor = HddsProtos.ReplicationFactor.valueOf(factor.getValue());
    OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
    for (OmKeyLocationInfo info : keyInfo.getLatestVersionLocations().getLocationList()) {
        ContainerInfo container = storageContainerLocationClient.getContainer(info.getContainerID());
        if (!ReplicationConfig.getLegacyFactor(container.getReplicationConfig()).equals(replicationFactor) || (container.getReplicationType() != replicationType)) {
            return false;
        }
    }
    return true;
}