Examples with OzoneVolume org.apache.hadoop.ozone.client.OzoneVolume used on opensource projects

Example 1 with OzoneVolume

use of org.apache.hadoop.ozone.client.OzoneVolume in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}

Example 2 with OzoneVolume

use of org.apache.hadoop.ozone.client.OzoneVolume in project ozone by apache.

the class TestStorageContainerManagerHA method testPutKey.

public void testPutKey() throws Exception {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    Instant testStartTime = Instant.now();
    ObjectStore store = OzoneClientFactory.getRpcClient(cluster.getConf()).getObjectStore();
    String value = "sample value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    String keyName = UUID.randomUUID().toString();
    OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, RATIS, ONE, new HashMap<>());
    out.write(value.getBytes(UTF_8));
    out.close();
    OzoneKey key = bucket.getKey(keyName);
    Assert.assertEquals(keyName, key.getName());
    OzoneInputStream is = bucket.readKey(keyName);
    byte[] fileContent = new byte[value.getBytes(UTF_8).length];
    is.read(fileContent);
    Assert.assertEquals(value, new String(fileContent, UTF_8));
    Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
    Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
    is.close();
    final OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName).setBucketName(bucketName).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setKeyName(keyName).setRefreshPipeline(true).build();
    final OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs);
    final List<OmKeyLocationInfo> keyLocationInfos = keyInfo.getKeyLocationVersions().get(0).getBlocksLatestVersionOnly();
    long index = -1;
    for (StorageContainerManager scm : cluster.getStorageContainerManagers()) {
        if (scm.checkLeader()) {
            index = getLastAppliedIndex(scm);
        }
    }
    Assert.assertFalse(index == -1);
    long finalIndex = index;
    // Ensure all follower scms have caught up with the leader
    GenericTestUtils.waitFor(() -> areAllScmInSync(finalIndex), 100, 10000);
    final long containerID = keyLocationInfos.get(0).getContainerID();
    for (int k = 0; k < numOfSCMs; k++) {
        StorageContainerManager scm = cluster.getStorageContainerManagers().get(k);
        // flush to DB on each SCM
        ((SCMRatisServerImpl) scm.getScmHAManager().getRatisServer()).getStateMachine().takeSnapshot();
        Assert.assertTrue(scm.getContainerManager().containerExist(ContainerID.valueOf(containerID)));
        Assert.assertNotNull(scm.getScmMetadataStore().getContainerTable().get(ContainerID.valueOf(containerID)));
    }
}

Example 3 with OzoneVolume

use of org.apache.hadoop.ozone.client.OzoneVolume in project ozone by apache.

the class TestOzoneManagerHA method createKeyTest.

protected void createKeyTest(boolean checkSuccess) throws Exception {
    String userName = "user" + RandomStringUtils.randomNumeric(5);
    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    VolumeArgs createVolumeArgs = VolumeArgs.newBuilder().setOwner(userName).setAdmin(adminName).build();
    try {
        getObjectStore().createVolume(volumeName, createVolumeArgs);
        OzoneVolume retVolumeinfo = getObjectStore().getVolume(volumeName);
        Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
        Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
        Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
        String bucketName = UUID.randomUUID().toString();
        String keyName = UUID.randomUUID().toString();
        retVolumeinfo.createBucket(bucketName);
        OzoneBucket ozoneBucket = retVolumeinfo.getBucket(bucketName);
        Assert.assertTrue(ozoneBucket.getName().equals(bucketName));
        Assert.assertTrue(ozoneBucket.getVolumeName().equals(volumeName));
        String value = "random data";
        OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(keyName, value.length(), ReplicationType.RATIS, ReplicationFactor.ONE, new HashMap<>());
        ozoneOutputStream.write(value.getBytes(UTF_8), 0, value.length());
        ozoneOutputStream.close();
        OzoneInputStream ozoneInputStream = ozoneBucket.readKey(keyName);
        byte[] fileContent = new byte[value.getBytes(UTF_8).length];
        ozoneInputStream.read(fileContent);
        Assert.assertEquals(value, new String(fileContent, UTF_8));
    } catch (ConnectException | RemoteException e) {
        if (!checkSuccess) {
            // last running OM as it would fail to get a quorum.
            if (e instanceof RemoteException) {
                GenericTestUtils.assertExceptionContains("OMNotLeaderException", e);
            }
        } else {
            throw e;
        }
    }
}

Example 4 with OzoneVolume

use of org.apache.hadoop.ozone.client.OzoneVolume in project ozone by apache.

the class TestOzoneManagerHAMetadataOnly method testAllBucketOperations.

@Test
public void testAllBucketOperations() throws Exception {
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    String bucketName = "volume" + RandomStringUtils.randomNumeric(5);
    OzoneVolume retVolume = createAndCheckVolume(volumeName);
    BucketArgs bucketArgs = BucketArgs.newBuilder().setStorageType(StorageType.DISK).setVersioning(true).build();
    retVolume.createBucket(bucketName, bucketArgs);
    OzoneBucket ozoneBucket = retVolume.getBucket(bucketName);
    Assert.assertEquals(volumeName, ozoneBucket.getVolumeName());
    Assert.assertEquals(bucketName, ozoneBucket.getName());
    Assert.assertTrue(ozoneBucket.getVersioning());
    Assert.assertEquals(StorageType.DISK, ozoneBucket.getStorageType());
    Assert.assertFalse(ozoneBucket.getCreationTime().isAfter(Instant.now()));
    // Change versioning to false
    ozoneBucket.setVersioning(false);
    ozoneBucket = retVolume.getBucket(bucketName);
    Assert.assertFalse(ozoneBucket.getVersioning());
    retVolume.deleteBucket(bucketName);
    OzoneTestUtils.expectOmException(OMException.ResultCodes.BUCKET_NOT_FOUND, () -> retVolume.deleteBucket(bucketName));
}

Example 5 with OzoneVolume

use of org.apache.hadoop.ozone.client.OzoneVolume in project ozone by apache.

the class TestScmSafeMode method testSafeModeOperations.

@Test(timeout = 300_000)
public void testSafeModeOperations() throws Exception {
    // Create {numKeys} random names keys.
    TestStorageContainerManagerHelper helper = new TestStorageContainerManagerHelper(cluster, conf);
    Map<String, OmKeyInfo> keyLocations = helper.createKeys(100, 4096);
    final List<ContainerInfo> containers = cluster.getStorageContainerManager().getContainerManager().getContainers();
    GenericTestUtils.waitFor(() -> containers.size() >= 3, 100, 1000);
    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
    String keyName = "key" + RandomStringUtils.randomNumeric(5);
    ObjectStore store = cluster.getRpcClient().getObjectStore();
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    bucket.createKey(keyName, 1000, RATIS, ONE, new HashMap<>());
    cluster.stop();
    try {
        cluster = builder.build();
    } catch (IOException e) {
        fail("failed");
    }
    StorageContainerManager scm;
    scm = cluster.getStorageContainerManager();
    Assert.assertTrue(scm.isInSafeMode());
    om = cluster.getOzoneManager();
    final OzoneBucket bucket1 = cluster.getRpcClient().getObjectStore().getVolume(volumeName).getBucket(bucketName);
    // As cluster is restarted with out datanodes restart
    LambdaTestUtils.intercept(IOException.class, "SafeModePrecheck failed for allocateBlock", () -> bucket1.createKey(keyName, 1000, RATIS, ONE, new HashMap<>()));
}