Examples with BlockID org.apache.hadoop.hdds.client.BlockID used on opensource projects

Example 1 with BlockID

use of org.apache.hadoop.hdds.client.BlockID in project ozone by apache.

the class ReplicatedFileChecksumHelper method getChunkInfos.

// copied from BlockInputStream
/**
 * Send RPC call to get the block info from the container.
 * @return List of chunks in this block.
 */
protected List<ContainerProtos.ChunkInfo> getChunkInfos(OmKeyLocationInfo keyLocationInfo) throws IOException {
    // irrespective of the container state, we will always read via Standalone
    // protocol.
    Token<OzoneBlockTokenIdentifier> token = keyLocationInfo.getToken();
    Pipeline pipeline = keyLocationInfo.getPipeline();
    BlockID blockID = keyLocationInfo.getBlockID();
    if (pipeline.getType() != HddsProtos.ReplicationType.STAND_ALONE) {
        pipeline = Pipeline.newBuilder(pipeline).setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationConfig.getLegacyFactor(pipeline.getReplicationConfig()))).build();
    }
    boolean success = false;
    List<ContainerProtos.ChunkInfo> chunks;
    XceiverClientSpi xceiverClientSpi = null;
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Initializing BlockInputStream for get key to access {}", blockID.getContainerID());
        }
        xceiverClientSpi = getXceiverClientFactory().acquireClientForReadData(pipeline);
        ContainerProtos.DatanodeBlockID datanodeBlockID = blockID.getDatanodeBlockIDProtobuf();
        ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls.getBlock(xceiverClientSpi, datanodeBlockID, token);
        chunks = response.getBlockData().getChunksList();
        success = true;
    } finally {
        if (!success && xceiverClientSpi != null) {
            getXceiverClientFactory().releaseClientForReadData(xceiverClientSpi, false);
        }
    }
    return chunks;
}

Example 2 with BlockID

use of org.apache.hadoop.hdds.client.BlockID in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}

Example 3 with BlockID

use of org.apache.hadoop.hdds.client.BlockID in project ozone by apache.

the class TestXceiverClientGrpc method invokeXceiverClientReadChunk.

private void invokeXceiverClientReadChunk(XceiverClientSpi client) throws IOException {
    BlockID bid = new BlockID(1, 1);
    bid.setBlockCommitSequenceId(1);
    ContainerProtocolCalls.readChunk(client, ContainerProtos.ChunkInfo.newBuilder().setChunkName("Anything").setChecksumData(ContainerProtos.ChecksumData.newBuilder().setBytesPerChecksum(512).setType(ContainerProtos.ChecksumType.CRC32).build()).setLen(100).setOffset(100).build(), bid, null, null);
}

Example 4 with BlockID

use of org.apache.hadoop.hdds.client.BlockID in project ozone by apache.

the class TestXceiverClientMetrics method testMetrics.

@Test
public void testMetrics() throws Exception {
    OzoneConfiguration conf = new OzoneConfiguration();
    String metaDir = GenericTestUtils.getTempPath(TestXceiverClientManager.class.getName() + UUID.randomUUID());
    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(SCMTestUtils.getReplicationType(conf), SCMTestUtils.getReplicationFactor(conf), OzoneConsts.OZONE);
    XceiverClientSpi client = clientManager.acquireClient(container.getPipeline());
    ContainerCommandRequestProto request = ContainerTestHelper.getCreateContainerRequest(container.getContainerInfo().getContainerID(), container.getPipeline());
    client.sendCommand(request);
    MetricsRecordBuilder containerMetrics = getMetrics(XceiverClientMetrics.SOURCE_NAME);
    // Above request command is in a synchronous way, so there will be no
    // pending requests.
    assertCounter("PendingOps", 0L, containerMetrics);
    assertCounter("numPendingCreateContainer", 0L, containerMetrics);
    // the counter value of average latency metric should be increased
    assertCounter("CreateContainerLatencyNumOps", 1L, containerMetrics);
    breakFlag = false;
    latch = new CountDownLatch(1);
    int numRequest = 10;
    List<CompletableFuture<ContainerCommandResponseProto>> computeResults = new ArrayList<>();
    // start new thread to send async requests
    Thread sendThread = new Thread(() -> {
        while (!breakFlag) {
            try {
                // use async interface for testing pending metrics
                for (int i = 0; i < numRequest; i++) {
                    BlockID blockID = ContainerTestHelper.getTestBlockID(container.getContainerInfo().getContainerID());
                    ContainerProtos.ContainerCommandRequestProto smallFileRequest;
                    smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest(client.getPipeline(), blockID, 1024);
                    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> response = client.sendCommandAsync(smallFileRequest).getResponse();
                    computeResults.add(response);
                }
                Thread.sleep(1000);
            } catch (Exception ignored) {
            }
        }
        latch.countDown();
    });
    sendThread.start();
    GenericTestUtils.waitFor(() -> {
        // check if pending metric count is increased
        MetricsRecordBuilder metric = getMetrics(XceiverClientMetrics.SOURCE_NAME);
        long pendingOps = getLongCounter("PendingOps", metric);
        long pendingPutSmallFileOps = getLongCounter("numPendingPutSmallFile", metric);
        if (pendingOps > 0 && pendingPutSmallFileOps > 0) {
            // reset break flag
            breakFlag = true;
            return true;
        } else {
            return false;
        }
    }, 100, 60000);
    // blocking until we stop sending async requests
    latch.await();
    // Wait for all futures being done.
    GenericTestUtils.waitFor(() -> {
        for (CompletableFuture future : computeResults) {
            if (!future.isDone()) {
                return false;
            }
        }
        return true;
    }, 100, 60000);
    // the counter value of pending metrics should be decreased to 0
    containerMetrics = getMetrics(XceiverClientMetrics.SOURCE_NAME);
    assertCounter("PendingOps", 0L, containerMetrics);
    assertCounter("numPendingPutSmallFile", 0L, containerMetrics);
    clientManager.close();
}

Example 5 with BlockID

use of org.apache.hadoop.hdds.client.BlockID in project ozone by apache.

the class TestReconWithOzoneManager method addKeys.

/**
 * Helper function to add voli/bucketi/keyi to containeri to OM Metadata.
 * For test purpose each container will have only one key.
 */
private void addKeys(int start, int end) throws Exception {
    for (int i = start; i < end; i++) {
        Pipeline pipeline = HddsTestUtils.getRandomPipeline();
        List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
        BlockID blockID = new BlockID(i, 1);
        OmKeyLocationInfo omKeyLocationInfo1 = getOmKeyLocationInfo(blockID, pipeline);
        omKeyLocationInfoList.add(omKeyLocationInfo1);
        OmKeyLocationInfoGroup omKeyLocationInfoGroup = new OmKeyLocationInfoGroup(0, omKeyLocationInfoList);
        writeDataToOm("key" + i, "bucket" + i, "vol" + i, Collections.singletonList(omKeyLocationInfoGroup));
    }
}