Examples with AllocatedBlock org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock used on opensource projects

Example 11 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class BlockManagerImpl method newBlock.

/**
 * newBlock - returns a new block assigned to a container.
 *
 * @param containerInfo - Container Info.
 * @return AllocatedBlock
 */
private AllocatedBlock newBlock(ContainerInfo containerInfo) throws NotLeaderException {
    try {
        final Pipeline pipeline = pipelineManager.getPipeline(containerInfo.getPipelineID());
        long localID = sequenceIdGen.getNextId(LOCAL_ID);
        long containerID = containerInfo.getContainerID();
        AllocatedBlock.Builder abb = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(containerID, localID)).setPipeline(pipeline);
        if (LOG.isTraceEnabled()) {
            LOG.trace("New block allocated : {} Container ID: {}", localID, containerID);
        }
        pipelineManager.incNumBlocksAllocatedMetric(pipeline.getId());
        return abb.build();
    } catch (PipelineNotFoundException ex) {
        LOG.error("Pipeline Machine count is zero.", ex);
        return null;
    }
}

Example 12 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class TestBlockManager method testBlockDistributionWithMultipleDisks.

@Test
public void testBlockDistributionWithMultipleDisks() throws Exception {
    int threadCount = numContainerPerOwnerInPipeline * numContainerPerOwnerInPipeline;
    nodeManager.setNumHealthyVolumes(numContainerPerOwnerInPipeline);
    nodeManager.setNumPipelinePerDatanode(1);
    List<ExecutorService> executors = new ArrayList<>(threadCount);
    for (int i = 0; i < threadCount; i++) {
        executors.add(Executors.newSingleThreadExecutor());
    }
    pipelineManager.createPipeline(replicationConfig);
    HddsTestUtils.openAllRatisPipelines(pipelineManager);
    Map<Long, List<AllocatedBlock>> allocatedBlockMap = new ConcurrentHashMap<>();
    List<CompletableFuture<AllocatedBlock>> futureList = new ArrayList<>(threadCount);
    for (int i = 0; i < threadCount; i++) {
        final CompletableFuture<AllocatedBlock> future = new CompletableFuture<>();
        CompletableFuture.supplyAsync(() -> {
            try {
                List<AllocatedBlock> blockList;
                AllocatedBlock block = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, replicationConfig, OzoneConsts.OZONE, new ExcludeList());
                long containerId = block.getBlockID().getContainerID();
                if (!allocatedBlockMap.containsKey(containerId)) {
                    blockList = new ArrayList<>();
                } else {
                    blockList = allocatedBlockMap.get(containerId);
                }
                blockList.add(block);
                allocatedBlockMap.put(containerId, blockList);
                future.complete(block);
            } catch (IOException e) {
                future.completeExceptionally(e);
            }
            return future;
        }, executors.get(i));
        futureList.add(future);
    }
    try {
        CompletableFuture.allOf(futureList.toArray(new CompletableFuture[futureList.size()])).get();
        Assert.assertEquals(1, pipelineManager.getPipelines(replicationConfig).size());
        Pipeline pipeline = pipelineManager.getPipelines(replicationConfig).get(0);
        // total no of containers to be created will be number of healthy
        // volumes * number of numContainerPerOwnerInPipeline which is equal to
        // the thread count
        Assert.assertEquals(threadCount, pipelineManager.getNumberOfContainers(pipeline.getId()));
        Assert.assertEquals(threadCount, allocatedBlockMap.size());
        Assert.assertEquals(threadCount, allocatedBlockMap.values().size());
        allocatedBlockMap.values().stream().forEach(v -> {
            Assert.assertEquals(1, v.size());
        });
    } catch (Exception e) {
        Assert.fail("testAllocateBlockInParallel failed");
    }
}

Example 13 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class ScmBlockLocationProtocolClientSideTranslatorPB method allocateBlock.

/**
 * Asks SCM where a block should be allocated. SCM responds with the
 * set of datanodes that should be used creating this block.
 *
 * @param size              - size of the block.
 * @param num               - number of blocks.
 * @param replicationConfig - replication configuration of the blocks.
 * @param excludeList       - exclude list while allocating blocks.
 * @return allocated block accessing info (key, pipeline).
 * @throws IOException
 */
@Override
public List<AllocatedBlock> allocateBlock(long size, int num, ReplicationConfig replicationConfig, String owner, ExcludeList excludeList) throws IOException {
    Preconditions.checkArgument(size > 0, "block size must be greater than 0");
    final AllocateScmBlockRequestProto.Builder requestBuilder = AllocateScmBlockRequestProto.newBuilder().setSize(size).setNumBlocks(num).setType(replicationConfig.getReplicationType()).setOwner(owner).setExcludeList(excludeList.getProtoBuf());
    switch(replicationConfig.getReplicationType()) {
        case STAND_ALONE:
            requestBuilder.setFactor(((StandaloneReplicationConfig) replicationConfig).getReplicationFactor());
            break;
        case RATIS:
            requestBuilder.setFactor(((RatisReplicationConfig) replicationConfig).getReplicationFactor());
            break;
        default:
            throw new IllegalArgumentException("Unsupported replication type " + replicationConfig.getReplicationType());
    }
    AllocateScmBlockRequestProto request = requestBuilder.build();
    SCMBlockLocationRequest wrapper = createSCMBlockRequest(Type.AllocateScmBlock).setAllocateScmBlockRequest(request).build();
    final SCMBlockLocationResponse wrappedResponse = handleError(submitRequest(wrapper));
    final AllocateScmBlockResponseProto response = wrappedResponse.getAllocateScmBlockResponse();
    List<AllocatedBlock> blocks = new ArrayList<>(response.getBlocksCount());
    for (AllocateBlockResponse resp : response.getBlocksList()) {
        AllocatedBlock.Builder builder = new AllocatedBlock.Builder().setContainerBlockID(ContainerBlockID.getFromProtobuf(resp.getContainerBlockID())).setPipeline(Pipeline.getFromProtobuf(resp.getPipeline()));
        blocks.add(builder.build());
    }
    return blocks;
}

Example 14 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class TestSCMPipelineMetrics method testNumBlocksAllocated.

@Test
public void testNumBlocksAllocated() throws IOException {
    AllocatedBlock block = cluster.getStorageContainerManager().getScmBlockManager().allocateBlock(5, RatisReplicationConfig.getInstance(ReplicationFactor.ONE), "Test", new ExcludeList());
    MetricsRecordBuilder metrics = getMetrics(SCMPipelineMetrics.class.getSimpleName());
    Pipeline pipeline = block.getPipeline();
    long numBlocksAllocated = getLongCounter(SCMPipelineMetrics.getBlockAllocationMetricName(pipeline), metrics);
    Assert.assertEquals(numBlocksAllocated, 1);
    // destroy the pipeline
    try {
        cluster.getStorageContainerManager().getClientProtocolServer().closePipeline(pipeline.getId().getProtobuf());
    } catch (IOException e) {
        e.printStackTrace();
        Assert.fail();
    }
    metrics = getMetrics(SCMPipelineMetrics.class.getSimpleName());
    try {
        getLongCounter(SCMPipelineMetrics.getBlockAllocationMetricName(pipeline), metrics);
        Assert.fail("Metric should not be present for closed pipeline.");
    } catch (AssertionError e) {
        Assert.assertTrue(e.getMessage().contains("Expected exactly one metric for name " + SCMPipelineMetrics.getBlockAllocationMetricName(block.getPipeline())));
    }
}