Examples with AllocatedBlock org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock used on opensource projects

Example 1 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class TestOMKeyRequest method setup.

@Before
public void setup() throws Exception {
    ozoneManager = Mockito.mock(OzoneManager.class);
    omMetrics = OMMetrics.create();
    OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    when(ozoneManager.getMetrics()).thenReturn(omMetrics);
    when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
    when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
    OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
    when(lvm.getMetadataLayoutVersion()).thenReturn(0);
    when(ozoneManager.getVersionManager()).thenReturn(lvm);
    when(ozoneManager.isRatisEnabled()).thenReturn(true);
    auditLogger = Mockito.mock(AuditLogger.class);
    when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
    when(ozoneManager.isAdmin(any(String.class))).thenReturn(true);
    when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
    Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
    scmClient = Mockito.mock(ScmClient.class);
    ozoneBlockTokenSecretManager = Mockito.mock(OzoneBlockTokenSecretManager.class);
    scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
    keyManager = new KeyManagerImpl(ozoneManager, scmClient, ozoneConfiguration, "");
    when(ozoneManager.getScmClient()).thenReturn(scmClient);
    when(ozoneManager.getBlockTokenSecretManager()).thenReturn(ozoneBlockTokenSecretManager);
    when(ozoneManager.getScmBlockSize()).thenReturn(scmBlockSize);
    when(ozoneManager.getPreallocateBlocksMax()).thenReturn(2);
    when(ozoneManager.isGrpcBlockTokenEnabled()).thenReturn(false);
    when(ozoneManager.getOMNodeId()).thenReturn(UUID.randomUUID().toString());
    when(scmClient.getBlockClient()).thenReturn(scmBlockLocationProtocol);
    when(ozoneManager.getKeyManager()).thenReturn(keyManager);
    prepareState = new OzoneManagerPrepareState(ozoneConfiguration);
    when(ozoneManager.getPrepareState()).thenReturn(prepareState);
    Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.OPEN).setId(PipelineID.randomId()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE)).setNodes(new ArrayList<>()).build();
    AllocatedBlock allocatedBlock = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID)).setPipeline(pipeline).build();
    List<AllocatedBlock> allocatedBlocks = new ArrayList<>();
    allocatedBlocks.add(allocatedBlock);
    when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), anyString(), any())).thenReturn(allocatedBlocks);
    volumeName = UUID.randomUUID().toString();
    bucketName = UUID.randomUUID().toString();
    keyName = UUID.randomUUID().toString();
    replicationFactor = HddsProtos.ReplicationFactor.ONE;
    replicationType = HddsProtos.ReplicationType.RATIS;
    clientID = Time.now();
    dataSize = 1000L;
    random = new Random();
    version = 0L;
    Pair<String, String> volumeAndBucket = Pair.of(volumeName, bucketName);
    when(ozoneManager.resolveBucketLink(any(KeyArgs.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
    when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
}

Example 2 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class ScmBlockLocationTestingClient method allocateBlock.

/**
 * Returns Fake blocks to the BlockManager so we get blocks in the Database.
 * @param size - size of the block.
 * @param owner - String owner.
 * @param excludeList list of dns/pipelines to exclude
 * @return
 * @throws IOException
 */
@Override
public List<AllocatedBlock> allocateBlock(long size, int num, ReplicationConfig config, String owner, ExcludeList excludeList) throws IOException {
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    Pipeline pipeline = createPipeline(datanodeDetails);
    long containerID = Time.monotonicNow();
    long localID = Time.monotonicNow();
    AllocatedBlock.Builder abb = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(containerID, localID)).setPipeline(pipeline);
    return Collections.singletonList(abb.build());
}

Example 3 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class ScmBlockLocationProtocolServerSideTranslatorPB method allocateScmBlock.

public AllocateScmBlockResponseProto allocateScmBlock(AllocateScmBlockRequestProto request, int clientVersion) throws IOException {
    List<AllocatedBlock> allocatedBlocks = impl.allocateBlock(request.getSize(), request.getNumBlocks(), ReplicationConfig.fromProtoTypeAndFactor(request.getType(), request.getFactor()), request.getOwner(), ExcludeList.getFromProtoBuf(request.getExcludeList()));
    AllocateScmBlockResponseProto.Builder builder = AllocateScmBlockResponseProto.newBuilder();
    if (allocatedBlocks.size() < request.getNumBlocks()) {
        throw new SCMException("Allocated " + allocatedBlocks.size() + " blocks. Requested " + request.getNumBlocks() + " blocks", SCMException.ResultCodes.FAILED_TO_ALLOCATE_ENOUGH_BLOCKS);
    }
    for (AllocatedBlock block : allocatedBlocks) {
        builder.addBlocks(AllocateBlockResponse.newBuilder().setContainerBlockID(block.getBlockID().getProtobuf()).setPipeline(block.getPipeline().getProtobufMessage(clientVersion)));
    }
    return builder.build();
}

Example 4 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class OMKeyRequest method allocateBlock.

/**
 * This methods avoids multiple rpc calls to SCM by allocating multiple blocks
 * in one rpc call.
 * @throws IOException
 */
@SuppressWarnings("parameternumber")
protected List<OmKeyLocationInfo> allocateBlock(ScmClient scmClient, OzoneBlockTokenSecretManager secretManager, HddsProtos.ReplicationType replicationType, HddsProtos.ReplicationFactor replicationFactor, ExcludeList excludeList, long requestedSize, long scmBlockSize, int preallocateBlocksMax, boolean grpcBlockTokenEnabled, String omID) throws IOException {
    int numBlocks = Math.min((int) ((requestedSize - 1) / scmBlockSize + 1), preallocateBlocksMax);
    List<OmKeyLocationInfo> locationInfos = new ArrayList<>(numBlocks);
    String remoteUser = getRemoteUser().getShortUserName();
    List<AllocatedBlock> allocatedBlocks;
    try {
        allocatedBlocks = scmClient.getBlockClient().allocateBlock(scmBlockSize, numBlocks, ReplicationConfig.fromProtoTypeAndFactor(replicationType, replicationFactor), omID, excludeList);
    } catch (SCMException ex) {
        if (ex.getResult().equals(SCMException.ResultCodes.SAFE_MODE_EXCEPTION)) {
            throw new OMException(ex.getMessage(), OMException.ResultCodes.SCM_IN_SAFE_MODE);
        }
        throw ex;
    }
    for (AllocatedBlock allocatedBlock : allocatedBlocks) {
        BlockID blockID = new BlockID(allocatedBlock.getBlockID());
        OmKeyLocationInfo.Builder builder = new OmKeyLocationInfo.Builder().setBlockID(blockID).setLength(scmBlockSize).setOffset(0).setPipeline(allocatedBlock.getPipeline());
        if (grpcBlockTokenEnabled) {
            builder.setToken(secretManager.generateToken(remoteUser, blockID, EnumSet.of(READ, WRITE), scmBlockSize));
        }
        locationInfos.add(builder.build());
    }
    return locationInfos;
}

Example 5 with AllocatedBlock

use of org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock in project ozone by apache.

the class TestBlockManager method testMultipleBlockAllocation.

@Test(timeout = 10000)
public void testMultipleBlockAllocation() throws IOException, TimeoutException, InterruptedException {
    pipelineManager.createPipeline(replicationConfig);
    pipelineManager.createPipeline(replicationConfig);
    HddsTestUtils.openAllRatisPipelines(pipelineManager);
    AllocatedBlock allocatedBlock = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, replicationConfig, OzoneConsts.OZONE, new ExcludeList());
    // block should be allocated in different pipelines
    GenericTestUtils.waitFor(() -> {
        try {
            AllocatedBlock block = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, replicationConfig, OzoneConsts.OZONE, new ExcludeList());
            return !block.getPipeline().getId().equals(allocatedBlock.getPipeline().getId());
        } catch (IOException e) {
        }
        return false;
    }, 100, 1000);
}