Search in sources :

Example 1 with ContainerBlockID

use of org.apache.hadoop.hdds.client.ContainerBlockID in project ozone by apache.

the class TestOMKeyRequest method setup.

@Before
public void setup() throws Exception {
    ozoneManager = Mockito.mock(OzoneManager.class);
    omMetrics = OMMetrics.create();
    OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
    ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath());
    ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, folder.newFolder().getAbsolutePath());
    omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration);
    when(ozoneManager.getMetrics()).thenReturn(omMetrics);
    when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
    when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
    OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
    when(lvm.getMetadataLayoutVersion()).thenReturn(0);
    when(ozoneManager.getVersionManager()).thenReturn(lvm);
    when(ozoneManager.isRatisEnabled()).thenReturn(true);
    auditLogger = Mockito.mock(AuditLogger.class);
    when(ozoneManager.getAuditLogger()).thenReturn(auditLogger);
    when(ozoneManager.isAdmin(any(String.class))).thenReturn(true);
    when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
    Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class));
    scmClient = Mockito.mock(ScmClient.class);
    ozoneBlockTokenSecretManager = Mockito.mock(OzoneBlockTokenSecretManager.class);
    scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
    keyManager = new KeyManagerImpl(ozoneManager, scmClient, ozoneConfiguration, "");
    when(ozoneManager.getScmClient()).thenReturn(scmClient);
    when(ozoneManager.getBlockTokenSecretManager()).thenReturn(ozoneBlockTokenSecretManager);
    when(ozoneManager.getScmBlockSize()).thenReturn(scmBlockSize);
    when(ozoneManager.getPreallocateBlocksMax()).thenReturn(2);
    when(ozoneManager.isGrpcBlockTokenEnabled()).thenReturn(false);
    when(ozoneManager.getOMNodeId()).thenReturn(UUID.randomUUID().toString());
    when(scmClient.getBlockClient()).thenReturn(scmBlockLocationProtocol);
    when(ozoneManager.getKeyManager()).thenReturn(keyManager);
    prepareState = new OzoneManagerPrepareState(ozoneConfiguration);
    when(ozoneManager.getPrepareState()).thenReturn(prepareState);
    Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.OPEN).setId(PipelineID.randomId()).setReplicationConfig(StandaloneReplicationConfig.getInstance(ReplicationFactor.ONE)).setNodes(new ArrayList<>()).build();
    AllocatedBlock allocatedBlock = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(CONTAINER_ID, LOCAL_ID)).setPipeline(pipeline).build();
    List<AllocatedBlock> allocatedBlocks = new ArrayList<>();
    allocatedBlocks.add(allocatedBlock);
    when(scmBlockLocationProtocol.allocateBlock(anyLong(), anyInt(), any(), anyString(), any())).thenReturn(allocatedBlocks);
    volumeName = UUID.randomUUID().toString();
    bucketName = UUID.randomUUID().toString();
    keyName = UUID.randomUUID().toString();
    replicationFactor = HddsProtos.ReplicationFactor.ONE;
    replicationType = HddsProtos.ReplicationType.RATIS;
    clientID = Time.now();
    dataSize = 1000L;
    random = new Random();
    version = 0L;
    Pair<String, String> volumeAndBucket = Pair.of(volumeName, bucketName);
    when(ozoneManager.resolveBucketLink(any(KeyArgs.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
    when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class))).thenReturn(new ResolvedBucket(volumeAndBucket, volumeAndBucket));
}
Also used : AuditMessage(org.apache.hadoop.ozone.audit.AuditMessage) ArrayList(java.util.ArrayList) OzoneManagerPrepareState(org.apache.hadoop.ozone.om.OzoneManagerPrepareState) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) OMLayoutVersionManager(org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager) Random(java.util.Random) OzoneManager(org.apache.hadoop.ozone.om.OzoneManager) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) ResolvedBucket(org.apache.hadoop.ozone.om.ResolvedBucket) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Pair(org.apache.commons.lang3.tuple.Pair) OMClientRequest(org.apache.hadoop.ozone.om.request.OMClientRequest) AuditLogger(org.apache.hadoop.ozone.audit.AuditLogger) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) KeyArgs(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs) ScmClient(org.apache.hadoop.ozone.om.ScmClient) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OmMetadataManagerImpl(org.apache.hadoop.ozone.om.OmMetadataManagerImpl) KeyManagerImpl(org.apache.hadoop.ozone.om.KeyManagerImpl) ScmBlockLocationProtocol(org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol) AllocatedBlock(org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock) OzoneBlockTokenSecretManager(org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager) Before(org.junit.Before)

Example 2 with ContainerBlockID

use of org.apache.hadoop.hdds.client.ContainerBlockID in project ozone by apache.

the class ScmBlockLocationTestingClient method allocateBlock.

/**
 * Returns Fake blocks to the BlockManager so we get blocks in the Database.
 * @param size - size of the block.
 * @param owner - String owner.
 * @param excludeList list of dns/pipelines to exclude
 * @return
 * @throws IOException
 */
@Override
public List<AllocatedBlock> allocateBlock(long size, int num, ReplicationConfig config, String owner, ExcludeList excludeList) throws IOException {
    DatanodeDetails datanodeDetails = randomDatanodeDetails();
    Pipeline pipeline = createPipeline(datanodeDetails);
    long containerID = Time.monotonicNow();
    long localID = Time.monotonicNow();
    AllocatedBlock.Builder abb = new AllocatedBlock.Builder().setContainerBlockID(new ContainerBlockID(containerID, localID)).setPipeline(pipeline);
    return Collections.singletonList(abb.build());
}
Also used : MockDatanodeDetails.randomDatanodeDetails(org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) AllocatedBlock(org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline)

Example 3 with ContainerBlockID

use of org.apache.hadoop.hdds.client.ContainerBlockID in project ozone by apache.

the class OmKeyInfo method verifyAndGetKeyLocations.

private List<OmKeyLocationInfo> verifyAndGetKeyLocations(List<OmKeyLocationInfo> locationInfoList, OmKeyLocationInfoGroup keyLocationInfoGroup) {
    List<OmKeyLocationInfo> allocatedBlockLocations = keyLocationInfoGroup.getBlocksLatestVersionOnly();
    List<OmKeyLocationInfo> updatedBlockLocations = new ArrayList<>();
    List<ContainerBlockID> existingBlockIDs = new ArrayList<>();
    for (OmKeyLocationInfo existingLocationInfo : allocatedBlockLocations) {
        BlockID existingBlockID = existingLocationInfo.getBlockID();
        existingBlockIDs.add(existingBlockID.getContainerBlockID());
    }
    for (OmKeyLocationInfo modifiedLocationInfo : locationInfoList) {
        BlockID modifiedBlockID = modifiedLocationInfo.getBlockID();
        if (existingBlockIDs.contains(modifiedBlockID.getContainerBlockID())) {
            updatedBlockLocations.add(modifiedLocationInfo);
        } else {
            LOG.warn("Unknown BlockLocation:{}, where the blockID of given " + "location doesn't match with the stored/allocated block of" + " keyName:{}", modifiedLocationInfo, keyName);
        }
    }
    return updatedBlockLocations;
}
Also used : ArrayList(java.util.ArrayList) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID)

Example 4 with ContainerBlockID

use of org.apache.hadoop.hdds.client.ContainerBlockID in project ozone by apache.

the class TestBlockInputStream method setup.

@Before
public void setup() throws Exception {
    BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
    checksum = new Checksum(ChecksumType.NONE, CHUNK_SIZE);
    createChunkList(5);
    blockStream = new DummyBlockInputStream(blockID, blockSize, null, null, false, null, refreshPipeline, chunks, chunkDataMap);
}
Also used : Checksum(org.apache.hadoop.ozone.common.Checksum) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) Before(org.junit.Before)

Example 5 with ContainerBlockID

use of org.apache.hadoop.hdds.client.ContainerBlockID in project ozone by apache.

the class TestBlockInputStream method testGetBlockInfoFailWithIOException.

@Test
public void testGetBlockInfoFailWithIOException() throws Exception {
    GenericTestUtils.setLogLevel(BlockInputStream.getLog(), Level.DEBUG);
    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(LoggerFactory.getLogger(BlockInputStream.class));
    BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
    AtomicBoolean isRefreshed = new AtomicBoolean();
    createChunkList(5);
    BlockInputStream blockInputStreamWithRetry = new DummyBlockInputStreamWithRetry(blockID, blockSize, MockPipeline.createSingleNodePipeline(), null, false, null, chunks, chunkDataMap, isRefreshed, new IOException("unavailable"));
    try {
        Assert.assertFalse(isRefreshed.get());
        byte[] b = new byte[200];
        blockInputStreamWithRetry.read(b, 0, 200);
        // As in case of IOException we do not do do refresh.
        Assert.assertFalse(isRefreshed.get());
        Assert.assertTrue(logCapturer.getOutput().contains("Retry to get chunk info fail"));
    } finally {
        blockInputStreamWithRetry.close();
    }
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) GenericTestUtils(org.apache.ozone.test.GenericTestUtils) BlockID(org.apache.hadoop.hdds.client.BlockID) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) ContainerBlockID(org.apache.hadoop.hdds.client.ContainerBlockID) IOException(java.io.IOException) Test(org.junit.Test)

Aggregations

ContainerBlockID (org.apache.hadoop.hdds.client.ContainerBlockID)12 BlockID (org.apache.hadoop.hdds.client.BlockID)9 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)7 Test (org.junit.Test)6 ChunkInfo (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo)4 MockPipeline (org.apache.hadoop.hdds.scm.pipeline.MockPipeline)4 AllocatedBlock (org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock)3 StorageContainerException (org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException)3 ArrayList (java.util.ArrayList)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Before (org.junit.Before)2 IOException (java.io.IOException)1 Random (java.util.Random)1 Pair (org.apache.commons.lang3.tuple.Pair)1 OzoneConfiguration (org.apache.hadoop.hdds.conf.OzoneConfiguration)1 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)1 MockDatanodeDetails.randomDatanodeDetails (org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails)1 XceiverClientFactory (org.apache.hadoop.hdds.scm.XceiverClientFactory)1 XceiverClientSpi (org.apache.hadoop.hdds.scm.XceiverClientSpi)1 PipelineNotFoundException (org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException)1