Search in sources :

Example 1 with XceiverClientFactory

use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.

the class TestReplicatedFileChecksumHelper method init.

@Before
public void init() throws IOException {
    ConfigurationSource config = new InMemoryConfiguration();
    rpcClient = new RpcClient(config, null) {

        @Override
        protected OmTransport createOmTransport(String omServiceId) throws IOException {
            return new MockOmTransport();
        }

        @NotNull
        @Override
        protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
            return new MockXceiverClientFactory();
        }
    };
    client = new OzoneClient(config, rpcClient);
    store = client.getObjectStore();
}
Also used : ConfigurationSource(org.apache.hadoop.hdds.conf.ConfigurationSource) MockOmTransport(org.apache.hadoop.ozone.client.MockOmTransport) MockOmTransport(org.apache.hadoop.ozone.client.MockOmTransport) OmTransport(org.apache.hadoop.ozone.om.protocolPB.OmTransport) ByteString(org.apache.ratis.thirdparty.com.google.protobuf.ByteString) IOException(java.io.IOException) OzoneClient(org.apache.hadoop.ozone.client.OzoneClient) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) NotNull(org.jetbrains.annotations.NotNull) X509Certificate(java.security.cert.X509Certificate) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) InMemoryConfiguration(org.apache.hadoop.hdds.conf.InMemoryConfiguration) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Before(org.junit.Before)

Example 2 with XceiverClientFactory

use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.

the class TestReplicatedFileChecksumHelper method testOneBlock.

@Test
public void testOneBlock() throws IOException {
    // test the file checksum of a file with one block.
    OzoneConfiguration conf = new OzoneConfiguration();
    RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
    List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
    Pipeline pipeline;
    pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
    XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {

        @Override
        public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
            return buildValidResponse();
        }
    };
    XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
    when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
    when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
    OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
    when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
    BlockID blockID = new BlockID(1, 1);
    OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
    List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
    OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
    when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
    OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
    when(mockVolume.getName()).thenReturn("vol1");
    OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
    when(bucket.getName()).thenReturn("bucket1");
    ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
    helper.compute();
    FileChecksum fileChecksum = helper.getFileChecksum();
    assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
    assertEquals(1, helper.getKeyLocationInfoList().size());
}
Also used : OzoneManagerProtocol(org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol) OzoneConfiguration(org.apache.hadoop.hdds.conf.OzoneConfiguration) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) MockXceiverClientFactory(org.apache.hadoop.ozone.client.MockXceiverClientFactory) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) MD5MD5CRC32GzipFileChecksum(org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum) FileChecksum(org.apache.hadoop.fs.FileChecksum) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OmKeyLocationInfoGroup(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) XceiverClientGrpc(org.apache.hadoop.hdds.scm.XceiverClientGrpc) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) RpcClient(org.apache.hadoop.ozone.client.rpc.RpcClient) Test(org.junit.Test)

Example 3 with XceiverClientFactory

use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.

the class TestChunkInputStream method connectsToNewPipeline.

@Test
public void connectsToNewPipeline() throws Exception {
    // GIVEN
    Pipeline pipeline = MockPipeline.createSingleNodePipeline();
    Pipeline newPipeline = MockPipeline.createSingleNodePipeline();
    XceiverClientFactory clientFactory = mock(XceiverClientFactory.class);
    XceiverClientSpi client = mock(XceiverClientSpi.class);
    when(clientFactory.acquireClientForReadData(pipeline)).thenReturn(client);
    AtomicReference<Pipeline> pipelineRef = new AtomicReference<>(pipeline);
    ChunkInputStream subject = new ChunkInputStream(chunkInfo, null, clientFactory, pipelineRef::get, false, null) {

        @Override
        protected ByteBuffer[] readChunk(ChunkInfo readChunkInfo) {
            return ByteString.copyFrom(chunkData).asReadOnlyByteBufferList().toArray(new ByteBuffer[0]);
        }
    };
    try {
        // WHEN
        subject.unbuffer();
        pipelineRef.set(newPipeline);
        int b = subject.read();
        // THEN
        Assert.assertNotEquals(-1, b);
        verify(clientFactory).acquireClientForReadData(newPipeline);
    } finally {
        subject.close();
    }
}
Also used : ChunkInfo(org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo) AtomicReference(java.util.concurrent.atomic.AtomicReference) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) XceiverClientSpi(org.apache.hadoop.hdds.scm.XceiverClientSpi) ByteBuffer(java.nio.ByteBuffer) MockPipeline(org.apache.hadoop.hdds.scm.pipeline.MockPipeline) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) Test(org.junit.Test)

Example 4 with XceiverClientFactory

use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.

the class TestReadRetries method testPutKeyAndGetKeyThreeNodes.

@Test
public void testPutKeyAndGetKeyThreeNodes() throws Exception {
    String volumeName = UUID.randomUUID().toString();
    String bucketName = UUID.randomUUID().toString();
    String value = "sample value";
    store.createVolume(volumeName);
    OzoneVolume volume = store.getVolume(volumeName);
    volume.createBucket(bucketName);
    OzoneBucket bucket = volume.getBucket(bucketName);
    String keyName = "a/b/c/" + UUID.randomUUID().toString();
    OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS, ReplicationFactor.THREE, new HashMap<>());
    KeyOutputStream groupOutputStream = (KeyOutputStream) out.getOutputStream();
    XceiverClientFactory factory = groupOutputStream.getXceiverClientFactory();
    out.write(value.getBytes(UTF_8));
    out.close();
    // First, confirm the key info from the client matches the info in OM.
    OmKeyArgs.Builder builder = new OmKeyArgs.Builder();
    builder.setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true);
    OmKeyLocationInfo keyInfo = ozoneManager.lookupKey(builder.build()).getKeyLocationVersions().get(0).getBlocksLatestVersionOnly().get(0);
    long containerID = keyInfo.getContainerID();
    long localID = keyInfo.getLocalID();
    OzoneKeyDetails keyDetails = bucket.getKey(keyName);
    Assert.assertEquals(keyName, keyDetails.getName());
    List<OzoneKeyLocation> keyLocations = keyDetails.getOzoneKeyLocations();
    Assert.assertEquals(1, keyLocations.size());
    Assert.assertEquals(containerID, keyLocations.get(0).getContainerID());
    Assert.assertEquals(localID, keyLocations.get(0).getLocalID());
    // Make sure that the data size matched.
    Assert.assertEquals(value.getBytes(UTF_8).length, keyLocations.get(0).getLength());
    ContainerInfo container = cluster.getStorageContainerManager().getContainerManager().getContainer(ContainerID.valueOf(containerID));
    Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(container.getPipelineID());
    List<DatanodeDetails> datanodes = pipeline.getNodes();
    DatanodeDetails datanodeDetails = datanodes.get(0);
    Assert.assertNotNull(datanodeDetails);
    XceiverClientSpi clientSpi = factory.acquireClient(pipeline);
    Assert.assertTrue(clientSpi instanceof XceiverClientRatis);
    XceiverClientRatis ratisClient = (XceiverClientRatis) clientSpi;
    ratisClient.watchForCommit(keyInfo.getBlockCommitSequenceId());
    // shutdown the datanode
    cluster.shutdownHddsDatanode(datanodeDetails);
    // try to read, this should be successful
    readKey(bucket, keyName, value);
    // read intermediate directory
    verifyIntermediateDir(bucket, "a/b/c");
    // shutdown the second datanode
    datanodeDetails = datanodes.get(1);
    cluster.shutdownHddsDatanode(datanodeDetails);
    // we still should be able to read via Standalone protocol
    // try to read
    readKey(bucket, keyName, value);
    // shutdown the 3rd datanode
    datanodeDetails = datanodes.get(2);
    cluster.shutdownHddsDatanode(datanodeDetails);
    try {
        // try to read
        readKey(bucket, keyName, value);
        fail("Expected exception not thrown");
    } catch (IOException e) {
    // it should throw an ioException as none of the servers
    // are available
    }
    factory.releaseClient(clientSpi, false);
}
Also used : OzoneKeyLocation(org.apache.hadoop.ozone.client.OzoneKeyLocation) XceiverClientRatis(org.apache.hadoop.hdds.scm.XceiverClientRatis) OzoneOutputStream(org.apache.hadoop.ozone.client.io.OzoneOutputStream) IOException(java.io.IOException) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) XceiverClientSpi(org.apache.hadoop.hdds.scm.XceiverClientSpi) OmKeyArgs(org.apache.hadoop.ozone.om.helpers.OmKeyArgs) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) OzoneVolume(org.apache.hadoop.ozone.client.OzoneVolume) OzoneBucket(org.apache.hadoop.ozone.client.OzoneBucket) OzoneKeyDetails(org.apache.hadoop.ozone.client.OzoneKeyDetails) DatanodeDetails(org.apache.hadoop.hdds.protocol.DatanodeDetails) ContainerInfo(org.apache.hadoop.hdds.scm.container.ContainerInfo) KeyOutputStream(org.apache.hadoop.ozone.client.io.KeyOutputStream) Test(org.junit.Test)

Example 5 with XceiverClientFactory

use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.

the class KeyInputStream method initialize.

private synchronized void initialize(OmKeyInfo keyInfo, List<OmKeyLocationInfo> blockInfos, XceiverClientFactory xceiverClientFactory, boolean verifyChecksum, Function<OmKeyInfo, OmKeyInfo> retryFunction) {
    this.key = keyInfo.getKeyName();
    this.blockOffsets = new long[blockInfos.size()];
    long keyLength = 0;
    for (int i = 0; i < blockInfos.size(); i++) {
        OmKeyLocationInfo omKeyLocationInfo = blockInfos.get(i);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding stream for accessing {}. The stream will be " + "initialized later.", omKeyLocationInfo);
        }
        // We also pass in functional reference which is used to refresh the
        // pipeline info for a given OM Key location info.
        addStream(omKeyLocationInfo, xceiverClientFactory, verifyChecksum, keyLocationInfo -> {
            OmKeyInfo newKeyInfo = retryFunction.apply(keyInfo);
            BlockID blockID = keyLocationInfo.getBlockID();
            List<OmKeyLocationInfo> collect = newKeyInfo.getLatestVersionLocations().getLocationList().stream().filter(l -> l.getBlockID().equals(blockID)).collect(Collectors.toList());
            if (CollectionUtils.isNotEmpty(collect)) {
                return collect.get(0).getPipeline();
            } else {
                return null;
            }
        });
        this.blockOffsets[i] = keyLength;
        keyLength += omKeyLocationInfo.getLength();
    }
    this.length = keyLength;
}
Also used : Arrays(java.util.Arrays) BlockID(org.apache.hadoop.hdds.client.BlockID) BlockInputStream(org.apache.hadoop.hdds.scm.storage.BlockInputStream) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) Function(java.util.function.Function) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) Seekable(org.apache.hadoop.fs.Seekable) CanUnbuffer(org.apache.hadoop.fs.CanUnbuffer) ByteArrayReader(org.apache.hadoop.hdds.scm.storage.ByteArrayReader) CollectionUtils(org.apache.commons.collections.CollectionUtils) Map(java.util.Map) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo) XceiverClientFactory(org.apache.hadoop.hdds.scm.XceiverClientFactory) Logger(org.slf4j.Logger) FSExceptionMessages(org.apache.hadoop.fs.FSExceptionMessages) IOException(java.io.IOException) Pipeline(org.apache.hadoop.hdds.scm.pipeline.Pipeline) EOFException(java.io.EOFException) Collectors(java.util.stream.Collectors) ByteBufferReadable(org.apache.hadoop.fs.ByteBufferReadable) List(java.util.List) ByteBufferReader(org.apache.hadoop.hdds.scm.storage.ByteBufferReader) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) InputStream(java.io.InputStream) ByteReaderStrategy(org.apache.hadoop.hdds.scm.storage.ByteReaderStrategy) OmKeyInfo(org.apache.hadoop.ozone.om.helpers.OmKeyInfo) BlockID(org.apache.hadoop.hdds.client.BlockID) OmKeyLocationInfo(org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo)

Aggregations

XceiverClientFactory (org.apache.hadoop.hdds.scm.XceiverClientFactory)6 Pipeline (org.apache.hadoop.hdds.scm.pipeline.Pipeline)5 Test (org.junit.Test)4 IOException (java.io.IOException)3 BlockID (org.apache.hadoop.hdds.client.BlockID)3 XceiverClientSpi (org.apache.hadoop.hdds.scm.XceiverClientSpi)3 OmKeyLocationInfo (org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo)3 ByteBuffer (java.nio.ByteBuffer)2 DatanodeDetails (org.apache.hadoop.hdds.protocol.DatanodeDetails)2 ChunkInfo (org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo)2 MockPipeline (org.apache.hadoop.hdds.scm.pipeline.MockPipeline)2 MockXceiverClientFactory (org.apache.hadoop.ozone.client.MockXceiverClientFactory)2 OzoneBucket (org.apache.hadoop.ozone.client.OzoneBucket)2 OzoneVolume (org.apache.hadoop.ozone.client.OzoneVolume)2 RpcClient (org.apache.hadoop.ozone.client.rpc.RpcClient)2 OmKeyInfo (org.apache.hadoop.ozone.om.helpers.OmKeyInfo)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 EOFException (java.io.EOFException)1 InputStream (java.io.InputStream)1