use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.
the class TestReplicatedFileChecksumHelper method init.
@Before
public void init() throws IOException {
ConfigurationSource config = new InMemoryConfiguration();
rpcClient = new RpcClient(config, null) {
@Override
protected OmTransport createOmTransport(String omServiceId) throws IOException {
return new MockOmTransport();
}
@NotNull
@Override
protected XceiverClientFactory createXceiverClientFactory(List<X509Certificate> x509Certificates) throws IOException {
return new MockXceiverClientFactory();
}
};
client = new OzoneClient(config, rpcClient);
store = client.getObjectStore();
}
use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.
the class TestReplicatedFileChecksumHelper method testOneBlock.
@Test
public void testOneBlock() throws IOException {
// test the file checksum of a file with one block.
OzoneConfiguration conf = new OzoneConfiguration();
RpcClient mockRpcClient = Mockito.mock(RpcClient.class);
List<DatanodeDetails> dns = Arrays.asList(DatanodeDetails.newBuilder().setUuid(UUID.randomUUID()).build());
Pipeline pipeline;
pipeline = Pipeline.newBuilder().setId(PipelineID.randomId()).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)).setState(Pipeline.PipelineState.CLOSED).setNodes(dns).build();
XceiverClientGrpc xceiverClientGrpc = new XceiverClientGrpc(pipeline, conf) {
@Override
public XceiverClientReply sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request, DatanodeDetails dn) {
return buildValidResponse();
}
};
XceiverClientFactory factory = Mockito.mock(XceiverClientFactory.class);
when(factory.acquireClientForReadData(ArgumentMatchers.any())).thenReturn(xceiverClientGrpc);
when(mockRpcClient.getXceiverClientManager()).thenReturn(factory);
OzoneManagerProtocol om = Mockito.mock(OzoneManagerProtocol.class);
when(mockRpcClient.getOzoneManagerClient()).thenReturn(om);
BlockID blockID = new BlockID(1, 1);
OmKeyLocationInfo omKeyLocationInfo = new OmKeyLocationInfo.Builder().setPipeline(pipeline).setBlockID(blockID).build();
List<OmKeyLocationInfo> omKeyLocationInfoList = Arrays.asList(omKeyLocationInfo);
OmKeyInfo omKeyInfo = new OmKeyInfo.Builder().setVolumeName(null).setBucketName(null).setKeyName(null).setOmKeyLocationInfos(Collections.singletonList(new OmKeyLocationInfoGroup(0, omKeyLocationInfoList))).setCreationTime(Time.now()).setModificationTime(Time.now()).setDataSize(0).setReplicationConfig(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)).setFileEncryptionInfo(null).setAcls(null).build();
when(om.lookupKey(ArgumentMatchers.any())).thenReturn(omKeyInfo);
OzoneVolume mockVolume = Mockito.mock(OzoneVolume.class);
when(mockVolume.getName()).thenReturn("vol1");
OzoneBucket bucket = Mockito.mock(OzoneBucket.class);
when(bucket.getName()).thenReturn("bucket1");
ReplicatedFileChecksumHelper helper = new ReplicatedFileChecksumHelper(mockVolume, bucket, "dummy", 10, mockRpcClient);
helper.compute();
FileChecksum fileChecksum = helper.getFileChecksum();
assertTrue(fileChecksum instanceof MD5MD5CRC32GzipFileChecksum);
assertEquals(1, helper.getKeyLocationInfoList().size());
}
use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.
the class TestChunkInputStream method connectsToNewPipeline.
@Test
public void connectsToNewPipeline() throws Exception {
// GIVEN
Pipeline pipeline = MockPipeline.createSingleNodePipeline();
Pipeline newPipeline = MockPipeline.createSingleNodePipeline();
XceiverClientFactory clientFactory = mock(XceiverClientFactory.class);
XceiverClientSpi client = mock(XceiverClientSpi.class);
when(clientFactory.acquireClientForReadData(pipeline)).thenReturn(client);
AtomicReference<Pipeline> pipelineRef = new AtomicReference<>(pipeline);
ChunkInputStream subject = new ChunkInputStream(chunkInfo, null, clientFactory, pipelineRef::get, false, null) {
@Override
protected ByteBuffer[] readChunk(ChunkInfo readChunkInfo) {
return ByteString.copyFrom(chunkData).asReadOnlyByteBufferList().toArray(new ByteBuffer[0]);
}
};
try {
// WHEN
subject.unbuffer();
pipelineRef.set(newPipeline);
int b = subject.read();
// THEN
Assert.assertNotEquals(-1, b);
verify(clientFactory).acquireClientForReadData(newPipeline);
} finally {
subject.close();
}
}
use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.
the class TestReadRetries method testPutKeyAndGetKeyThreeNodes.
@Test
public void testPutKeyAndGetKeyThreeNodes() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
String value = "sample value";
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
String keyName = "a/b/c/" + UUID.randomUUID().toString();
OzoneOutputStream out = bucket.createKey(keyName, value.getBytes(UTF_8).length, ReplicationType.RATIS, ReplicationFactor.THREE, new HashMap<>());
KeyOutputStream groupOutputStream = (KeyOutputStream) out.getOutputStream();
XceiverClientFactory factory = groupOutputStream.getXceiverClientFactory();
out.write(value.getBytes(UTF_8));
out.close();
// First, confirm the key info from the client matches the info in OM.
OmKeyArgs.Builder builder = new OmKeyArgs.Builder();
builder.setVolumeName(volumeName).setBucketName(bucketName).setKeyName(keyName).setRefreshPipeline(true);
OmKeyLocationInfo keyInfo = ozoneManager.lookupKey(builder.build()).getKeyLocationVersions().get(0).getBlocksLatestVersionOnly().get(0);
long containerID = keyInfo.getContainerID();
long localID = keyInfo.getLocalID();
OzoneKeyDetails keyDetails = bucket.getKey(keyName);
Assert.assertEquals(keyName, keyDetails.getName());
List<OzoneKeyLocation> keyLocations = keyDetails.getOzoneKeyLocations();
Assert.assertEquals(1, keyLocations.size());
Assert.assertEquals(containerID, keyLocations.get(0).getContainerID());
Assert.assertEquals(localID, keyLocations.get(0).getLocalID());
// Make sure that the data size matched.
Assert.assertEquals(value.getBytes(UTF_8).length, keyLocations.get(0).getLength());
ContainerInfo container = cluster.getStorageContainerManager().getContainerManager().getContainer(ContainerID.valueOf(containerID));
Pipeline pipeline = cluster.getStorageContainerManager().getPipelineManager().getPipeline(container.getPipelineID());
List<DatanodeDetails> datanodes = pipeline.getNodes();
DatanodeDetails datanodeDetails = datanodes.get(0);
Assert.assertNotNull(datanodeDetails);
XceiverClientSpi clientSpi = factory.acquireClient(pipeline);
Assert.assertTrue(clientSpi instanceof XceiverClientRatis);
XceiverClientRatis ratisClient = (XceiverClientRatis) clientSpi;
ratisClient.watchForCommit(keyInfo.getBlockCommitSequenceId());
// shutdown the datanode
cluster.shutdownHddsDatanode(datanodeDetails);
// try to read, this should be successful
readKey(bucket, keyName, value);
// read intermediate directory
verifyIntermediateDir(bucket, "a/b/c");
// shutdown the second datanode
datanodeDetails = datanodes.get(1);
cluster.shutdownHddsDatanode(datanodeDetails);
// we still should be able to read via Standalone protocol
// try to read
readKey(bucket, keyName, value);
// shutdown the 3rd datanode
datanodeDetails = datanodes.get(2);
cluster.shutdownHddsDatanode(datanodeDetails);
try {
// try to read
readKey(bucket, keyName, value);
fail("Expected exception not thrown");
} catch (IOException e) {
// it should throw an ioException as none of the servers
// are available
}
factory.releaseClient(clientSpi, false);
}
use of org.apache.hadoop.hdds.scm.XceiverClientFactory in project ozone by apache.
the class KeyInputStream method initialize.
private synchronized void initialize(OmKeyInfo keyInfo, List<OmKeyLocationInfo> blockInfos, XceiverClientFactory xceiverClientFactory, boolean verifyChecksum, Function<OmKeyInfo, OmKeyInfo> retryFunction) {
this.key = keyInfo.getKeyName();
this.blockOffsets = new long[blockInfos.size()];
long keyLength = 0;
for (int i = 0; i < blockInfos.size(); i++) {
OmKeyLocationInfo omKeyLocationInfo = blockInfos.get(i);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding stream for accessing {}. The stream will be " + "initialized later.", omKeyLocationInfo);
}
// We also pass in functional reference which is used to refresh the
// pipeline info for a given OM Key location info.
addStream(omKeyLocationInfo, xceiverClientFactory, verifyChecksum, keyLocationInfo -> {
OmKeyInfo newKeyInfo = retryFunction.apply(keyInfo);
BlockID blockID = keyLocationInfo.getBlockID();
List<OmKeyLocationInfo> collect = newKeyInfo.getLatestVersionLocations().getLocationList().stream().filter(l -> l.getBlockID().equals(blockID)).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(collect)) {
return collect.get(0).getPipeline();
} else {
return null;
}
});
this.blockOffsets[i] = keyLength;
keyLength += omKeyLocationInfo.getLength();
}
this.length = keyLength;
}
Aggregations