use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.
the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.
/**
* Tests below cover the race conditions during concurrent executions.
*
* If a worker is new to the cluster, no clients should know this worker.
* Therefore there is no concurrent client-incurred write operations on this worker.
* The races happen typically when the worker re-registers with the master,
* where some clients already know this worker and can direct invoke writes on the worker.
*
* Tests here verify the integrity of the worker-side metadata.
* In other words, even a delete/move happens on the worker during the register stream,
* the change should be successful and the update should be recorded correctly.
* The update should later be reported to the master.
*/
@Test
public void deleteDuringRegisterStream() throws Exception {
// Generate a request stream of blocks
List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
// Select a block to remove concurrent with the stream
long blockToRemove = findFirstBlock(requestChunks);
// Manually create a stream and control the request/response observers
BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
CountDownLatch signalLatch = new CountDownLatch(1);
ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
requestObserver.setResponseObserver(responseObserver);
// Prepare the block worker to use the overriden stream
MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
// On heartbeat, the expected values will be checked against
List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
initBlockWorker();
// Prepare the blocks in the BlockWorker storage that match the register stream
prepareBlocksOnWorker(TIER_CONFIG);
// Let a delete job wait on the signal
AtomicReference<Throwable> error = new AtomicReference<>();
Future f = mExecutorService.submit(() -> {
try {
signalLatch.await();
mBlockWorker.removeBlock(1L, blockToRemove);
} catch (Exception e) {
error.set(e);
}
});
// Kick off the register stream
AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
// Check the next heartbeat to be sent to the master
f.get();
assertNull(error.get());
// Validation will happen on the heartbeat
sync.heartbeat();
}
use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.
the class BlockWorkerRegisterStreamIntegrationTest method masterHangsOnCompleted.
@Test
public void masterHangsOnCompleted() throws Exception {
// Create the stream and control the request/response observers
BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.HANG_ON_COMPLETED);
when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
requestObserver.setResponseObserver(responseObserver);
// The master will hang at the completion step which exceeds the deadline between messages
assertThrows(DeadlineExceededException.class, () -> {
registerStreamer.registerWithMaster();
});
}
use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.
the class BlockWorkerRegisterStreamIntegrationTest method registerWorkerErrorAtCompletion.
@Test
public void registerWorkerErrorAtCompletion() throws Exception {
// Create the stream and control the request/response observers
BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.ON_COMPLETED);
when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
requestObserver.setResponseObserver(responseObserver);
assertThrows(InternalException.class, () -> {
// An error received from the master side will be InternalException
registerStreamer.registerWithMaster();
});
}
use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.
the class BlockWorkerRegisterStreamIntegrationTest method normalFlow.
@Test
public void normalFlow() throws Exception {
// Create the stream and control the request/response observers
BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
AtomicInteger receivedCount = new AtomicInteger();
AtomicBoolean completed = new AtomicBoolean();
ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.COMPLETE_AND_REPORT, receivedCount, completed);
when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
requestObserver.setResponseObserver(responseObserver);
registerStreamer.registerWithMaster();
int expectedBatchCount = (int) Math.ceil((TIER_BLOCK_TOTAL) / (double) BATCH_SIZE);
assertEquals(expectedBatchCount, receivedCount.get());
assertEquals(true, completed.get());
}
use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.
the class BlockWorkerRegisterStreamIntegrationTest method registerWorkerErrorDuringStream.
@Test
public void registerWorkerErrorDuringStream() throws Exception {
// Create the stream and control the request/response observers
BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SECOND_REQUEST);
when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
requestObserver.setResponseObserver(responseObserver);
assertThrows(InternalException.class, () -> {
// An error received from the master side will be InternalException
registerStreamer.registerWithMaster();
});
}
Aggregations