Search in sources :

Example 1 with BlockMasterSync

use of alluxio.worker.block.BlockMasterSync in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.

/**
 * Tests below cover the race conditions during concurrent executions.
 *
 * If a worker is new to the cluster, no clients should know this worker.
 * Therefore there is no concurrent client-incurred write operations on this worker.
 * The races happen typically when the worker re-registers with the master,
 * where some clients already know this worker and can direct invoke writes on the worker.
 *
 * Tests here verify the integrity of the worker-side metadata.
 * In other words, even a delete/move happens on the worker during the register stream,
 * the change should be successful and the update should be recorded correctly.
 * The update should later be reported to the master.
 */
@Test
public void deleteDuringRegisterStream() throws Exception {
    // Generate a request stream of blocks
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Select a block to remove concurrent with the stream
    long blockToRemove = findFirstBlock(requestChunks);
    // Manually create a stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    CountDownLatch signalLatch = new CountDownLatch(1);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // Prepare the block worker to use the overriden stream
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
    // On heartbeat, the expected values will be checked against
    List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
    Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
    mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
    initBlockWorker();
    // Prepare the blocks in the BlockWorker storage that match the register stream
    prepareBlocksOnWorker(TIER_CONFIG);
    // Let a delete job wait on the signal
    AtomicReference<Throwable> error = new AtomicReference<>();
    Future f = mExecutorService.submit(() -> {
        try {
            signalLatch.await();
            mBlockWorker.removeBlock(1L, blockToRemove);
        } catch (Exception e) {
            error.set(e);
        }
    });
    // Kick off the register stream
    AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
    BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
    // Check the next heartbeat to be sent to the master
    f.get();
    assertNull(error.get());
    // Validation will happen on the heartbeat
    sync.heartbeat();
}
Also used : MasterClientContext(alluxio.master.MasterClientContext) BlockMasterSync(alluxio.worker.block.BlockMasterSync) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) AtomicReference(java.util.concurrent.atomic.AtomicReference) RegisterWorkerPRequest(alluxio.grpc.RegisterWorkerPRequest) CountDownLatch(java.util.concurrent.CountDownLatch) DeadlineExceededException(alluxio.exception.status.DeadlineExceededException) InternalException(alluxio.exception.status.InternalException) StatusException(io.grpc.StatusException) IOException(java.io.IOException) NotFoundException(alluxio.exception.status.NotFoundException) RegisterStreamer(alluxio.worker.block.RegisterStreamer) RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) Future(java.util.concurrent.Future) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) StorageList(alluxio.grpc.StorageList) BlockStoreLocation(alluxio.worker.block.BlockStoreLocation) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

DeadlineExceededException (alluxio.exception.status.DeadlineExceededException)1 InternalException (alluxio.exception.status.InternalException)1 NotFoundException (alluxio.exception.status.NotFoundException)1 BlockMasterWorkerServiceStub (alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub)1 RegisterWorkerPRequest (alluxio.grpc.RegisterWorkerPRequest)1 RegisterWorkerPResponse (alluxio.grpc.RegisterWorkerPResponse)1 StorageList (alluxio.grpc.StorageList)1 MasterClientContext (alluxio.master.MasterClientContext)1 BlockMasterSync (alluxio.worker.block.BlockMasterSync)1 BlockStoreLocation (alluxio.worker.block.BlockStoreLocation)1 RegisterStreamer (alluxio.worker.block.RegisterStreamer)1 ImmutableList (com.google.common.collect.ImmutableList)1 StatusException (io.grpc.StatusException)1 IOException (java.io.IOException)1 List (java.util.List)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Future (java.util.concurrent.Future)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 Test (org.junit.Test)1 ArgumentMatchers.anyLong (org.mockito.ArgumentMatchers.anyLong)1