Search in sources :

Example 1 with BlockMasterWorkerServiceStub

use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.

/**
 * Tests below cover the race conditions during concurrent executions.
 *
 * If a worker is new to the cluster, no clients should know this worker.
 * Therefore there is no concurrent client-incurred write operations on this worker.
 * The races happen typically when the worker re-registers with the master,
 * where some clients already know this worker and can direct invoke writes on the worker.
 *
 * Tests here verify the integrity of the worker-side metadata.
 * In other words, even a delete/move happens on the worker during the register stream,
 * the change should be successful and the update should be recorded correctly.
 * The update should later be reported to the master.
 */
@Test
public void deleteDuringRegisterStream() throws Exception {
    // Generate a request stream of blocks
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Select a block to remove concurrent with the stream
    long blockToRemove = findFirstBlock(requestChunks);
    // Manually create a stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    CountDownLatch signalLatch = new CountDownLatch(1);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // Prepare the block worker to use the overriden stream
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
    // On heartbeat, the expected values will be checked against
    List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
    Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
    mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
    initBlockWorker();
    // Prepare the blocks in the BlockWorker storage that match the register stream
    prepareBlocksOnWorker(TIER_CONFIG);
    // Let a delete job wait on the signal
    AtomicReference<Throwable> error = new AtomicReference<>();
    Future f = mExecutorService.submit(() -> {
        try {
            signalLatch.await();
            mBlockWorker.removeBlock(1L, blockToRemove);
        } catch (Exception e) {
            error.set(e);
        }
    });
    // Kick off the register stream
    AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
    BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
    // Check the next heartbeat to be sent to the master
    f.get();
    assertNull(error.get());
    // Validation will happen on the heartbeat
    sync.heartbeat();
}
Also used : MasterClientContext(alluxio.master.MasterClientContext) BlockMasterSync(alluxio.worker.block.BlockMasterSync) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) AtomicReference(java.util.concurrent.atomic.AtomicReference) RegisterWorkerPRequest(alluxio.grpc.RegisterWorkerPRequest) CountDownLatch(java.util.concurrent.CountDownLatch) DeadlineExceededException(alluxio.exception.status.DeadlineExceededException) InternalException(alluxio.exception.status.InternalException) StatusException(io.grpc.StatusException) IOException(java.io.IOException) NotFoundException(alluxio.exception.status.NotFoundException) RegisterStreamer(alluxio.worker.block.RegisterStreamer) RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) Future(java.util.concurrent.Future) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) StorageList(alluxio.grpc.StorageList) BlockStoreLocation(alluxio.worker.block.BlockStoreLocation) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 2 with BlockMasterWorkerServiceStub

use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method masterHangsOnCompleted.

@Test
public void masterHangsOnCompleted() throws Exception {
    // Create the stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.HANG_ON_COMPLETED);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // The master will hang at the completion step which exceeds the deadline between messages
    assertThrows(DeadlineExceededException.class, () -> {
        registerStreamer.registerWithMaster();
    });
}
Also used : RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) RegisterStreamer(alluxio.worker.block.RegisterStreamer) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 3 with BlockMasterWorkerServiceStub

use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method registerWorkerErrorAtCompletion.

@Test
public void registerWorkerErrorAtCompletion() throws Exception {
    // Create the stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.ON_COMPLETED);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    assertThrows(InternalException.class, () -> {
        // An error received from the master side will be InternalException
        registerStreamer.registerWithMaster();
    });
}
Also used : RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) RegisterStreamer(alluxio.worker.block.RegisterStreamer) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with BlockMasterWorkerServiceStub

use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method normalFlow.

@Test
public void normalFlow() throws Exception {
    // Create the stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    AtomicInteger receivedCount = new AtomicInteger();
    AtomicBoolean completed = new AtomicBoolean();
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.COMPLETE_AND_REPORT, receivedCount, completed);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    registerStreamer.registerWithMaster();
    int expectedBatchCount = (int) Math.ceil((TIER_BLOCK_TOTAL) / (double) BATCH_SIZE);
    assertEquals(expectedBatchCount, receivedCount.get());
    assertEquals(true, completed.get());
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) RegisterStreamer(alluxio.worker.block.RegisterStreamer) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 5 with BlockMasterWorkerServiceStub

use of alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method registerWorkerErrorDuringStream.

@Test
public void registerWorkerErrorDuringStream() throws Exception {
    // Create the stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SECOND_REQUEST);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    assertThrows(InternalException.class, () -> {
        // An error received from the master side will be InternalException
        registerStreamer.registerWithMaster();
    });
}
Also used : RegisterWorkerPResponse(alluxio.grpc.RegisterWorkerPResponse) BlockMasterWorkerServiceStub(alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub) RegisterStreamer(alluxio.worker.block.RegisterStreamer) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

BlockMasterWorkerServiceStub (alluxio.grpc.BlockMasterWorkerServiceGrpc.BlockMasterWorkerServiceStub)7 RegisterWorkerPResponse (alluxio.grpc.RegisterWorkerPResponse)7 RegisterStreamer (alluxio.worker.block.RegisterStreamer)7 Test (org.junit.Test)7 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)7 DeadlineExceededException (alluxio.exception.status.DeadlineExceededException)1 InternalException (alluxio.exception.status.InternalException)1 NotFoundException (alluxio.exception.status.NotFoundException)1 RegisterWorkerPRequest (alluxio.grpc.RegisterWorkerPRequest)1 StorageList (alluxio.grpc.StorageList)1 MasterClientContext (alluxio.master.MasterClientContext)1 BlockMasterSync (alluxio.worker.block.BlockMasterSync)1 BlockStoreLocation (alluxio.worker.block.BlockStoreLocation)1 ImmutableList (com.google.common.collect.ImmutableList)1 StatusException (io.grpc.StatusException)1 IOException (java.io.IOException)1 List (java.util.List)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Future (java.util.concurrent.Future)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1