Examples with RegisterWorkerPRequest alluxio.grpc.RegisterWorkerPRequest used on opensource projects

Example 6 with RegisterWorkerPRequest

use of alluxio.grpc.RegisterWorkerPRequest in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method workerRegisterBatchThrowsError.

@Test
public void workerRegisterBatchThrowsError() throws Exception {
    // Hijack the block master so it throws errors
    BrokenBlockMaster brokenBlockMaster = new BrokenBlockMaster(mMetricsMaster, mMasterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService), WorkerRegisterMode.ERROR_STREAM);
    mBlockMaster = brokenBlockMaster;
    mHandler = new BlockMasterWorkerServiceHandler(brokenBlockMaster);
    long workerId = brokenBlockMaster.getWorkerId(NET_ADDRESS_1);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    RegisterStreamObserver streamOb = new RegisterStreamObserver(brokenBlockMaster, getErrorCapturingResponseObserver(errorQueue));
    // Generate requests
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(workerId);
    prepareBlocksOnMaster(requestChunks);
    for (RegisterWorkerPRequest chunk : requestChunks) {
        streamOb.onNext(chunk);
    }
    streamOb.onCompleted();
    // All requests except the 1st one + onCompleted should receive an error
    assertEquals(requestChunks.size() - 1 + 1, errorQueue.size());
    // The BlockMaster is not throwing error this time
    brokenBlockMaster.setCorrect();
    verifyWorkerCanReregister(workerId, requestChunks, TIER_BLOCK_TOTAL);
}

Example 7 with RegisterWorkerPRequest

use of alluxio.grpc.RegisterWorkerPRequest in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method sendStreamToMaster.

private void sendStreamToMaster(List<RegisterWorkerPRequest> requestChunks, StreamObserver<RegisterWorkerPResponse> responseObserver) {
    StreamObserver<RegisterWorkerPRequest> requestObserver = mHandler.registerWorkerStream(responseObserver);
    for (RegisterWorkerPRequest chunk : requestChunks) {
        requestObserver.onNext(chunk);
    }
    requestObserver.onCompleted();
}

Example 8 with RegisterWorkerPRequest

use of alluxio.grpc.RegisterWorkerPRequest in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method workerRegisterStartThrowsError.

@Test
public void workerRegisterStartThrowsError() throws Exception {
    // Hijack the block master so it throws errors
    BrokenBlockMaster brokenBlockMaster = new BrokenBlockMaster(mMetricsMaster, mMasterContext, mClock, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService), WorkerRegisterMode.ERROR_START);
    mBlockMaster = brokenBlockMaster;
    mHandler = new BlockMasterWorkerServiceHandler(brokenBlockMaster);
    long workerId = brokenBlockMaster.getWorkerId(NET_ADDRESS_1);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    RegisterStreamObserver streamOb = new RegisterStreamObserver(brokenBlockMaster, getErrorCapturingResponseObserver(errorQueue));
    // Generate requests
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(workerId);
    prepareBlocksOnMaster(requestChunks);
    for (RegisterWorkerPRequest chunk : requestChunks) {
        streamOb.onNext(chunk);
    }
    streamOb.onCompleted();
    // All requests + onCompleted should receive an error
    assertEquals(requestChunks.size() + 1, errorQueue.size());
    // The BlockMaster is not throwing error this time
    brokenBlockMaster.setCorrect();
    verifyWorkerCanReregister(workerId, requestChunks, TIER_BLOCK_TOTAL);
}

Example 9 with RegisterWorkerPRequest

use of alluxio.grpc.RegisterWorkerPRequest in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.

/**
 * Tests below cover the race conditions during concurrent executions.
 *
 * If a worker is new to the cluster, no clients should know this worker.
 * Therefore there is no concurrent client-incurred write operations on this worker.
 * The races happen typically when the worker re-registers with the master,
 * where some clients already know this worker and can direct invoke writes on the worker.
 *
 * Tests here verify the integrity of the worker-side metadata.
 * In other words, even a delete/move happens on the worker during the register stream,
 * the change should be successful and the update should be recorded correctly.
 * The update should later be reported to the master.
 */
@Test
public void deleteDuringRegisterStream() throws Exception {
    // Generate a request stream of blocks
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Select a block to remove concurrent with the stream
    long blockToRemove = findFirstBlock(requestChunks);
    // Manually create a stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    CountDownLatch signalLatch = new CountDownLatch(1);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // Prepare the block worker to use the overriden stream
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
    // On heartbeat, the expected values will be checked against
    List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
    Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
    mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
    initBlockWorker();
    // Prepare the blocks in the BlockWorker storage that match the register stream
    prepareBlocksOnWorker(TIER_CONFIG);
    // Let a delete job wait on the signal
    AtomicReference<Throwable> error = new AtomicReference<>();
    Future f = mExecutorService.submit(() -> {
        try {
            signalLatch.await();
            mBlockWorker.removeBlock(1L, blockToRemove);
        } catch (Exception e) {
            error.set(e);
        }
    });
    // Kick off the register stream
    AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
    BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
    // Check the next heartbeat to be sent to the master
    f.get();
    assertNull(error.get());
    // Validation will happen on the heartbeat
    sync.heartbeat();
}

Example 10 with RegisterWorkerPRequest

use of alluxio.grpc.RegisterWorkerPRequest in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method requestsForEmptyWorker.

/**
 * Tests below cover the most normal cases.
 */
@Test
public void requestsForEmptyWorker() throws Exception {
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForEmptyWorker(WORKER_ID);
    // Verify the size and content of the requests
    assertEquals(1, requestChunks.size());
    RegisterWorkerPRequest request = requestChunks.get(0);
    assertEquals(WORKER_ID, request.getWorkerId());
    assertEquals(MEM_USAGE_EMPTY, request.getUsedBytesOnTiersMap());
    assertEquals(MEM_CAPACITY, request.getTotalBytesOnTiersMap());
    Map<String, StorageList> lostMap = request.getLostStorageMap();
    assertEquals(1, lostMap.size());
    assertEquals(StorageList.newBuilder().build(), lostMap.get("MEM"));
    assertEquals(ImmutableList.of("MEM"), request.getStorageTiersList());
    List<LocationBlockIdListEntry> entries = request.getCurrentBlocksList();
    assertEquals(0, entries.size());
}