Examples with RegisterWorkerPResponse alluxio.grpc.RegisterWorkerPResponse used on opensource projects

Example 1 with RegisterWorkerPResponse

use of alluxio.grpc.RegisterWorkerPResponse in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandlerTest method registerLeaseExpired.

@Test
public void registerLeaseExpired() {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // Prepare LocationBlockIdListEntry objects
    BlockStoreLocation loc = new BlockStoreLocation("MEM", 0);
    BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
    BlockIdList blockIdList1 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(1L, 2L)).build();
    LocationBlockIdListEntry listEntry1 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList1).build();
    // Prepare a lease
    GetRegisterLeasePRequest leaseRequest = GetRegisterLeasePRequest.newBuilder().setWorkerId(workerId).setBlockCount(blockIdList1.getBlockIdCount()).build();
    Optional<RegisterLease> lease = mBlockMaster.tryAcquireRegisterLease(leaseRequest);
    assertTrue(lease.isPresent());
    // Sleep for a while so that the lease expires
    SleepUtils.sleepMs(5000);
    // The lease is recycled and taken away
    GetRegisterLeasePRequest newLeaseRequest = GetRegisterLeasePRequest.newBuilder().setWorkerId(workerId + 1).setBlockCount(blockIdList1.getBlockIdCount()).build();
    Optional<RegisterLease> newLease = mBlockMaster.tryAcquireRegisterLease(newLeaseRequest);
    assertTrue(newLease.isPresent());
    RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addStorageTiers("MEM").putTotalBytesOnTiers("MEM", 1000L).putUsedBytesOnTiers("MEM", 0L).setOptions(RegisterWorkerPOptions.getDefaultInstance()).addCurrentBlocks(listEntry1).build();
    // Noop response observer
    Queue<Throwable> errors = new ConcurrentLinkedDeque<>();
    StreamObserver<RegisterWorkerPResponse> noopResponseObserver = new StreamObserver<RegisterWorkerPResponse>() {

        @Override
        public void onNext(RegisterWorkerPResponse response) {
        }

        @Override
        public void onError(Throwable t) {
            errors.offer(t);
        }

        @Override
        public void onCompleted() {
        }
    };
    mHandler.registerWorker(request, noopResponseObserver);
    assertEquals(1, errors.size());
    Throwable t = errors.poll();
    Assert.assertThat(t.getMessage(), containsString("does not have a lease or the lease has expired."));
    mBlockMaster.releaseRegisterLease(workerId + 1);
}

Example 2 with RegisterWorkerPResponse

use of alluxio.grpc.RegisterWorkerPResponse in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandlerTest method registerWithNoLeaseIsRejected.

@Test
public void registerWithNoLeaseIsRejected() {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // Prepare LocationBlockIdListEntry objects
    BlockStoreLocation loc = new BlockStoreLocation("MEM", 0);
    BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
    BlockIdList blockIdList1 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(1L, 2L)).build();
    LocationBlockIdListEntry listEntry1 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList1).build();
    RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addStorageTiers("MEM").putTotalBytesOnTiers("MEM", 1000L).putUsedBytesOnTiers("MEM", 0L).setOptions(RegisterWorkerPOptions.getDefaultInstance()).addCurrentBlocks(listEntry1).build();
    Queue<Throwable> errors = new ConcurrentLinkedDeque<>();
    StreamObserver<RegisterWorkerPResponse> noopResponseObserver = new StreamObserver<RegisterWorkerPResponse>() {

        @Override
        public void onNext(RegisterWorkerPResponse response) {
        }

        @Override
        public void onError(Throwable t) {
            errors.offer(t);
        }

        @Override
        public void onCompleted() {
        }
    };
    // The responseObserver should see an error
    mHandler.registerWorker(request, noopResponseObserver);
    assertEquals(1, errors.size());
    Throwable t = errors.poll();
    Assert.assertThat(t.getMessage(), containsString("does not have a lease or the lease has expired."));
}

Example 3 with RegisterWorkerPResponse

use of alluxio.grpc.RegisterWorkerPResponse in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandlerTest method registerWorkerFailsOnDuplicateBlockLocation.

@Test
public void registerWorkerFailsOnDuplicateBlockLocation() throws Exception {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // Prepare LocationBlockIdListEntry objects
    BlockStoreLocation loc = new BlockStoreLocation("MEM", 0);
    BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
    BlockIdList blockIdList1 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(1L, 2L)).build();
    LocationBlockIdListEntry listEntry1 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList1).build();
    BlockIdList blockIdList2 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(3L, 4L)).build();
    LocationBlockIdListEntry listEntry2 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList2).build();
    // Prepare a lease
    GetRegisterLeasePRequest leaseRequest = GetRegisterLeasePRequest.newBuilder().setWorkerId(workerId).setBlockCount(blockIdList1.getBlockIdCount() + blockIdList2.getBlockIdCount()).build();
    Optional<RegisterLease> lease = mBlockMaster.tryAcquireRegisterLease(leaseRequest);
    assertTrue(lease.isPresent());
    // The request is not deduplicated
    RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addStorageTiers("MEM").putTotalBytesOnTiers("MEM", 1000L).putUsedBytesOnTiers("MEM", 0L).setOptions(RegisterWorkerPOptions.getDefaultInstance()).addCurrentBlocks(listEntry1).addCurrentBlocks(listEntry2).build();
    // Noop response observer
    StreamObserver<RegisterWorkerPResponse> noopResponseObserver = new StreamObserver<RegisterWorkerPResponse>() {

        @Override
        public void onNext(RegisterWorkerPResponse response) {
        }

        @Override
        public void onError(Throwable t) {
        }

        @Override
        public void onCompleted() {
        }
    };
    assertThrows(AssertionError.class, () -> {
        mHandler.registerWorker(request, noopResponseObserver);
    });
    mBlockMaster.releaseRegisterLease(workerId);
}

Example 4 with RegisterWorkerPResponse

use of alluxio.grpc.RegisterWorkerPResponse in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandlerTest method registerLeaseTurnedOff.

@Test
public void registerLeaseTurnedOff() throws Exception {
    initServiceHandler(false);
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // Prepare LocationBlockIdListEntry objects
    BlockStoreLocation loc = new BlockStoreLocation("MEM", 0);
    BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
    BlockIdList blockIdList1 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(1L, 2L)).build();
    LocationBlockIdListEntry listEntry1 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList1).build();
    // No lease is acquired
    RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addStorageTiers("MEM").putTotalBytesOnTiers("MEM", 1000L).putUsedBytesOnTiers("MEM", 0L).setOptions(RegisterWorkerPOptions.getDefaultInstance()).addCurrentBlocks(listEntry1).build();
    Queue<Throwable> errors = new ConcurrentLinkedDeque<>();
    StreamObserver<RegisterWorkerPResponse> noopResponseObserver = new StreamObserver<RegisterWorkerPResponse>() {

        @Override
        public void onNext(RegisterWorkerPResponse response) {
        }

        @Override
        public void onError(Throwable t) {
            errors.offer(t);
        }

        @Override
        public void onCompleted() {
        }
    };
    mHandler.registerWorker(request, noopResponseObserver);
    assertEquals(0, errors.size());
}

Example 5 with RegisterWorkerPResponse

use of alluxio.grpc.RegisterWorkerPResponse in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method deleteDuringRegisterStream.

/**
 * Tests below cover the race conditions during concurrent executions.
 *
 * If a worker is new to the cluster, no clients should know this worker.
 * Therefore there is no concurrent client-incurred write operations on this worker.
 * The races happen typically when the worker re-registers with the master,
 * where some clients already know this worker and can direct invoke writes on the worker.
 *
 * Tests here verify the integrity of the worker-side metadata.
 * In other words, even a delete/move happens on the worker during the register stream,
 * the change should be successful and the update should be recorded correctly.
 * The update should later be reported to the master.
 */
@Test
public void deleteDuringRegisterStream() throws Exception {
    // Generate a request stream of blocks
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Select a block to remove concurrent with the stream
    long blockToRemove = findFirstBlock(requestChunks);
    // Manually create a stream and control the request/response observers
    BlockMasterWorkerServiceStub asyncClient = PowerMockito.mock(BlockMasterWorkerServiceStub.class);
    when(asyncClient.withDeadlineAfter(anyLong(), any())).thenReturn(asyncClient);
    CountDownLatch signalLatch = new CountDownLatch(1);
    ManualRegisterStreamObserver requestObserver = new ManualRegisterStreamObserver(MasterMode.SIGNAL_IN_STREAM, signalLatch);
    when(asyncClient.registerWorkerStream(any())).thenReturn(requestObserver);
    RegisterStreamer registerStreamer = new RegisterStreamer(asyncClient, WORKER_ID, mTierAliases, mCapacityMap, mUsedMap, mBlockMap, LOST_STORAGE, EMPTY_CONFIG);
    StreamObserver<RegisterWorkerPResponse> responseObserver = getResponseObserver(registerStreamer);
    requestObserver.setResponseObserver(responseObserver);
    // Prepare the block worker to use the overriden stream
    MasterClientContext context = MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build();
    // On heartbeat, the expected values will be checked against
    List<Long> expectedLostBlocks = ImmutableList.of(blockToRemove);
    Map<BlockStoreLocation, List<Long>> expectedAddedBlocks = ImmutableMap.of();
    mBlockMasterClient = new TestBlockMasterClient(context, registerStreamer, expectedLostBlocks, expectedAddedBlocks);
    initBlockWorker();
    // Prepare the blocks in the BlockWorker storage that match the register stream
    prepareBlocksOnWorker(TIER_CONFIG);
    // Let a delete job wait on the signal
    AtomicReference<Throwable> error = new AtomicReference<>();
    Future f = mExecutorService.submit(() -> {
        try {
            signalLatch.await();
            mBlockWorker.removeBlock(1L, blockToRemove);
        } catch (Exception e) {
            error.set(e);
        }
    });
    // Kick off the register stream
    AtomicReference<Long> workerId = new AtomicReference<>(WORKER_ID);
    BlockMasterSync sync = new BlockMasterSync(mBlockWorker, workerId, NET_ADDRESS_1, mBlockMasterClientPool);
    // Check the next heartbeat to be sent to the master
    f.get();
    assertNull(error.get());
    // Validation will happen on the heartbeat
    sync.heartbeat();
}