Examples with ErrorEntry io.grpc.grpclb.GrpclbState.ErrorEntry used on opensource projects

Example 1 with ErrorEntry

use of io.grpc.grpclb.GrpclbState.ErrorEntry in project grpc-java by grpc.

the class GrpclbLoadBalancerTest method grpclbWorking_pickFirstMode.

@Test
public void grpclbWorking_pickFirstMode() throws Exception {
    InOrder inOrder = inOrder(helper);
    List<EquivalentAddressGroup> grpclbBalancerList = createResolvedBalancerAddresses(1);
    deliverResolvedAddresses(Collections.<EquivalentAddressGroup>emptyList(), grpclbBalancerList, GrpclbConfig.create(Mode.PICK_FIRST));
    assertEquals(1, fakeOobChannels.size());
    verify(mockLbService).balanceLoad(lbResponseObserverCaptor.capture());
    StreamObserver<LoadBalanceResponse> lbResponseObserver = lbResponseObserverCaptor.getValue();
    assertEquals(1, lbRequestObservers.size());
    StreamObserver<LoadBalanceRequest> lbRequestObserver = lbRequestObservers.poll();
    verify(lbRequestObserver).onNext(eq(LoadBalanceRequest.newBuilder().setInitialRequest(InitialLoadBalanceRequest.newBuilder().setName(SERVICE_AUTHORITY).build()).build()));
    // Simulate receiving LB response
    List<ServerEntry> backends1 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), new ServerEntry("127.0.0.1", 2010, "token0002"));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    lbResponseObserver.onNext(buildInitialResponse());
    lbResponseObserver.onNext(buildLbResponse(backends1));
    inOrder.verify(helper).createSubchannel(createSubchannelArgsCaptor.capture());
    CreateSubchannelArgs createSubchannelArgs = createSubchannelArgsCaptor.getValue();
    assertThat(createSubchannelArgs.getAddresses()).containsExactly(new EquivalentAddressGroup(backends1.get(0).addr, eagAttrsWithToken("token0001")), new EquivalentAddressGroup(backends1.get(1).addr, eagAttrsWithToken("token0002")));
    // Initially IDLE
    inOrder.verify(helper).updateBalancingState(eq(IDLE), pickerCaptor.capture());
    RoundRobinPicker picker0 = (RoundRobinPicker) pickerCaptor.getValue();
    // Only one subchannel is created
    assertThat(mockSubchannels).hasSize(1);
    Subchannel subchannel = mockSubchannels.poll();
    assertThat(picker0.dropList).containsExactly(null, null);
    assertThat(picker0.pickList).containsExactly(new IdleSubchannelEntry(subchannel, syncContext));
    // PICK_FIRST doesn't eagerly connect
    verify(subchannel, never()).requestConnection();
    // CONNECTING
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
    RoundRobinPicker picker1 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker1.dropList).containsExactly(null, null);
    assertThat(picker1.pickList).containsExactly(BUFFER_ENTRY);
    // TRANSIENT_FAILURE
    Status error = Status.UNAVAILABLE.withDescription("Simulated connection error");
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forTransientFailure(error));
    inOrder.verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture());
    RoundRobinPicker picker2 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker2.dropList).containsExactly(null, null);
    assertThat(picker2.pickList).containsExactly(new ErrorEntry(error));
    // READY
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker3 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker3.dropList).containsExactly(null, null);
    assertThat(picker3.pickList).containsExactly(new BackendEntry(subchannel, new TokenAttachingTracerFactory(getLoadRecorder())));
    // New server list with drops
    List<ServerEntry> backends2 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), // drop
    new ServerEntry("token0003"), new ServerEntry("127.0.0.1", 2020, "token0004"));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    lbResponseObserver.onNext(buildLbResponse(backends2));
    // new addresses will be updated to the existing subchannel
    // createSubchannel() has ever been called only once
    verify(helper, times(1)).createSubchannel(any(CreateSubchannelArgs.class));
    assertThat(mockSubchannels).isEmpty();
    verify(subchannel).updateAddresses(eq(Arrays.asList(new EquivalentAddressGroup(backends2.get(0).addr, eagAttrsWithToken("token0001")), new EquivalentAddressGroup(backends2.get(2).addr, eagAttrsWithToken("token0004")))));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker4 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker4.dropList).containsExactly(null, new DropEntry(getLoadRecorder(), "token0003"), null);
    assertThat(picker4.pickList).containsExactly(new BackendEntry(subchannel, new TokenAttachingTracerFactory(getLoadRecorder())));
    // Subchannel goes IDLE, but PICK_FIRST will not try to reconnect
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(IDLE));
    inOrder.verify(helper).updateBalancingState(eq(IDLE), pickerCaptor.capture());
    RoundRobinPicker picker5 = (RoundRobinPicker) pickerCaptor.getValue();
    verify(subchannel, never()).requestConnection();
    // ... until it's selected
    PickSubchannelArgs args = mock(PickSubchannelArgs.class);
    PickResult pick = picker5.pickSubchannel(args);
    assertThat(pick).isSameInstanceAs(PickResult.withNoResult());
    verify(subchannel).requestConnection();
    // ... or requested by application
    balancer.requestConnection();
    verify(subchannel, times(2)).requestConnection();
    // PICK_FIRST doesn't use subchannelPool
    verify(subchannelPool, never()).takeOrCreateSubchannel(any(EquivalentAddressGroup.class), any(Attributes.class));
    verify(subchannelPool, never()).returnSubchannel(any(Subchannel.class), any(ConnectivityStateInfo.class));
}

Example 2 with ErrorEntry

use of io.grpc.grpclb.GrpclbState.ErrorEntry in project grpc-java by grpc.

the class GrpclbLoadBalancerTest method grpclbWorking_pickFirstMode_lbSendsEmptyAddress.

@Test
public void grpclbWorking_pickFirstMode_lbSendsEmptyAddress() throws Exception {
    InOrder inOrder = inOrder(helper);
    List<EquivalentAddressGroup> grpclbBalancerList = createResolvedBalancerAddresses(1);
    deliverResolvedAddresses(Collections.<EquivalentAddressGroup>emptyList(), grpclbBalancerList, GrpclbConfig.create(Mode.PICK_FIRST));
    assertEquals(1, fakeOobChannels.size());
    verify(mockLbService).balanceLoad(lbResponseObserverCaptor.capture());
    StreamObserver<LoadBalanceResponse> lbResponseObserver = lbResponseObserverCaptor.getValue();
    assertEquals(1, lbRequestObservers.size());
    StreamObserver<LoadBalanceRequest> lbRequestObserver = lbRequestObservers.poll();
    verify(lbRequestObserver).onNext(eq(LoadBalanceRequest.newBuilder().setInitialRequest(InitialLoadBalanceRequest.newBuilder().setName(SERVICE_AUTHORITY).build()).build()));
    // Simulate receiving LB response
    List<ServerEntry> backends1 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), new ServerEntry("127.0.0.1", 2010, "token0002"));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    lbResponseObserver.onNext(buildInitialResponse());
    lbResponseObserver.onNext(buildLbResponse(backends1));
    inOrder.verify(helper).createSubchannel(createSubchannelArgsCaptor.capture());
    CreateSubchannelArgs createSubchannelArgs = createSubchannelArgsCaptor.getValue();
    assertThat(createSubchannelArgs.getAddresses()).containsExactly(new EquivalentAddressGroup(backends1.get(0).addr, eagAttrsWithToken("token0001")), new EquivalentAddressGroup(backends1.get(1).addr, eagAttrsWithToken("token0002")));
    // Initially IDLE
    inOrder.verify(helper).updateBalancingState(eq(IDLE), pickerCaptor.capture());
    RoundRobinPicker picker0 = (RoundRobinPicker) pickerCaptor.getValue();
    // Only one subchannel is created
    assertThat(mockSubchannels).hasSize(1);
    Subchannel subchannel = mockSubchannels.poll();
    assertThat(picker0.dropList).containsExactly(null, null);
    assertThat(picker0.pickList).containsExactly(new IdleSubchannelEntry(subchannel, syncContext));
    // PICK_FIRST doesn't eagerly connect
    verify(subchannel, never()).requestConnection();
    // CONNECTING
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
    RoundRobinPicker picker1 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker1.dropList).containsExactly(null, null);
    assertThat(picker1.pickList).containsExactly(BUFFER_ENTRY);
    // TRANSIENT_FAILURE
    Status error = Status.UNAVAILABLE.withDescription("Simulated connection error");
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forTransientFailure(error));
    inOrder.verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture());
    RoundRobinPicker picker2 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker2.dropList).containsExactly(null, null);
    assertThat(picker2.pickList).containsExactly(new ErrorEntry(error));
    // READY
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker3 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker3.dropList).containsExactly(null, null);
    assertThat(picker3.pickList).containsExactly(new BackendEntry(subchannel, new TokenAttachingTracerFactory(getLoadRecorder())));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    // Empty addresses from LB
    lbResponseObserver.onNext(buildLbResponse(Collections.<ServerEntry>emptyList()));
    // new addresses will be updated to the existing subchannel
    // createSubchannel() has ever been called only once
    inOrder.verify(helper, never()).createSubchannel(any(CreateSubchannelArgs.class));
    assertThat(mockSubchannels).isEmpty();
    verify(subchannel).shutdown();
    // RPC error status includes message of no backends provided by balancer
    inOrder.verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture());
    RoundRobinPicker errorPicker = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(errorPicker.pickList).containsExactly(new ErrorEntry(GrpclbState.NO_AVAILABLE_BACKENDS_STATUS));
    lbResponseObserver.onNext(buildLbResponse(Collections.<ServerEntry>emptyList()));
    // Test recover from new LB response with addresses
    // New server list with drops
    List<ServerEntry> backends2 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), // drop
    new ServerEntry("token0003"), new ServerEntry("127.0.0.1", 2020, "token0004"));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    lbResponseObserver.onNext(buildLbResponse(backends2));
    // new addresses will be updated to the existing subchannel
    inOrder.verify(helper, times(1)).createSubchannel(any(CreateSubchannelArgs.class));
    inOrder.verify(helper).updateBalancingState(eq(IDLE), pickerCaptor.capture());
    subchannel = mockSubchannels.poll();
    // Subchannel became READY
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(CONNECTING));
    deliverSubchannelState(subchannel, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker4 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker4.pickList).containsExactly(new BackendEntry(subchannel, new TokenAttachingTracerFactory(getLoadRecorder())));
}

Example 3 with ErrorEntry

use of io.grpc.grpclb.GrpclbState.ErrorEntry in project grpc-java by grpc.

the class GrpclbLoadBalancerTest method roundRobinMode_subchannelStayTransientFailureUntilReady.

@Test
public void roundRobinMode_subchannelStayTransientFailureUntilReady() {
    InOrder inOrder = inOrder(helper);
    List<EquivalentAddressGroup> grpclbBalancerList = createResolvedBalancerAddresses(1);
    deliverResolvedAddresses(Collections.<EquivalentAddressGroup>emptyList(), grpclbBalancerList);
    verify(mockLbService).balanceLoad(lbResponseObserverCaptor.capture());
    StreamObserver<LoadBalanceResponse> lbResponseObserver = lbResponseObserverCaptor.getValue();
    // Simulate receiving LB response
    List<ServerEntry> backends1 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), new ServerEntry("127.0.0.1", 2010, "token0002"));
    lbResponseObserver.onNext(buildInitialResponse());
    lbResponseObserver.onNext(buildLbResponse(backends1));
    assertEquals(2, mockSubchannels.size());
    Subchannel subchannel1 = mockSubchannels.poll();
    Subchannel subchannel2 = mockSubchannels.poll();
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(CONNECTING));
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), any(SubchannelPicker.class));
    // Switch all subchannels to TRANSIENT_FAILURE, making the general state TRANSIENT_FAILURE too.
    Status error = Status.UNAVAILABLE.withDescription("error");
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forTransientFailure(error));
    inOrder.verify(helper).refreshNameResolution();
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forTransientFailure(error));
    inOrder.verify(helper).refreshNameResolution();
    inOrder.verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture());
    assertThat(((RoundRobinPicker) pickerCaptor.getValue()).pickList).containsExactly(new ErrorEntry(error));
    // Switch subchannel1 to IDLE, then to CONNECTING, which are ignored since the previous
    // subchannel state is TRANSIENT_FAILURE. General state is unchanged.
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(IDLE));
    inOrder.verify(helper).refreshNameResolution();
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verifyNoMoreInteractions();
    // Switch subchannel1 to READY, which will affect the general state
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    assertThat(((RoundRobinPicker) pickerCaptor.getValue()).pickList).containsExactly(new BackendEntry(subchannel1, getLoadRecorder(), "token0001"));
    inOrder.verifyNoMoreInteractions();
}

Example 4 with ErrorEntry

use of io.grpc.grpclb.GrpclbState.ErrorEntry in project grpc-java by grpc.

the class GrpclbLoadBalancerTest method grpclbWorking.

@Test
public void grpclbWorking() {
    InOrder inOrder = inOrder(helper, subchannelPool);
    List<EquivalentAddressGroup> grpclbBalancerList = createResolvedBalancerAddresses(1);
    deliverResolvedAddresses(Collections.<EquivalentAddressGroup>emptyList(), grpclbBalancerList);
    // Fallback timer is started as soon as the addresses are resolved.
    assertEquals(1, fakeClock.numPendingTasks(FALLBACK_MODE_TASK_FILTER));
    verify(helper).createOobChannel(eq(xattr(grpclbBalancerList)), eq(lbAuthority(0) + NO_USE_AUTHORITY_SUFFIX));
    assertEquals(1, fakeOobChannels.size());
    ManagedChannel oobChannel = fakeOobChannels.poll();
    verify(mockLbService).balanceLoad(lbResponseObserverCaptor.capture());
    StreamObserver<LoadBalanceResponse> lbResponseObserver = lbResponseObserverCaptor.getValue();
    assertEquals(1, lbRequestObservers.size());
    StreamObserver<LoadBalanceRequest> lbRequestObserver = lbRequestObservers.poll();
    verify(lbRequestObserver).onNext(eq(LoadBalanceRequest.newBuilder().setInitialRequest(InitialLoadBalanceRequest.newBuilder().setName(SERVICE_AUTHORITY).build()).build()));
    // Simulate receiving LB response
    List<ServerEntry> backends1 = Arrays.asList(new ServerEntry("127.0.0.1", 2000, "token0001"), new ServerEntry("127.0.0.1", 2010, "token0002"));
    inOrder.verify(helper, never()).updateBalancingState(any(ConnectivityState.class), any(SubchannelPicker.class));
    logs.clear();
    lbResponseObserver.onNext(buildInitialResponse());
    assertThat(logs).containsExactly("INFO: [grpclb-<api.google.com>] Got an LB initial response: " + buildInitialResponse());
    logs.clear();
    lbResponseObserver.onNext(buildLbResponse(backends1));
    inOrder.verify(subchannelPool).takeOrCreateSubchannel(eq(new EquivalentAddressGroup(backends1.get(0).addr, LB_BACKEND_ATTRS)), any(Attributes.class));
    inOrder.verify(subchannelPool).takeOrCreateSubchannel(eq(new EquivalentAddressGroup(backends1.get(1).addr, LB_BACKEND_ATTRS)), any(Attributes.class));
    assertEquals(2, mockSubchannels.size());
    Subchannel subchannel1 = mockSubchannels.poll();
    Subchannel subchannel2 = mockSubchannels.poll();
    verify(subchannel1).requestConnection();
    verify(subchannel2).requestConnection();
    assertEquals(new EquivalentAddressGroup(backends1.get(0).addr, LB_BACKEND_ATTRS), subchannel1.getAddresses());
    assertEquals(new EquivalentAddressGroup(backends1.get(1).addr, LB_BACKEND_ATTRS), subchannel2.getAddresses());
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(CONNECTING));
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
    RoundRobinPicker picker0 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker0.dropList).containsExactly(null, null);
    assertThat(picker0.pickList).containsExactly(BUFFER_ENTRY);
    inOrder.verifyNoMoreInteractions();
    assertThat(logs).containsExactly("DEBUG: [grpclb-<api.google.com>] Got an LB response: " + buildLbResponse(backends1)).inOrder();
    logs.clear();
    // Let subchannels be connected
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker1 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker1.dropList).containsExactly(null, null);
    assertThat(picker1.pickList).containsExactly(new BackendEntry(subchannel2, getLoadRecorder(), "token0002"));
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker2 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker2.dropList).containsExactly(null, null);
    assertThat(picker2.pickList).containsExactly(new BackendEntry(subchannel1, getLoadRecorder(), "token0001"), new BackendEntry(subchannel2, getLoadRecorder(), "token0002")).inOrder();
    // Disconnected subchannels
    verify(subchannel1).requestConnection();
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(IDLE));
    verify(subchannel1, times(2)).requestConnection();
    inOrder.verify(helper).refreshNameResolution();
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker3 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker3.dropList).containsExactly(null, null);
    assertThat(picker3.pickList).containsExactly(new BackendEntry(subchannel2, getLoadRecorder(), "token0002"));
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(CONNECTING));
    inOrder.verifyNoMoreInteractions();
    // As long as there is at least one READY subchannel, round robin will work.
    ConnectivityStateInfo errorState1 = ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE.withDescription("error1"));
    deliverSubchannelState(subchannel1, errorState1);
    inOrder.verify(helper).refreshNameResolution();
    inOrder.verifyNoMoreInteractions();
    // If no subchannel is READY, some with error and the others are IDLE, will report CONNECTING
    verify(subchannel2).requestConnection();
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(IDLE));
    verify(subchannel2, times(2)).requestConnection();
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
    RoundRobinPicker picker4 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker4.dropList).containsExactly(null, null);
    assertThat(picker4.pickList).containsExactly(BUFFER_ENTRY);
    // Update backends, with a drop entry
    List<ServerEntry> backends2 = Arrays.asList(// New address
    new ServerEntry("127.0.0.1", 2030, "token0003"), // drop
    new ServerEntry("token0003"), // Existing address with token changed
    new ServerEntry("127.0.0.1", 2010, "token0004"), // New address appearing second time
    new ServerEntry("127.0.0.1", 2030, "token0005"), // drop
    new ServerEntry("token0006"));
    verify(subchannelPool, never()).returnSubchannel(same(subchannel1), any(ConnectivityStateInfo.class));
    lbResponseObserver.onNext(buildLbResponse(backends2));
    assertThat(logs).containsExactly("DEBUG: [grpclb-<api.google.com>] Got an LB response: " + buildLbResponse(backends2)).inOrder();
    logs.clear();
    // not in backends2, closed
    verify(subchannelPool).returnSubchannel(same(subchannel1), same(errorState1));
    // backends2[2], will be kept
    verify(subchannelPool, never()).returnSubchannel(same(subchannel2), any(ConnectivityStateInfo.class));
    inOrder.verify(subchannelPool, never()).takeOrCreateSubchannel(eq(new EquivalentAddressGroup(backends2.get(2).addr, LB_BACKEND_ATTRS)), any(Attributes.class));
    inOrder.verify(subchannelPool).takeOrCreateSubchannel(eq(new EquivalentAddressGroup(backends2.get(0).addr, LB_BACKEND_ATTRS)), any(Attributes.class));
    ConnectivityStateInfo errorOnCachedSubchannel1 = ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE.withDescription("You can get this error even if you are cached"));
    deliverSubchannelState(subchannel1, errorOnCachedSubchannel1);
    assertEquals(1, mockSubchannels.size());
    Subchannel subchannel3 = mockSubchannels.poll();
    verify(subchannel3).requestConnection();
    assertEquals(new EquivalentAddressGroup(backends2.get(0).addr, LB_BACKEND_ATTRS), subchannel3.getAddresses());
    inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
    RoundRobinPicker picker7 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker7.dropList).containsExactly(null, new DropEntry(getLoadRecorder(), "token0003"), null, null, new DropEntry(getLoadRecorder(), "token0006")).inOrder();
    assertThat(picker7.pickList).containsExactly(BUFFER_ENTRY);
    // State updates on obsolete subchannel1 will only be passed to the pool
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(READY));
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE));
    deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(SHUTDOWN));
    deliverSubchannelState(subchannel3, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker8 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker8.dropList).containsExactly(null, new DropEntry(getLoadRecorder(), "token0003"), null, null, new DropEntry(getLoadRecorder(), "token0006")).inOrder();
    // subchannel2 is still IDLE, thus not in the active list
    assertThat(picker8.pickList).containsExactly(new BackendEntry(subchannel3, getLoadRecorder(), "token0003"), new BackendEntry(subchannel3, getLoadRecorder(), "token0005")).inOrder();
    // subchannel2 becomes READY and makes it into the list
    deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(READY));
    inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
    RoundRobinPicker picker9 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker9.dropList).containsExactly(null, new DropEntry(getLoadRecorder(), "token0003"), null, null, new DropEntry(getLoadRecorder(), "token0006")).inOrder();
    assertThat(picker9.pickList).containsExactly(new BackendEntry(subchannel3, getLoadRecorder(), "token0003"), new BackendEntry(subchannel2, getLoadRecorder(), "token0004"), new BackendEntry(subchannel3, getLoadRecorder(), "token0005")).inOrder();
    verify(subchannelPool, never()).returnSubchannel(same(subchannel3), any(ConnectivityStateInfo.class));
    // Update backends, with no entry
    lbResponseObserver.onNext(buildLbResponse(Collections.<ServerEntry>emptyList()));
    verify(subchannelPool).returnSubchannel(same(subchannel2), eq(ConnectivityStateInfo.forNonError(READY)));
    verify(subchannelPool).returnSubchannel(same(subchannel3), eq(ConnectivityStateInfo.forNonError(READY)));
    inOrder.verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture());
    RoundRobinPicker picker10 = (RoundRobinPicker) pickerCaptor.getValue();
    assertThat(picker10.dropList).isEmpty();
    assertThat(picker10.pickList).containsExactly(new ErrorEntry(GrpclbState.NO_AVAILABLE_BACKENDS_STATUS));
    assertFalse(oobChannel.isShutdown());
    assertEquals(0, lbRequestObservers.size());
    verify(lbRequestObserver, never()).onCompleted();
    verify(lbRequestObserver, never()).onError(any(Throwable.class));
    // Load reporting was not requested, thus never scheduled
    assertEquals(0, fakeClock.numPendingTasks(LOAD_REPORTING_TASK_FILTER));
    verify(subchannelPool, never()).clear();
    balancer.shutdown();
    verify(subchannelPool).clear();
}