Examples with Discoverable org.apache.twill.discovery.Discoverable used on opensource projects

Example 26 with Discoverable

use of org.apache.twill.discovery.Discoverable in project cdap by caskdata.

the class ResourceCoordinatorTest method testAssignment.

@Test
public void testAssignment() throws InterruptedException, ExecutionException {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    String serviceName = "test-assignment";
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new DiscoveryRuntimeModule().getDistributedModules());
    ZKClientService zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class);
    try {
        ResourceCoordinator coordinator = new ResourceCoordinator(zkClient, injector.getInstance(DiscoveryServiceClient.class), new BalancedAssignmentStrategy());
        coordinator.startAndWait();
        try {
            ResourceCoordinatorClient client = new ResourceCoordinatorClient(zkClient);
            client.startAndWait();
            try {
                // Create a requirement
                ResourceRequirement requirement = ResourceRequirement.builder(serviceName).addPartitions("p", 5, 1).build();
                client.submitRequirement(requirement).get();
                // Fetch the requirement, just to verify it's the same as the one get submitted.
                Assert.assertEquals(requirement, client.fetchRequirement(requirement.getName()).get());
                // Register a discovery endpoint
                final Discoverable discoverable1 = createDiscoverable(serviceName, 10000);
                Cancellable cancelDiscoverable1 = discoveryService.register(ResolvingDiscoverable.of(discoverable1));
                // Add a change handler for this discoverable.
                final BlockingQueue<Collection<PartitionReplica>> assignmentQueue = new SynchronousQueue<>();
                final Semaphore finishSemaphore = new Semaphore(0);
                Cancellable cancelSubscribe1 = subscribe(client, discoverable1, assignmentQueue, finishSemaphore);
                // Assert that it received the changes.
                Collection<PartitionReplica> assigned = assignmentQueue.poll(30, TimeUnit.SECONDS);
                Assert.assertNotNull(assigned);
                Assert.assertEquals(5, assigned.size());
                // Unregister from discovery, the handler should receive a change with empty collection
                cancelDiscoverable1.cancel();
                Assert.assertTrue(assignmentQueue.poll(30, TimeUnit.SECONDS).isEmpty());
                // Register to discovery again, would receive changes.
                cancelDiscoverable1 = discoveryService.register(ResolvingDiscoverable.of(discoverable1));
                assigned = assignmentQueue.poll(30, TimeUnit.SECONDS);
                Assert.assertNotNull(assigned);
                Assert.assertEquals(5, assigned.size());
                // Register another discoverable
                final Discoverable discoverable2 = createDiscoverable(serviceName, 10001);
                Cancellable cancelDiscoverable2 = discoveryService.register(ResolvingDiscoverable.of(discoverable2));
                // Changes should be received by the handler, with only 3 resources,
                // as 2 out of 5 should get moved to the new discoverable.
                assigned = assignmentQueue.poll(30, TimeUnit.SECONDS);
                Assert.assertNotNull(assigned);
                Assert.assertEquals(3, assigned.size());
                // Cancel the first discoverable again, should expect empty result.
                // This also make sure the latest assignment get cached in the ResourceCoordinatorClient.
                // It is the the next test step.
                cancelDiscoverable1.cancel();
                Assert.assertTrue(assignmentQueue.poll(30, TimeUnit.SECONDS).isEmpty());
                // Cancel the handler.
                cancelSubscribe1.cancel();
                Assert.assertTrue(finishSemaphore.tryAcquire(2, TimeUnit.SECONDS));
                // Subscribe to changes for the second discoverable,
                // it should see the latest assignment, even though no new fetch from ZK is triggered.
                Cancellable cancelSubscribe2 = subscribe(client, discoverable2, assignmentQueue, finishSemaphore);
                assigned = assignmentQueue.poll(30, TimeUnit.SECONDS);
                Assert.assertNotNull(assigned);
                Assert.assertEquals(5, assigned.size());
                // Update the requirement to be an empty requirement, the handler should receive an empty collection
                client.submitRequirement(ResourceRequirement.builder(serviceName).build());
                Assert.assertTrue(assignmentQueue.poll(30, TimeUnit.SECONDS).isEmpty());
                // Update the requirement to have one partition, the handler should receive one resource
                client.submitRequirement(ResourceRequirement.builder(serviceName).addPartitions("p", 1, 1).build());
                assigned = assignmentQueue.poll(30, TimeUnit.SECONDS);
                Assert.assertNotNull(assigned);
                Assert.assertEquals(1, assigned.size());
                // Delete the requirement, the handler should receive a empty collection
                client.deleteRequirement(requirement.getName());
                Assert.assertTrue(assignmentQueue.poll(30, TimeUnit.SECONDS).isEmpty());
                // Cancel the second handler.
                cancelSubscribe2.cancel();
                Assert.assertTrue(finishSemaphore.tryAcquire(2, TimeUnit.SECONDS));
                cancelDiscoverable2.cancel();
            } finally {
                client.stopAndWait();
            }
        } finally {
            coordinator.stopAndWait();
        }
    } finally {
        zkClient.stopAndWait();
    }
}

Example 27 with Discoverable

use of org.apache.twill.discovery.Discoverable in project cdap by caskdata.

the class ResourceBalancerServiceTest method testResourceBalancerService.

@Test
public void testResourceBalancerService() throws Exception {
    // Simple test for resource balancer does react to discovery changes correct
    // More detailed tests are in ResourceCoordinatorTest, which the ResourceBalancerService depends on
    ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    zkClient.startAndWait();
    try (ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient)) {
        // Test the failure on stop case
        final TestBalancerService stopFailureService = new TestBalancerService("test", 4, zkClient, discoveryService, discoveryService, false, false);
        stopFailureService.startAndWait();
        // Should get all four partitions
        Tasks.waitFor(ImmutableSet.of(0, 1, 2, 3), new Callable<Set<Integer>>() {

            @Override
            public Set<Integer> call() throws Exception {
                return stopFailureService.getPartitions();
            }
        }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Register a new discoverable, this should trigger a partition change in the resource balancer service
        Cancellable cancellable = discoveryService.register(new Discoverable("test", new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234)));
        try {
            // Should get reduced to two partitions
            Tasks.waitFor(2, new Callable<Integer>() {

                @Override
                public Integer call() throws Exception {
                    return stopFailureService.getPartitions().size();
                }
            }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        } finally {
            cancellable.cancel();
        }
    } finally {
        zkClient.stopAndWait();
    }
}

Example 28 with Discoverable

use of org.apache.twill.discovery.Discoverable in project cdap by caskdata.

the class ResourceBalancerServiceTest method testServiceStopFailure.

@Test
public void testServiceStopFailure() throws Exception {
    ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    zkClient.startAndWait();
    try (ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient)) {
        // Test the failure on stop case
        final TestBalancerService stopFailureService = new TestBalancerService("test", 4, zkClient, discoveryService, discoveryService, false, true);
        stopFailureService.startAndWait();
        // Should get four partitions
        Tasks.waitFor(ImmutableSet.of(0, 1, 2, 3), new Callable<Set<Integer>>() {

            @Override
            public Set<Integer> call() throws Exception {
                return stopFailureService.getPartitions();
            }
        }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Register a new discoverable, this should trigger a partition change in the resource balancer service
        Cancellable cancellable = discoveryService.register(new Discoverable("test", new InetSocketAddress(InetAddress.getLoopbackAddress(), 1234)));
        try {
            // When there is exception thrown by the underlying service during partition change,
            // the resource balancer service should fail.
            Tasks.waitFor(Service.State.FAILED, new Callable<Service.State>() {

                @Override
                public Service.State call() throws Exception {
                    return stopFailureService.state();
                }
            }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        } finally {
            cancellable.cancel();
        }
    } finally {
        zkClient.stopAndWait();
    }
}

Example 29 with Discoverable

use of org.apache.twill.discovery.Discoverable in project cdap by caskdata.

the class UserServiceEndpointStrategyTest method testStrategy.

@Test
public void testStrategy() throws Exception {
    ProgramId serviceId = new ApplicationId("n1", "a1").service("s1");
    String discoverableName = ServiceDiscoverable.getName(serviceId);
    List<Discoverable> candidates = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        candidates.add(new Discoverable(discoverableName, null, Bytes.toBytes(Integer.toString(i))));
    }
    SimpleServiceDiscovered serviceDiscovered = new SimpleServiceDiscovered(candidates);
    Map<String, Integer> routeToVersion = ImmutableMap.of("2", 100);
    Map<ProgramId, RouteConfig> routeMap = ImmutableMap.of(serviceId, new RouteConfig(routeToVersion));
    RouteStore configStore = new InMemoryRouteStore(routeMap);
    UserServiceEndpointStrategy strategy = new UserServiceEndpointStrategy(serviceDiscovered, configStore, serviceId);
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("2", Bytes.toString(picked.getPayload()));
    }
    // Switch config to choose version 3 always
    routeToVersion = ImmutableMap.of("3", 100);
    configStore.store(serviceId, new RouteConfig(routeToVersion));
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("3", Bytes.toString(picked.getPayload()));
    }
    // Switch config to choose verion 1 and 4 - 50% each
    routeToVersion = ImmutableMap.of("1", 50, "4", 50);
    configStore.store(serviceId, new RouteConfig(routeToVersion));
    Map<String, Integer> resultMap = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        Discoverable picked = strategy.pick();
        String version = Bytes.toString(picked.getPayload());
        if (resultMap.containsKey(version)) {
            resultMap.put(version, resultMap.get(version) + 1);
        } else {
            resultMap.put(version, 1);
        }
    }
    Assert.assertEquals(2, resultMap.size());
    double requestsToOne = resultMap.get("1");
    double requestsToTwo = resultMap.get("4");
    double requestRatio = requestsToOne / requestsToTwo;
    // Request Ratio should be close to 1.0 since we expect 50% of requests to go to each of these versions
    Assert.assertTrue(String.format("RequestRatio was %f and 1 got %f and 4 got %f", requestRatio, requestsToOne, requestsToTwo), requestRatio >= 0.7);
    Assert.assertTrue(String.format("RequestRatio was %f and 1 got %f and 4 got %f", requestRatio, requestsToOne, requestsToTwo), requestRatio <= 1.3);
    // Set the payload filter
    strategy = new UserServiceEndpointStrategy(serviceDiscovered, configStore, serviceId, null, "1");
    for (int i = 0; i < 1000; i++) {
        Discoverable picked = strategy.pick();
        Assert.assertEquals("1", Bytes.toString(picked.getPayload()));
    }
}

Example 30 with Discoverable

use of org.apache.twill.discovery.Discoverable in project cdap by caskdata.

the class HttpRequestHandler method getDiscoverable.

private WrappedDiscoverable getDiscoverable(final HttpRequest httpRequest, final InetSocketAddress address) {
    EndpointStrategy strategy = serviceLookup.getDiscoverable(address.getPort(), httpRequest);
    if (strategy == null) {
        throw new HandlerException(HttpResponseStatus.SERVICE_UNAVAILABLE, String.format("No endpoint strategy found for request : %s", httpRequest.getUri()));
    }
    Discoverable discoverable = strategy.pick();
    if (discoverable == null) {
        throw new HandlerException(HttpResponseStatus.SERVICE_UNAVAILABLE, String.format("No discoverable found for request : %s", httpRequest.getUri()));
    }
    return new WrappedDiscoverable(discoverable);
}