Examples with ExternalView org.apache.helix.model.ExternalView used on opensource projects

Example 21 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class RoutingTableTest method testCombinedKafkaRouting.

// Test that we can switch between llc and hlc routing depending on what the selector tells us.
@Test
public void testCombinedKafkaRouting() throws Exception {
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    final long now = System.currentTimeMillis();
    final String tableName = "table";
    final String resourceName = tableName + "_REALTIME";
    final String group1 = resourceName + "_" + Long.toString(now) + "_0";
    final String group2 = resourceName + "_" + Long.toString(now) + "_1";
    final String online = "ONLINE";
    final String consuming = "CONSUMING";
    final int partitionId = 1;
    final String partitionRange = "JUNK";
    final int segId1 = 1;
    final int segId2 = 2;
    final int port1 = 1;
    final int port2 = 2;
    final String host = "host";
    final ServerInstance serverInstance1 = new ServerInstance(host, port1);
    final ServerInstance serverInstance2 = new ServerInstance(host, port2);
    final String helixInstance1 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance1;
    final String helixInstance2 = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + serverInstance2;
    final HLCSegmentName s1HlcSegment1 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s1HlcSegment2 = new HLCSegmentName(group1, partitionRange, Integer.toString(segId2));
    final HLCSegmentName s2HlcSegment1 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId1));
    final HLCSegmentName s2HlcSegment2 = new HLCSegmentName(group2, partitionRange, Integer.toString(segId2));
    final LLCSegmentName llcSegment1 = new LLCSegmentName(tableName, partitionId, segId1, now);
    final LLCSegmentName llcSegment2 = new LLCSegmentName(tableName, partitionId, segId2, now);
    final List<InstanceConfig> instanceConfigs = new ArrayList<>(2);
    instanceConfigs.add(new InstanceConfig(helixInstance1));
    instanceConfigs.add(new InstanceConfig(helixInstance2));
    ExternalView ev = new ExternalView(resourceName);
    ev.setState(s1HlcSegment1.getSegmentName(), helixInstance1, online);
    ev.setState(s1HlcSegment2.getSegmentName(), helixInstance1, online);
    ev.setState(llcSegment1.getSegmentName(), helixInstance2, online);
    ev.setState(llcSegment2.getSegmentName(), helixInstance2, consuming);
    routingTable.markDataResourceOnline(resourceName, ev, instanceConfigs);
    RoutingTableLookupRequest request = new RoutingTableLookupRequest(resourceName, Collections.<String>emptyList());
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
    }
    // Now change the percent value in the routing table selector to be 100, and we should get only LLC segments.
    Configuration configuration = new PropertiesConfiguration();
    configuration.addProperty("class", PercentageBasedRoutingTableSelector.class.getName());
    configuration.addProperty("table." + resourceName, new Integer(100));
    RoutingTableSelector selector = RoutingTableSelectorFactory.getRoutingTableSelector(configuration, null);
    selector.init(configuration, null);
    Field selectorField = HelixExternalViewBasedRouting.class.getDeclaredField("_routingTableSelector");
    selectorField.setAccessible(true);
    selectorField.set(routingTable, selector);
    // And we should find only LLC segments.
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
        Assert.assertEquals(segments.size(), 2);
        Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
        Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
    }
    // Now change it to 50, and we should find both (at least 10 times each).
    configuration = new PropertiesConfiguration();
    configuration.addProperty("table." + resourceName, new Integer(50));
    selector = new PercentageBasedRoutingTableSelector();
    selector.init(configuration, null);
    selectorField.set(routingTable, selector);
    int hlc = 0;
    int llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    // If we do the above iteration 100 times, we should get at least 10 of each type of routing.
    // If this test fails
    Assert.assertTrue(hlc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    Assert.assertTrue(llc >= 10, "Got low values hlc=" + hlc + ",llc=" + llc);
    // Check that force HLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_HLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 100);
    Assert.assertEquals(llc, 0);
    // Check that force LLC works
    request = new RoutingTableLookupRequest(resourceName, Collections.singletonList("FORCE_LLC"));
    hlc = 0;
    llc = 0;
    for (int i = 0; i < 100; i++) {
        Map<ServerInstance, SegmentIdSet> routingMap = routingTable.findServers(request);
        Assert.assertEquals(routingMap.size(), 1);
        if (routingMap.containsKey(serverInstance2)) {
            List<String> segments = routingMap.get(serverInstance2).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(llcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(llcSegment2.getSegmentName()));
            llc++;
        } else {
            List<String> segments = routingMap.get(serverInstance1).getSegmentsNameList();
            Assert.assertEquals(segments.size(), 2);
            Assert.assertTrue(segments.contains(s1HlcSegment1.getSegmentName()));
            Assert.assertTrue(segments.contains(s1HlcSegment2.getSegmentName()));
            hlc++;
        }
    }
    Assert.assertEquals(hlc, 0);
    Assert.assertEquals(llc, 100);
}

Example 22 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class BalancedRandomRoutingTableBuilderTest method isRandom.

@Test
public void isRandom() {
    // Build dummy external view
    BalancedRandomRoutingTableBuilder routingTableBuilder = new BalancedRandomRoutingTableBuilder();
    List<InstanceConfig> instanceConfigList = new ArrayList<>();
    ExternalView externalView = new ExternalView("dummy");
    externalView.setState("segment_1", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_1", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_1", "Server_1.2.3.6_3456", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_2", "Server_1.2.3.6_3456", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.4_1234", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.5_2345", "ONLINE");
    externalView.setState("segment_3", "Server_1.2.3.6_3456", "ONLINE");
    // Create configs for above instances.
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.4_1234"));
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.5_2345"));
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.6_3456"));
    // Build routing table
    List<ServerToSegmentSetMap> routingTable = routingTableBuilder.computeRoutingTableFromExternalView("dummy", externalView, instanceConfigList);
    // Check that at least two routing tables are different
    Iterator<ServerToSegmentSetMap> routingTableIterator = routingTable.iterator();
    ServerToSegmentSetMap previous = routingTableIterator.next();
    while (routingTableIterator.hasNext()) {
        ServerToSegmentSetMap current = routingTableIterator.next();
        //      System.out.println("previous = " + previous);
        if (!current.equals(previous)) {
            // Routing tables differ, test is successful
            return;
        }
    }
    Assert.fail("All routing tables are equal!");
}

Example 23 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testMultipleConsumingSegments.

@Test
public void testMultipleConsumingSegments() {
    final int SEGMENT_COUNT = 10;
    final int ONLINE_SEGMENT_COUNT = 8;
    final int CONSUMING_SEGMENT_COUNT = SEGMENT_COUNT - ONLINE_SEGMENT_COUNT;
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    List<SegmentName> segmentNames = new ArrayList<SegmentName>();
    for (int i = 0; i < SEGMENT_COUNT; ++i) {
        segmentNames.add(new LLCSegmentName("table", 0, i, System.currentTimeMillis()));
    }
    List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
    InstanceConfig instanceConfig = new InstanceConfig("Server_localhost_1234");
    instanceConfigs.add(instanceConfig);
    instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
    // Generate an external view for a single server with some consuming segments
    ExternalView externalView = new ExternalView("table_REALTIME");
    for (int i = 0; i < ONLINE_SEGMENT_COUNT; i++) {
        externalView.setState(segmentNames.get(i).getSegmentName(), "Server_localhost_1234", "ONLINE");
    }
    for (int i = ONLINE_SEGMENT_COUNT; i < SEGMENT_COUNT; ++i) {
        externalView.setState(segmentNames.get(i).getSegmentName(), "Server_localhost_1234", "CONSUMING");
    }
    List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table", externalView, instanceConfigs);
    for (ServerToSegmentSetMap routingTable : routingTables) {
        for (String server : routingTable.getServerSet()) {
            Set<String> segmentSet = routingTable.getSegmentSet(server);
            assertEquals(segmentSet.size(), ONLINE_SEGMENT_COUNT + 1, "");
            // Should only contain the first consuming segment, not the second
            assertTrue(segmentSet.contains(segmentNames.get(ONLINE_SEGMENT_COUNT).getSegmentName()), "Segment set does not contain the first segment in consuming state");
            for (int i = ONLINE_SEGMENT_COUNT + 1; i < SEGMENT_COUNT; i++) {
                assertFalse(segmentSet.contains(segmentNames.get(i).getSegmentName()), "Segment set contains a segment in consuming state that should not be there");
            }
        }
    }
}

Example 24 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableSizeGenerallyHasConfiguredServerCount.

@Test
public void testRoutingTableSizeGenerallyHasConfiguredServerCount() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 100;
    final int replicationFactor = 10;
    final int instanceCount = 50;
    final int desiredServerCount = 20;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    List<ServerToSegmentSetMap> routingTables = _largeClusterRoutingTableBuilder.computeRoutingTableFromExternalView(tableName, externalView, instanceConfigs);
    int routingTableCount = 0;
    int largerThanDesiredRoutingTableCount = 0;
    for (ServerToSegmentSetMap routingTable : routingTables) {
        routingTableCount++;
        if (desiredServerCount < routingTable.getServerSet().size()) {
            largerThanDesiredRoutingTableCount++;
        }
    }
    assertTrue(largerThanDesiredRoutingTableCount / 0.6 < routingTableCount, "More than 60% of routing tables exceed the desired routing table size, RANDOM_SEED = " + RANDOM_SEED);
}

Example 25 with ExternalView

use of org.apache.helix.model.ExternalView in project pinot by linkedin.

the class LargeClusterRoutingTableBuilderTest method testRoutingTableExcludesDisabledAndRebootingInstances.

@Test
public void testRoutingTableExcludesDisabledAndRebootingInstances() {
    final String tableName = "fakeTable_OFFLINE";
    final int segmentCount = 100;
    final int replicationFactor = 6;
    final int instanceCount = 50;
    ExternalView externalView = createExternalView(tableName, segmentCount, replicationFactor, instanceCount);
    List<InstanceConfig> instanceConfigs = createInstanceConfigs(instanceCount);
    final InstanceConfig disabledHelixInstance = instanceConfigs.get(0);
    final String disabledHelixInstanceName = disabledHelixInstance.getInstanceName();
    disabledHelixInstance.setInstanceEnabled(false);
    final InstanceConfig shuttingDownInstance = instanceConfigs.get(1);
    final String shuttingDownInstanceName = shuttingDownInstance.getInstanceName();
    shuttingDownInstance.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, Boolean.toString(true));
    validateAssertionForOneRoutingTable(new RoutingTableValidator() {

        @Override
        public boolean isRoutingTableValid(ServerToSegmentSetMap routingTable, ExternalView externalView, List<InstanceConfig> instanceConfigs) {
            for (String server : routingTable.getServerSet()) {
                // These servers should not appear in the routing table
                if (server.equals(disabledHelixInstanceName) || server.equals(shuttingDownInstanceName)) {
                    return false;
                }
            }
            return true;
        }
    }, "Routing table should not contain disabled instances", externalView, instanceConfigs, tableName);
}